+2012-02-10 Ryosuke Niwa <rniwa@webkit.org>
+
+ Perf-o-matic should process reports in background
+ https://bugs.webkit.org/show_bug.cgi?id=78309
+
+ Reviewed by Hajime Morita.
+
+ Split the logic to create Build, Test, and TestResult objects from ReportHandler into ReportProcessHandler.
+ ReportHandler now merely creates ReportLog and schedules a task to process it.
+
+ Also added ReportLogHandler to manage stale ReportLogs.
+
+ * Websites/webkit-perf.appspot.com/app.yaml:
+ * Websites/webkit-perf.appspot.com/controller.py:
+ (schedule_manifest_update):
+ (schedule_dashboard_update):
+ (schedule_runs_update):
+ (CachedRunsHandler.get):
+ (schedule_report_process):
+ * Websites/webkit-perf.appspot.com/main.py:
+ * Websites/webkit-perf.appspot.com/merge_tests.html: Renamed from Websites/webkit-perf.appspot.com/merge_tests.yaml.
+ * Websites/webkit-perf.appspot.com/models.py:
+ (ReportLog):
+ (ReportLog._parsed_payload):
+ (ReportLog.get_value):
+ (ReportLog.results):
+ (ReportLog.builder):
+ (ReportLog.branch):
+ (ReportLog.platform):
+ (ReportLog.build_number):
+ (ReportLog.webkit_revision):
+ (ReportLog.chromium_revision):
+ (ReportLog._model_by_key_name_in_payload):
+ (ReportLog._integer_in_payload):
+ (ReportLog.timestamp):
+ * Websites/webkit-perf.appspot.com/report_handler.py:
+ (ReportHandler.post):
+ (ReportHandler._output):
+ (ReportHandler._results_are_valid):
+ (ReportHandler._results_are_valid._is_float_convertible):
+ (ReportHandler):
+ * Websites/webkit-perf.appspot.com/report_logs.html: Added.
+ * Websites/webkit-perf.appspot.com/report_logs_handler.py: Added.
+ (ReportLogsHandler):
+ (ReportLogsHandler.get):
+ (ReportLogsHandler.post):
+ (ReportLogsHandler._error):
+ * Websites/webkit-perf.appspot.com/report_process_handler.py: Copied from Websites/webkit-perf.appspot.com/report_handler.py.
+ (ReportProcessHandler):
+ (ReportProcessHandler.post):
+ (ReportProcessHandler._create_build_if_possible):
+ (ReportProcessHandler._create_build_if_possible.execute):
+ (ReportProcessHandler._add_test_if_needed):
+
2012-02-09 Ryosuke Niwa <rniwa@webkit.org>
Perf-o-matic shouldn't rely on memcache to store cached JSON responses
application: webkit-perf
-version: 12
+version: 13
runtime: python27
api_version: 1
threadsafe: false
def schedule_manifest_update():
- taskqueue.add(url='/api/test/update')
+ taskqueue.add(url='/api/test/update', name='manifest_update')
class CachedManifestHandler(webapp2.RequestHandler):
def schedule_dashboard_update():
- taskqueue.add(url='/api/test/dashboard/update')
+ taskqueue.add(url='/api/test/dashboard/update', name='dashboard_update')
class CachedDashboardHandler(webapp2.RequestHandler):
def schedule_runs_update(test_id, branch_id, platform_id):
- taskqueue.add(url='/api/test/runs/update', params={'id': test_id, 'branchid': branch_id, 'platformid': platform_id})
+ taskqueue.add(url='/api/test/runs/update', name='runs_update_%d_%d_%d' % (test_id, branch_id, platform_id),
+ params={'id': test_id, 'branchid': branch_id, 'platformid': platform_id})
class CachedRunsHandler(webapp2.RequestHandler):
self.response.out.write(runs)
else:
schedule_runs_update(test_id, branch_id, platform_id)
+
+
+def schedule_report_process(log):
+ taskqueue.add(url='/api/test/report/process', params={'id': log.key().id()})
from manifest_handler import ManifestHandler
from report_handler import ReportHandler
from report_handler import AdminReportHandler
+from report_process_handler import ReportProcessHandler
+from report_logs_handler import ReportLogsHandler
from runs_handler import RunsHandler
from merge_tests_handler import MergeTestsHandler
routes = [
('/admin/report/?', AdminReportHandler),
('/admin/merge-tests/?', MergeTestsHandler),
+ ('/admin/report-logs/?', ReportLogsHandler),
('/admin/create/(.*)', CreateHandler),
('/api/test/?', CachedManifestHandler),
('/api/test/update', ManifestHandler),
('/api/test/report/?', ReportHandler),
+ ('/api/test/report/process', ReportProcessHandler),
('/api/test/runs/?', CachedRunsHandler),
('/api/test/runs/update', RunsHandler),
('/api/test/dashboard/?', CachedDashboardHandler),
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
import hashlib
+import json
import re
+from datetime import datetime
from google.appengine.ext import db
timestamp = db.DateTimeProperty(required=True)
headers = db.TextProperty()
payload = db.TextProperty()
+ commit = db.BooleanProperty()
+
+ def _parsed_payload(self):
+ if self.__dict__.get('_parsed') == None:
+ try:
+ self._parsed = json.loads(self.payload)
+ except ValueError:
+ self._parsed = False
+ return self._parsed
+
+ def get_value(self, keyName):
+ if not self._parsed_payload():
+ return None
+ return self._parsed.get(keyName, '')
+
+ def results(self):
+ return self.get_value('results')
+
+ def builder(self):
+ return self._model_by_key_name_in_payload(Builder, 'builder-name')
+
+ def branch(self):
+ return self._model_by_key_name_in_payload(Branch, 'branch')
+
+ def platform(self):
+ return self._model_by_key_name_in_payload(Platform, 'platform')
+
+ def build_number(self):
+ return self._integer_in_payload('build-number')
+
+ def webkit_revision(self):
+ return self._integer_in_payload('webkit-revision')
+
+ def chromium_revision(self):
+ return self._integer_in_payload('chromium-revision')
+
+ def _model_by_key_name_in_payload(self, model, keyName):
+ key = self.get_value(keyName)
+ if not key:
+ return None
+ return model.get_by_key_name(key)
+
+ def _integer_in_payload(self, keyName):
+ try:
+ return int(self.get_value(keyName))
+ except ValueError:
+ return None
+
+ def timestamp(self):
+ try:
+ return datetime.fromtimestamp(self._integer_in_payload('timestamp'))
+ except TypeError:
+ return None
+ except ValueError:
+ return None
# Used when memcache entry is evicted
import json
import re
-import time
from datetime import datetime
-from controller import schedule_runs_update
-from controller import schedule_dashboard_update
-from controller import schedule_manifest_update
-from models import Builder
-from models import Branch
-from models import Build
-from models import NumericIdHolder
-from models import Platform
from models import ReportLog
-from models import Test
-from models import TestResult
-from models import create_in_transaction_with_numeric_id_holder
+from controller import schedule_report_process
class ReportHandler(webapp2.RequestHandler):
log = ReportLog(timestamp=datetime.now(), headers=headers, payload=request_body_without_password)
log.put()
+ self._encountered_error = False
+
try:
- self._body = json.loads(self.request.body)
+ parsedPayload = json.loads(self.request.body)
+ password = parsedPayload.get('password', '')
except ValueError:
return self._output('Failed to parse the payload as a json. Report key: %d' % log.key().id())
- builder = self._model_by_key_name_in_body_or_error(Builder, 'builder-name')
- branch = self._model_by_key_name_in_body_or_error(Branch, 'branch')
- platform = self._model_by_key_name_in_body_or_error(Platform, 'platform')
- build_number = self._integer_in_body('build-number')
- timestamp = self._timestamp_in_body()
- revision = self._integer_in_body('webkit-revision')
- chromium_revision = self._integer_in_body('webkit-revision') if 'chromium-revision' in self._body else None
+ builder = log.builder()
+ builder != None or self._output('No builder named "%s"' % log.get_value('builder-name'))
+ log.branch() != None or self._output('No branch named "%s"' % log.get_value('branch'))
+ log.platform() != None or self._output('No platform named "%s"' % log.get_value('platform'))
+ log.build_number() != None or self._output('Invalid build number "%s"' % log.get_value('build-number'))
+ log.timestamp() != None or self._output('Invalid timestamp "%s"' % log.get_value('timestamp'))
+ log.webkit_revision() != None or self._output('Invalid webkit revision "%s"' % log.get_value('webkit-revision'))
failed = False
- if builder and not (self.bypass_authentication() or builder.authenticate(self._body.get('password', ''))):
+ if builder and not (self.bypass_authentication() or builder.authenticate(password)):
self._output('Authentication failed')
- failed = True
- if not self._results_are_valid():
+ if not self._results_are_valid(log):
self._output("The payload doesn't contain results or results are malformed")
- failed = True
-
- if not (builder and branch and platform and build_number and revision and timestamp) or failed:
- return
- build = self._create_build_if_possible(builder, build_number, branch, platform, timestamp, revision, chromium_revision)
- if not build:
+ if self._encountered_error:
return
- def _float_or_none(dictionary, key):
- value = dictionary.get(key)
- if value:
- return float(value)
- return None
-
- for test_name, result in self._body['results'].iteritems():
- test = self._add_test_if_needed(test_name, branch, platform)
- schedule_runs_update(test.id, branch.id, platform.id)
- if isinstance(result, dict):
- TestResult(name=test_name, build=build, value=float(result['avg']), valueMedian=_float_or_none(result, 'median'),
- valueStdev=_float_or_none(result, 'stdev'), valueMin=_float_or_none(result, 'min'), valueMax=_float_or_none(result, 'max')).put()
- else:
- TestResult(name=test_name, build=build, value=float(result)).put()
-
- log = ReportLog.get(log.key())
- log.delete()
-
- # We need to update dashboard and manifest because they are affected by the existance of test results
- schedule_dashboard_update()
- schedule_manifest_update()
-
- return self._output('OK')
-
- def _model_by_key_name_in_body_or_error(self, model, keyName):
- key = self._body.get(keyName, '')
- instance = key and model.get_by_key_name(key)
- if not instance:
- self._output('There are no %s named "%s"' % (model.__name__.lower(), key))
- return instance
-
- def _integer_in_body(self, key):
- value = self._body.get(key, '')
- try:
- return int(value)
- except:
- return self._output('Invalid %s: "%s"' % (key.replace('-', ' '), value))
-
- def _timestamp_in_body(self):
- value = self._body.get('timestamp', '')
- try:
- return datetime.fromtimestamp(int(value))
- except:
- return self._output('Failed to parse the timestamp: %s' % value)
+ log.commit = True
+ log.put()
+ schedule_report_process(log)
+ self._output("OK")
def _output(self, message):
+ self._encountered_error = True
self.response.out.write(message + '\n')
def bypass_authentication(self):
return False
- def _results_are_valid(self):
+ def _results_are_valid(self, log):
def _is_float_convertible(value):
try:
return True
except TypeError:
return False
+ except ValueError:
+ return False
- if 'results' not in self._body or not isinstance(self._body['results'], dict):
+ if not isinstance(log.results(), dict):
return False
- for testResult in self._body['results'].values():
+ for testResult in log.results().values():
if isinstance(testResult, dict):
for value in testResult.values():
if not _is_float_convertible(value):
return True
- def _create_build_if_possible(self, builder, build_number, branch, platform, timestamp, revision, chromium_revision):
- key_name = builder.name + ':' + str(int(time.mktime(timestamp.timetuple())))
-
- def execute():
- build = Build.get_by_key_name(key_name)
- if build:
- return self._output('The build at %s already exists for %s' % (str(timestamp), builder.name))
-
- return Build(branch=branch, platform=platform, builder=builder, buildNumber=build_number,
- timestamp=timestamp, revision=revision, chromiumRevision=chromium_revision, key_name=key_name).put()
- return db.run_in_transaction(execute)
-
- def _add_test_if_needed(self, test_name, branch, platform):
-
- def execute(id):
- test = Test.get_by_key_name(test_name)
- returnValue = None
- if not test:
- test = Test(id=id, name=test_name, key_name=test_name)
- returnValue = test
- if branch.key() not in test.branches:
- test.branches.append(branch.key())
- if platform.key() not in test.platforms:
- test.platforms.append(platform.key())
- test.put()
- return returnValue
- return create_in_transaction_with_numeric_id_holder(execute) or Test.get_by_key_name(test_name)
-
class AdminReportHandler(ReportHandler):
def bypass_authentication(self):
--- /dev/null
+<!DOCTYPE html>
+<html>
+<body>
+<pre>{{ status }}</pre>
+<h1>Report logs</h1>
+
+<style type="text/css" scoped>
+
+table { width: 100%; border: solid 1px #ccc; border-collapse: collapse; }
+td { border: solid 1px #ccc; padding: 5px; }
+.control code { display: none; }
+code { white-space: pre-wrap; }
+form { display: inline; }
+
+</style>
+<script>
+
+$ = function (id) { return document.getElementById(id); }
+
+function show(id) {
+ var row = $('row_' + id);
+ var code = row.querySelector('code');
+ if (!code)
+ return;
+ var newRow = document.createElement('tr');
+ newRow.appendChild(document.createElement('td'));
+ newRow.firstChild.appendChild(code);
+ newRow.firstChild.colSpan = 8;
+ row.parentNode.insertBefore(newRow, row.nextSibling);
+ try {
+ code.textContent = JSON.stringify(JSON.parse(code.textContent), null, 2);
+ } catch (e) { }
+}
+
+function submit(form, event) {
+ if (!confirm('Are you sure?'))
+ event.preventDefault();
+}
+
+</script>
+<table>
+<thead>
+<tr>
+ <td>Id</td>
+ <td>Branch</td>
+ <td>Platform</td>
+ <td>Builder</td>
+ <td>Build</td>
+ <td>WebKit revision</td>
+ <td>Chromium revision</td>
+ <td class="control">Payload</td>
+</tr>
+</thead>
+<tbody>
+{% for log in logs %}
+<tr id="row_{{ log.key.id }}">
+ <td>{{ log.key.id }}</td>
+ <td>{{ log.branch.name }}</td>
+ <td>{{ log.platform.name }}</td>
+ <td><a href="http://build.webkit.org/builders/{{ log.builder.name }}/">{{ log.builder.name }}</a></td>
+ <td><a href="http://build.webkit.org/builders/{{ log.builder.name }}/builds/{{ log.build_number }}">{{ log.build_number }}</a></td>
+ <td><a href="http://trac.webkit.org/changeset/{{ log.webkit_revision }}">{{ log.webkit_revision }}</a></td>
+ <td><a href="http://src.chromium.org/viewvc/chrome?view=rev&revision={{ log.chromium_revision }}">{{ log.chromium_revision }}</a></td>
+ <td class="control"><button onclick="show({{ log.key.id }})">Show</button>
+ <form method="post" action="/admin/report-logs" onsubmit="submit(this, event)">
+ <input type="hidden" name="id" value="{{ log.key.id }}">
+ <button name="commit" value="true">Commit</button>
+ <button name="delete" value="true">Delete</button></form>
+ <code>{{ log.payload }}</code></td>
+</tr>
+{% endfor %}
+</tbody>
+</table>
+
+</body>
+</html>
--- /dev/null
+#!/usr/bin/env python
+# Copyright (C) 2012 Google Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import webapp2
+from google.appengine.ext.webapp import template
+
+import os
+
+from controller import schedule_report_process
+from models import ReportLog
+
+
+class ReportLogsHandler(webapp2.RequestHandler):
+ def get(self):
+ self.response.out.write(template.render('report_logs.yaml', {'logs': ReportLog.all()}))
+
+ def post(self):
+ commit = bool(self.request.get('commit'))
+ delete = bool(self.request.get('delete'))
+ if commit == delete:
+ return self._error('Invalid request')
+
+ try:
+ log = ReportLog.get_by_id(int(self.request.get('id', 0)))
+ except:
+ return self._error('Invalid log id "%s"' % self.request.get('id', ''))
+
+ if not log:
+ return self._error('No log found for "%s"' % self.request.get('id', ''))
+
+ if commit:
+ log.commit = True
+ log.put()
+ schedule_report_process(log)
+ else:
+ log.delete()
+
+ self.response.out.write(template.render('report_logs.yaml', {'logs': ReportLog.all(), 'status': 'OK'}))
+
+ def _error(self, message):
+ self.response.headers['Content-Type'] = 'text/plain; charset=utf-8'
+ self.response.out.write(message + '\n')
--- /dev/null
+#!/usr/bin/env python
+# Copyright (C) 2012 Google Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import webapp2
+from google.appengine.ext import db
+
+import time
+
+from controller import schedule_runs_update
+from controller import schedule_dashboard_update
+from controller import schedule_manifest_update
+from models import Build
+from models import ReportLog
+from models import Test
+from models import TestResult
+from models import create_in_transaction_with_numeric_id_holder
+
+
+class ReportProcessHandler(webapp2.RequestHandler):
+ def post(self):
+ self.response.headers['Content-Type'] = 'text/plain; charset=utf-8'
+
+ log_id = int(self.request.get('id', 0))
+
+ log = ReportLog.get_by_id(log_id)
+ if not log or not log.commit:
+ self.response.out.write("Not processed")
+ return
+
+ def _float_or_none(dictionary, key):
+ value = dictionary.get(key)
+ if value:
+ return float(value)
+ return None
+
+ branch = log.branch()
+ platform = log.platform()
+ build = self._create_build_if_possible(log, branch, platform)
+
+ for test_name, result in log.results().iteritems():
+ test = self._add_test_if_needed(test_name, branch, platform)
+ schedule_runs_update(test.id, branch.id, platform.id)
+ if isinstance(result, dict):
+ TestResult(name=test_name, build=build, value=float(result['avg']), valueMedian=_float_or_none(result, 'median'),
+ valueStdev=_float_or_none(result, 'stdev'), valueMin=_float_or_none(result, 'min'), valueMax=_float_or_none(result, 'max')).put()
+ else:
+ TestResult(name=test_name, build=build, value=float(result)).put()
+
+ log = ReportLog.get(log.key())
+ log.delete()
+
+ # We need to update dashboard and manifest because they are affected by the existance of test results
+ schedule_dashboard_update()
+ schedule_manifest_update()
+
+ self.response.out.write('OK')
+
+ def _create_build_if_possible(self, log, branch, platform):
+ builder = log.builder()
+ key_name = builder.name + ':' + str(int(time.mktime(log.timestamp().timetuple())))
+
+ def execute():
+ build = Build.get_by_key_name(key_name)
+ if build:
+ return build
+
+ return Build(branch=branch, platform=platform, builder=builder, buildNumber=log.build_number(),
+ timestamp=log.timestamp(), revision=log.webkit_revision(), chromiumRevision=log.chromium_revision(),
+ key_name=key_name).put()
+ return db.run_in_transaction(execute)
+
+ def _add_test_if_needed(self, test_name, branch, platform):
+
+ def execute(id):
+ test = Test.get_by_key_name(test_name)
+ returnValue = None
+ if not test:
+ test = Test(id=id, name=test_name, key_name=test_name)
+ returnValue = test
+ if branch.key() not in test.branches:
+ test.branches.append(branch.key())
+ if platform.key() not in test.platforms:
+ test.platforms.append(platform.key())
+ test.put()
+ return returnValue
+ return create_in_transaction_with_numeric_id_holder(execute) or Test.get_by_key_name(test_name)
git://git.webkit.org / WebKit-https.git / commitdiff