Enable memcache on webkit-perf.appspot.com
authorrniwa@webkit.org <rniwa@webkit.org@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Tue, 31 Jan 2012 01:36:17 +0000 (01:36 +0000)
committerrniwa@webkit.org <rniwa@webkit.org@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Tue, 31 Jan 2012 01:36:17 +0000 (01:36 +0000)
https://bugs.webkit.org/show_bug.cgi?id=77378

Reviewed by Adam Barth.

Eanble memcache on dashboard, manifest, and runs handlers. Clear appropriate caches when new runs are reported,
or new models are created. We flush all caches when tests are merged since it's hard to figure out dependencies
in that case. Luckily, we merge tests only occassionally and manually (or hope so) so this shouldn't be an issue.

* Websites/webkit-perf.appspot.com/create_handler.py:
(CreateHandler.post):
* Websites/webkit-perf.appspot.com/dashboard_handler.py:
(DashboardHandler.get):
* Websites/webkit-perf.appspot.com/manifest_handler.py:
(ManifestHandler.get):
* Websites/webkit-perf.appspot.com/merge_tests_handler.py:
(MergeTestsHandler.post):
* Websites/webkit-perf.appspot.com/models.py:
(createInTransactionWithNumericIdHolder):
(Test):
(Test.cacheKey):
* Websites/webkit-perf.appspot.com/report_handler.py:
(ReportHandler.post):
(ReportHandler._addTestIfNeeded):
* Websites/webkit-perf.appspot.com/runs_handler.py:
(RunsHandler.get):

git-svn-id: https://svn.webkit.org/repository/webkit/trunk@106308 268f45cc-cd09-0410-ab3c-d52691b4dbfc

ChangeLog
Websites/webkit-perf.appspot.com/app.yaml
Websites/webkit-perf.appspot.com/create_handler.py
Websites/webkit-perf.appspot.com/dashboard_handler.py
Websites/webkit-perf.appspot.com/manifest_handler.py
Websites/webkit-perf.appspot.com/merge_tests_handler.py
Websites/webkit-perf.appspot.com/models.py
Websites/webkit-perf.appspot.com/report_handler.py
Websites/webkit-perf.appspot.com/runs_handler.py

index 9429755..dca82ed 100644 (file)
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,34 @@
 2012-01-30  Ryosuke Niwa  <rniwa@webkit.org>
 
+        Enable memcache on webkit-perf.appspot.com
+        https://bugs.webkit.org/show_bug.cgi?id=77378
+
+        Reviewed by Adam Barth.
+
+        Eanble memcache on dashboard, manifest, and runs handlers. Clear appropriate caches when new runs are reported,
+        or new models are created. We flush all caches when tests are merged since it's hard to figure out dependencies
+        in that case. Luckily, we merge tests only occassionally and manually (or hope so) so this shouldn't be an issue.
+
+        * Websites/webkit-perf.appspot.com/create_handler.py:
+        (CreateHandler.post):
+        * Websites/webkit-perf.appspot.com/dashboard_handler.py:
+        (DashboardHandler.get):
+        * Websites/webkit-perf.appspot.com/manifest_handler.py:
+        (ManifestHandler.get):
+        * Websites/webkit-perf.appspot.com/merge_tests_handler.py:
+        (MergeTestsHandler.post):
+        * Websites/webkit-perf.appspot.com/models.py:
+        (createInTransactionWithNumericIdHolder):
+        (Test):
+        (Test.cacheKey):
+        * Websites/webkit-perf.appspot.com/report_handler.py:
+        (ReportHandler.post):
+        (ReportHandler._addTestIfNeeded):
+        * Websites/webkit-perf.appspot.com/runs_handler.py:
+        (RunsHandler.get):
+
+2012-01-30  Ryosuke Niwa  <rniwa@webkit.org>
+
         webkit-perf.appspot.com should have an ability to merge tests
         https://bugs.webkit.org/show_bug.cgi?id=77250
 
index 8e2133e..ef129ae 100644 (file)
@@ -1,5 +1,5 @@
 application: webkit-perf
-version: 9
+version: 10
 runtime: python27
 api_version: 1
 threadsafe: false
index c20a56d..98b6b69 100644 (file)
@@ -28,6 +28,7 @@
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 import webapp2
+from google.appengine.api import memcache
 from google.appengine.ext import db
 
 import json
@@ -61,6 +62,8 @@ class CreateHandler(webapp2.RequestHandler):
         else:
             error = "Unknown model type: %s\n" % model
 
+        # No need to clear manifest or runs since they only contain ones with test results
+        memcache.delete('dashboard')
         self.response.out.write(error + '\n' if error else 'OK')
 
     def _createBuilder(self, name, password):
index 7973b02..c6356c7 100644 (file)
@@ -28,6 +28,7 @@
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 import webapp2
+from google.appengine.api import memcache
 
 import json
 
@@ -39,6 +40,13 @@ from models import Test
 
 class DashboardHandler(webapp2.RequestHandler):
     def get(self):
+        self.response.headers['Content-Type'] = 'application/json; charset=utf-8';
+
+        cache = memcache.get('dashboard')
+        if cache:
+            self.response.out.write(cache)
+            return
+
         webkitTrunk = Branch.get_by_key_name('webkit-trunk')
 
         # FIXME: Determine popular branches, platforms, and tests
@@ -55,5 +63,6 @@ class DashboardHandler(webapp2.RequestHandler):
         for test in Test.all():
             dashboard['testToId'][test.name] = test.id
 
-        self.response.headers['Content-Type'] = 'application/json; charset=utf-8';
-        self.response.out.write(json.dumps(dashboard))
+        result = json.dumps(dashboard)
+        self.response.out.write(result)
+        memcache.add('dashboard', result)
index 4d487dc..7f76a29 100644 (file)
@@ -28,6 +28,7 @@
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 import webapp2
+from google.appengine.api import memcache
 
 import json
 
@@ -40,7 +41,10 @@ from models import Test
 class ManifestHandler(webapp2.RequestHandler):
     def get(self):
         self.response.headers['Content-Type'] = 'text/plain; charset=utf-8';
-        self.response.out.write('{"testMap":')
+        cache = memcache.get('manifest')
+        if cache:
+            self.response.out.write(cache)
+            return
 
         testMap = {}
         platformIdMap = {}
@@ -64,9 +68,6 @@ class ManifestHandler(webapp2.RequestHandler):
                 branchIdMap[branchId]['tests'].append(test.id)
                 branchIdMap[branchId]['platforms'] += platformIds
 
-        self.response.out.write(json.dumps(testMap))
-        self.response.out.write(',"platformMap":')
-
         platformMap = {}
         for platform in Platform.all():
             if platform.id not in platformIdMap:
@@ -77,9 +78,6 @@ class ManifestHandler(webapp2.RequestHandler):
                 'branchIds': list(set(platformIdMap[platform.id]['branches'])),
             }
 
-        self.response.out.write(json.dumps(platformMap))
-        self.response.out.write(',"branchMap":')
-
         branchMap = {}
         for branch in Branch.all():
             if branch.id not in branchIdMap:
@@ -90,5 +88,6 @@ class ManifestHandler(webapp2.RequestHandler):
                 'platformIds': list(set(branchIdMap[branch.id]['platforms'])),
             }
 
-        self.response.out.write(json.dumps(branchMap))
-        self.response.out.write('}')
+        result = json.dumps({'testMap': testMap, 'platformMap': platformMap, 'branchMap': branchMap})
+        self.response.out.write(result)
+        memcache.add('manifest', result)
index ecc7b71..5fb8bba 100644 (file)
@@ -28,6 +28,7 @@
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 import webapp2
+from google.appengine.api import memcache
 from google.appengine.ext.webapp import template
 
 import os
@@ -56,6 +57,10 @@ class MergeTestsHandler(webapp2.RequestHandler):
             result.name = into.name
             result.put()
 
+        # Just flush everyting since we rarely merge tests and we need to flush
+        # dashboard, manifest, and all runs for this test here.
+        memcache.flush_all()
+
         deleteModelWithNumericIdHolder(merge)
 
         self.response.out.write('OK')
index 43f0fc6..cf90ff2 100644 (file)
@@ -50,7 +50,6 @@ def createInTransactionWithNumericIdHolder(callback):
         idHolder.delete()
     return owner
 
-
 def deleteModelWithNumericIdHolder(model):
     idHolder = NumericIdHolder.get_by_id(model.id)
     model.delete()
@@ -100,6 +99,10 @@ class Test(db.Model):
     branches = db.ListProperty(db.Key)
     platforms = db.ListProperty(db.Key)
 
+    @staticmethod
+    def cacheKey(testId, branchId, platformId):
+        return 'runs:%d,%d,%d' % (testId, branchId, platformId)
+
 
 class TestResult(db.Model):
     name = db.StringProperty(required=True)
index a1d3540..a7277e2 100644 (file)
@@ -28,6 +28,7 @@
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 import webapp2
+from google.appengine.api import memcache
 from google.appengine.ext import db
 
 import json
@@ -85,17 +86,22 @@ class ReportHandler(webapp2.RequestHandler):
         if not build:
             return
 
-        for test, result in self._body['results'].iteritems():
-            self._addTestIfNeeded(test, branch, platform)
+        for testName, result in self._body['results'].iteritems():
+            test = self._addTestIfNeeded(testName, branch, platform)
+            memcache.delete(Test.cacheKey(test.id, branch.id, platform.id))
             if isinstance(result, dict):
-                TestResult(name=test, build=build, value=float(result.get('avg', 0)), valueMedian=float(result.get('median', 0)),
+                TestResult(name=testName, build=build, value=float(result.get('avg', 0)), valueMedian=float(result.get('median', 0)),
                     valueStdev=float(result.get('stdev', 0)), valueMin=float(result.get('min', 0)), valueMax=float(result.get('max', 0))).put()
             else:
-                TestResult(name=test, build=build, value=float(result)).put()
+                TestResult(name=testName, build=build, value=float(result)).put()
 
         log = ReportLog.get(log.key())
         log.delete()
 
+        # We need to update dashboard and manifest because they are affected by the existance of test results
+        memcache.delete('dashboard')
+        memcache.delete('manifest')
+
         return self._output('OK')
 
     def _modelByKeyNameInBodyOrError(self, model, keyName):
@@ -176,7 +182,7 @@ class ReportHandler(webapp2.RequestHandler):
                 test.platforms.append(platform.key())
             test.put()
             return returnValue
-        createInTransactionWithNumericIdHolder(execute)
+        return createInTransactionWithNumericIdHolder(execute) or Test.get_by_key_name(testName)
 
 
 class AdminReportHandler(ReportHandler):
index 8793c28..d3bc64a 100644 (file)
@@ -28,6 +28,7 @@
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 import webapp2
+from google.appengine.api import memcache
 
 import json
 from time import mktime
@@ -45,6 +46,8 @@ from models import modelFromNumericId
 
 class RunsHandler(webapp2.RequestHandler):
     def get(self):
+        self.response.headers['Content-Type'] = 'application/json; charset=utf-8'
+
         try:
             testId = int(self.request.get('id', 0))
             branchId = int(self.request.get('branchid', 0))
@@ -58,6 +61,12 @@ class RunsHandler(webapp2.RequestHandler):
         # FIXME: Just fetch builds specified by "days"
         # days = self.request.get('days', 365)
 
+        cacheKey = Test.cacheKey(testId, branchId, platformId)
+        cache = memcache.get(cacheKey)
+        if cache:
+            self.response.out.write(cache)
+            return
+
         builds = Build.all()
         builds.filter('branch =', modelFromNumericId(branchId, Branch))
         builds.filter('platform =', modelFromNumericId(platformId, Platform))
@@ -84,11 +93,12 @@ class RunsHandler(webapp2.RequestHandler):
                 values.append(result.value)
                 timestamps.append(posixTimestamp)
 
-        self.response.headers['Content-Type'] = 'application/json; charset=utf-8';
-        self.response.out.write(json.dumps({
+        result = json.dumps({
             'test_runs': test_runs,
             'averages': averages,
             'min': min(values) if values else None,
             'max': max(values) if values else None,
             'date_range': [min(timestamps), max(timestamps)] if timestamps else None,
-            'stat': 'ok'}))
+            'stat': 'ok'})
+        self.response.out.write(result)
+        memcache.add(cacheKey, result)