Enable memcache on webkit-perf.appspot.com
authorrniwa@webkit.org <rniwa@webkit.org@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Tue, 31 Jan 2012 01:36:17 +0000 (01:36 +0000)
committerrniwa@webkit.org <rniwa@webkit.org@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Tue, 31 Jan 2012 01:36:17 +0000 (01:36 +0000)
https://bugs.webkit.org/show_bug.cgi?id=77378

Reviewed by Adam Barth.

Eanble memcache on dashboard, manifest, and runs handlers. Clear appropriate caches when new runs are reported,
or new models are created. We flush all caches when tests are merged since it's hard to figure out dependencies
in that case. Luckily, we merge tests only occassionally and manually (or hope so) so this shouldn't be an issue.

* Websites/webkit-perf.appspot.com/create_handler.py:
(CreateHandler.post):
* Websites/webkit-perf.appspot.com/dashboard_handler.py:
(DashboardHandler.get):
* Websites/webkit-perf.appspot.com/manifest_handler.py:
(ManifestHandler.get):
* Websites/webkit-perf.appspot.com/merge_tests_handler.py:
(MergeTestsHandler.post):
* Websites/webkit-perf.appspot.com/models.py:
(createInTransactionWithNumericIdHolder):
(Test):
(Test.cacheKey):
* Websites/webkit-perf.appspot.com/report_handler.py:
(ReportHandler.post):
(ReportHandler._addTestIfNeeded):
* Websites/webkit-perf.appspot.com/runs_handler.py:
(RunsHandler.get):

git-svn-id: https://svn.webkit.org/repository/webkit/trunk@106308 268f45cc-cd09-0410-ab3c-d52691b4dbfc

ChangeLog
Websites/webkit-perf.appspot.com/app.yaml
Websites/webkit-perf.appspot.com/create_handler.py
Websites/webkit-perf.appspot.com/dashboard_handler.py
Websites/webkit-perf.appspot.com/manifest_handler.py
Websites/webkit-perf.appspot.com/merge_tests_handler.py
Websites/webkit-perf.appspot.com/models.py
Websites/webkit-perf.appspot.com/report_handler.py
Websites/webkit-perf.appspot.com/runs_handler.py

index 9429755d742c169202dea22b42efbc8c1b76d57a..dca82eda35f6786aac8831c1f47fe76521567171 100644 (file)
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,32 @@
+2012-01-30  Ryosuke Niwa  <rniwa@webkit.org>
+
+        Enable memcache on webkit-perf.appspot.com
+        https://bugs.webkit.org/show_bug.cgi?id=77378
+
+        Reviewed by Adam Barth.
+
+        Eanble memcache on dashboard, manifest, and runs handlers. Clear appropriate caches when new runs are reported,
+        or new models are created. We flush all caches when tests are merged since it's hard to figure out dependencies
+        in that case. Luckily, we merge tests only occassionally and manually (or hope so) so this shouldn't be an issue.
+
+        * Websites/webkit-perf.appspot.com/create_handler.py:
+        (CreateHandler.post):
+        * Websites/webkit-perf.appspot.com/dashboard_handler.py:
+        (DashboardHandler.get):
+        * Websites/webkit-perf.appspot.com/manifest_handler.py:
+        (ManifestHandler.get):
+        * Websites/webkit-perf.appspot.com/merge_tests_handler.py:
+        (MergeTestsHandler.post):
+        * Websites/webkit-perf.appspot.com/models.py:
+        (createInTransactionWithNumericIdHolder):
+        (Test):
+        (Test.cacheKey):
+        * Websites/webkit-perf.appspot.com/report_handler.py:
+        (ReportHandler.post):
+        (ReportHandler._addTestIfNeeded):
+        * Websites/webkit-perf.appspot.com/runs_handler.py:
+        (RunsHandler.get):
+
 2012-01-30  Ryosuke Niwa  <rniwa@webkit.org>
 
         webkit-perf.appspot.com should have an ability to merge tests
index 8e2133e204959827e7119fcc39a2f736df45dafa..ef129aea6458acd58333b500783c42fbbfd91546 100644 (file)
@@ -1,5 +1,5 @@
 application: webkit-perf
-version: 9
+version: 10
 runtime: python27
 api_version: 1
 threadsafe: false
index c20a56d58c27374592747b24233f33fccde61322..98b6b695d721f7efdb3f47f351f827993a2a8ae0 100644 (file)
@@ -28,6 +28,7 @@
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 import webapp2
+from google.appengine.api import memcache
 from google.appengine.ext import db
 
 import json
@@ -61,6 +62,8 @@ class CreateHandler(webapp2.RequestHandler):
         else:
             error = "Unknown model type: %s\n" % model
 
+        # No need to clear manifest or runs since they only contain ones with test results
+        memcache.delete('dashboard')
         self.response.out.write(error + '\n' if error else 'OK')
 
     def _createBuilder(self, name, password):
index 7973b02cfd583fdf1c69ebcd5f8ec7257903b301..c6356c73119b33a18572cf85c3f93c2df228fd6d 100644 (file)
@@ -28,6 +28,7 @@
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 import webapp2
+from google.appengine.api import memcache
 
 import json
 
@@ -39,6 +40,13 @@ from models import Test
 
 class DashboardHandler(webapp2.RequestHandler):
     def get(self):
+        self.response.headers['Content-Type'] = 'application/json; charset=utf-8';
+
+        cache = memcache.get('dashboard')
+        if cache:
+            self.response.out.write(cache)
+            return
+
         webkitTrunk = Branch.get_by_key_name('webkit-trunk')
 
         # FIXME: Determine popular branches, platforms, and tests
@@ -55,5 +63,6 @@ class DashboardHandler(webapp2.RequestHandler):
         for test in Test.all():
             dashboard['testToId'][test.name] = test.id
 
-        self.response.headers['Content-Type'] = 'application/json; charset=utf-8';
-        self.response.out.write(json.dumps(dashboard))
+        result = json.dumps(dashboard)
+        self.response.out.write(result)
+        memcache.add('dashboard', result)
index 4d487dccd4844b0acae4b5f294b6f36e636fbbde..7f76a29c14a7fc3fb5e4f6a52a43863297be871a 100644 (file)
@@ -28,6 +28,7 @@
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 import webapp2
+from google.appengine.api import memcache
 
 import json
 
@@ -40,7 +41,10 @@ from models import Test
 class ManifestHandler(webapp2.RequestHandler):
     def get(self):
         self.response.headers['Content-Type'] = 'text/plain; charset=utf-8';
-        self.response.out.write('{"testMap":')
+        cache = memcache.get('manifest')
+        if cache:
+            self.response.out.write(cache)
+            return
 
         testMap = {}
         platformIdMap = {}
@@ -64,9 +68,6 @@ class ManifestHandler(webapp2.RequestHandler):
                 branchIdMap[branchId]['tests'].append(test.id)
                 branchIdMap[branchId]['platforms'] += platformIds
 
-        self.response.out.write(json.dumps(testMap))
-        self.response.out.write(',"platformMap":')
-
         platformMap = {}
         for platform in Platform.all():
             if platform.id not in platformIdMap:
@@ -77,9 +78,6 @@ class ManifestHandler(webapp2.RequestHandler):
                 'branchIds': list(set(platformIdMap[platform.id]['branches'])),
             }
 
-        self.response.out.write(json.dumps(platformMap))
-        self.response.out.write(',"branchMap":')
-
         branchMap = {}
         for branch in Branch.all():
             if branch.id not in branchIdMap:
@@ -90,5 +88,6 @@ class ManifestHandler(webapp2.RequestHandler):
                 'platformIds': list(set(branchIdMap[branch.id]['platforms'])),
             }
 
-        self.response.out.write(json.dumps(branchMap))
-        self.response.out.write('}')
+        result = json.dumps({'testMap': testMap, 'platformMap': platformMap, 'branchMap': branchMap})
+        self.response.out.write(result)
+        memcache.add('manifest', result)
index ecc7b718cb50ba10c8b76d61316f06e767ece8cc..5fb8bba7416cdd5e6ecaeabef7ac3a793e51de3a 100644 (file)
@@ -28,6 +28,7 @@
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 import webapp2
+from google.appengine.api import memcache
 from google.appengine.ext.webapp import template
 
 import os
@@ -56,6 +57,10 @@ class MergeTestsHandler(webapp2.RequestHandler):
             result.name = into.name
             result.put()
 
+        # Just flush everyting since we rarely merge tests and we need to flush
+        # dashboard, manifest, and all runs for this test here.
+        memcache.flush_all()
+
         deleteModelWithNumericIdHolder(merge)
 
         self.response.out.write('OK')
index 43f0fc611c982d5bbdd6e9f5fd490f6b10fd0d56..cf90ff2ac144e9d21d77e9cef6eb04e87f4de1db 100644 (file)
@@ -50,7 +50,6 @@ def createInTransactionWithNumericIdHolder(callback):
         idHolder.delete()
     return owner
 
-
 def deleteModelWithNumericIdHolder(model):
     idHolder = NumericIdHolder.get_by_id(model.id)
     model.delete()
@@ -100,6 +99,10 @@ class Test(db.Model):
     branches = db.ListProperty(db.Key)
     platforms = db.ListProperty(db.Key)
 
+    @staticmethod
+    def cacheKey(testId, branchId, platformId):
+        return 'runs:%d,%d,%d' % (testId, branchId, platformId)
+
 
 class TestResult(db.Model):
     name = db.StringProperty(required=True)
index a1d354080ef4edf5cc4137368848a3b8aacae64b..a7277e216f76ba60ba67c25cb10e71c4b40e99ae 100644 (file)
@@ -28,6 +28,7 @@
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 import webapp2
+from google.appengine.api import memcache
 from google.appengine.ext import db
 
 import json
@@ -85,17 +86,22 @@ class ReportHandler(webapp2.RequestHandler):
         if not build:
             return
 
-        for test, result in self._body['results'].iteritems():
-            self._addTestIfNeeded(test, branch, platform)
+        for testName, result in self._body['results'].iteritems():
+            test = self._addTestIfNeeded(testName, branch, platform)
+            memcache.delete(Test.cacheKey(test.id, branch.id, platform.id))
             if isinstance(result, dict):
-                TestResult(name=test, build=build, value=float(result.get('avg', 0)), valueMedian=float(result.get('median', 0)),
+                TestResult(name=testName, build=build, value=float(result.get('avg', 0)), valueMedian=float(result.get('median', 0)),
                     valueStdev=float(result.get('stdev', 0)), valueMin=float(result.get('min', 0)), valueMax=float(result.get('max', 0))).put()
             else:
-                TestResult(name=test, build=build, value=float(result)).put()
+                TestResult(name=testName, build=build, value=float(result)).put()
 
         log = ReportLog.get(log.key())
         log.delete()
 
+        # We need to update dashboard and manifest because they are affected by the existance of test results
+        memcache.delete('dashboard')
+        memcache.delete('manifest')
+
         return self._output('OK')
 
     def _modelByKeyNameInBodyOrError(self, model, keyName):
@@ -176,7 +182,7 @@ class ReportHandler(webapp2.RequestHandler):
                 test.platforms.append(platform.key())
             test.put()
             return returnValue
-        createInTransactionWithNumericIdHolder(execute)
+        return createInTransactionWithNumericIdHolder(execute) or Test.get_by_key_name(testName)
 
 
 class AdminReportHandler(ReportHandler):
index 8793c28ebebb4c76b1522c7f7877c9ee4f5f382e..d3bc64ad3b88be8ea775555ff26a3a2f84304512 100644 (file)
@@ -28,6 +28,7 @@
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 import webapp2
+from google.appengine.api import memcache
 
 import json
 from time import mktime
@@ -45,6 +46,8 @@ from models import modelFromNumericId
 
 class RunsHandler(webapp2.RequestHandler):
     def get(self):
+        self.response.headers['Content-Type'] = 'application/json; charset=utf-8'
+
         try:
             testId = int(self.request.get('id', 0))
             branchId = int(self.request.get('branchid', 0))
@@ -58,6 +61,12 @@ class RunsHandler(webapp2.RequestHandler):
         # FIXME: Just fetch builds specified by "days"
         # days = self.request.get('days', 365)
 
+        cacheKey = Test.cacheKey(testId, branchId, platformId)
+        cache = memcache.get(cacheKey)
+        if cache:
+            self.response.out.write(cache)
+            return
+
         builds = Build.all()
         builds.filter('branch =', modelFromNumericId(branchId, Branch))
         builds.filter('platform =', modelFromNumericId(platformId, Platform))
@@ -84,11 +93,12 @@ class RunsHandler(webapp2.RequestHandler):
                 values.append(result.value)
                 timestamps.append(posixTimestamp)
 
-        self.response.headers['Content-Type'] = 'application/json; charset=utf-8';
-        self.response.out.write(json.dumps({
+        result = json.dumps({
             'test_runs': test_runs,
             'averages': averages,
             'min': min(values) if values else None,
             'max': max(values) if values else None,
             'date_range': [min(timestamps), max(timestamps)] if timestamps else None,
-            'stat': 'ok'}))
+            'stat': 'ok'})
+        self.response.out.write(result)
+        memcache.add(cacheKey, result)