REGRESSION: Perf-o-matic adds duplicated test results
authorrniwa@webkit.org <rniwa@webkit.org@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Mon, 13 Feb 2012 05:45:08 +0000 (05:45 +0000)
committerrniwa@webkit.org <rniwa@webkit.org@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Mon, 13 Feb 2012 05:45:08 +0000 (05:45 +0000)
https://bugs.webkit.org/show_bug.cgi?id=78373

Reviewed by Hajime Morita.

Add TestResult inside a transaction. We use high replication databsae but this should prevent
the duplication in the most of the time.

Also fixed random bugs in Controller and ReportHandler.

* Websites/webkit-perf.appspot.com/controller.py:
(get_persistent_cache):
* Websites/webkit-perf.appspot.com/report_handler.py:
(ReportHandler.post):
* Websites/webkit-perf.appspot.com/report_process_handler.py:
(ReportProcessHandler.post):
(ReportProcessHandler._add_test_if_needed):
(ReportProcessHandler):
(ReportProcessHandler._add_test_result_if_needed):
(ReportProcessHandler._add_test_result_if_needed._float_or_none):
(ReportProcessHandler._add_test_result_if_needed.execute):

git-svn-id: https://svn.webkit.org/repository/webkit/trunk@107531 268f45cc-cd09-0410-ab3c-d52691b4dbfc

ChangeLog
Websites/webkit-perf.appspot.com/controller.py
Websites/webkit-perf.appspot.com/models.py
Websites/webkit-perf.appspot.com/report_handler.py
Websites/webkit-perf.appspot.com/report_process_handler.py

index ba45060..5877fdd 100644 (file)
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,27 @@
+2012-02-10  Ryosuke Niwa  <rniwa@webkit.org>
+
+        REGRESSION: Perf-o-matic adds duplicated test results
+        https://bugs.webkit.org/show_bug.cgi?id=78373
+
+        Reviewed by Hajime Morita.
+
+        Add TestResult inside a transaction. We use high replication databsae but this should prevent
+        the duplication in the most of the time.
+
+        Also fixed random bugs in Controller and ReportHandler.
+
+        * Websites/webkit-perf.appspot.com/controller.py:
+        (get_persistent_cache):
+        * Websites/webkit-perf.appspot.com/report_handler.py:
+        (ReportHandler.post):
+        * Websites/webkit-perf.appspot.com/report_process_handler.py:
+        (ReportProcessHandler.post):
+        (ReportProcessHandler._add_test_if_needed):
+        (ReportProcessHandler):
+        (ReportProcessHandler._add_test_result_if_needed):
+        (ReportProcessHandler._add_test_result_if_needed._float_or_none):
+        (ReportProcessHandler._add_test_result_if_needed.execute):
+
 2012-02-12  Carlos Garcia Campos  <cgarcia@igalia.com>
 
         Unreviewed. Fix GTK+ build after r107454.
index f5868e6..8e4e84c 100644 (file)
@@ -55,7 +55,7 @@ def get_persistent_cache(name):
     if value:
         return value
     cache = PersistentCache.get_by_key_name(name)
-    memcache.set(name, cache)
+    memcache.set(name, cache.value)
     return cache.value
 
 
index 7f7e41b..01d894c 100644 (file)
@@ -117,6 +117,10 @@ class TestResult(db.Model):
     valueMin = db.FloatProperty()
     valueMax = db.FloatProperty()
 
+    @staticmethod
+    def key_name(build, test_name):
+        return build.key().name() + ':' + test_name
+
 
 # Temporarily store log reports sent by bots
 class ReportLog(db.Model):
index f8f116d..b68891f 100644 (file)
@@ -45,7 +45,7 @@ class ReportHandler(webapp2.RequestHandler):
         headers = "\n".join([key + ': ' + value for key, value in self.request.headers.items()])
 
         # Do as best as we can to remove the password
-        request_body_without_password = re.sub(r'"password"\s*:\s*".+?",', '', self.request.body)
+        request_body_without_password = re.sub(r'"password"\s*:\s*".+?",?', '', self.request.body)
         log = ReportLog(timestamp=datetime.now(), headers=headers, payload=request_body_without_password)
         log.put()
 
index 9dbbf12..4cfa9ed 100644 (file)
@@ -53,24 +53,14 @@ class ReportProcessHandler(webapp2.RequestHandler):
             self.response.out.write("Not processed")
             return
 
-        def _float_or_none(dictionary, key):
-            value = dictionary.get(key)
-            if value:
-                return float(value)
-            return None
-
         branch = log.branch()
         platform = log.platform()
         build = self._create_build_if_possible(log, branch, platform)
 
         for test_name, result in log.results().iteritems():
             test = self._add_test_if_needed(test_name, branch, platform)
+            self._add_test_result_if_needed(test_name, build, result)
             schedule_runs_update(test.id, branch.id, platform.id)
-            if isinstance(result, dict):
-                TestResult(name=test_name, build=build, value=float(result['avg']), valueMedian=_float_or_none(result, 'median'),
-                    valueStdev=_float_or_none(result, 'stdev'), valueMin=_float_or_none(result, 'min'), valueMax=_float_or_none(result, 'max')).put()
-            else:
-                TestResult(name=test_name, build=build, value=float(result)).put()
 
         log = ReportLog.get(log.key())
         log.delete()
@@ -110,3 +100,19 @@ class ReportProcessHandler(webapp2.RequestHandler):
             test.put()
             return returnValue
         return create_in_transaction_with_numeric_id_holder(execute) or Test.get_by_key_name(test_name)
+
+    def _add_test_result_if_needed(self, test_name, build, result):
+        key_name = TestResult.key_name(build, test_name)
+
+        def _float_or_none(dictionary, key):
+            value = dictionary.get(key)
+            if value:
+                return float(value)
+            return None
+
+        if not isinstance(result, dict):
+            return TestResult.get_or_insert(key_name, name=test_name, build=build, value=float(result))
+
+        return TestResult.get_or_insert(key_name, name=test_name, build=build, value=float(result['avg']),
+            valueMedian=_float_or_none(result, 'median'), valueStdev=_float_or_none(result, 'stdev'),
+            valueMin=_float_or_none(result, 'min'), valueMax=_float_or_none(result, 'max'))