Perf-o-matic should store "values" and support array'ed input
authorrniwa@webkit.org <rniwa@webkit.org@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Tue, 25 Sep 2012 21:22:08 +0000 (21:22 +0000)
committerrniwa@webkit.org <rniwa@webkit.org@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Tue, 25 Sep 2012 21:22:08 +0000 (21:22 +0000)
https://bugs.webkit.org/show_bug.cgi?id=97601

Reviewed by Dirk Pranke.

Support new JSON format where the outermost structure is an array instead of a dictionary and results may
contain "values". This change will let us remove some code from run-perf-tests.

Old: {"webkit-revision": 123456, "results": {"test": {"avg": 123}}
New: [{"webkit-revision": 123456, "results": {"test": {"avg": 123, values: [122, 123, 124]}}}]

* Websites/webkit-perf.appspot.com/app.yaml: Incremented the version number.
* Websites/webkit-perf.appspot.com/models.py:
(TestResult): Added values property.
(TestResult.get_or_insert_from_parsed_json): Pass in "values" to the constructor if the value is present.
(ReportLog.get_value): Use the first item in the array if self._parsed uses the new format.
(ReportLog.results_are_well_formed): Verifies that items in "values" are floats convertible. Also verify that
if the JSON uses new format, there is exactly one set of results. In theory, we could support multiple results
but we don't do that now for its complexity.
* Websites/webkit-perf.appspot.com/models_unittest.py:
(TestResultTests.test_get_or_insert_stat_value): Make sure values is present and is an empty list.
(TestResultTests.test_get_or_insert_stat_value_with_values): Added.
(ReportLogTests.test_results_are_well_formed):
(ReportLogTests.test_chromium_revision): Renamed from chromium_revision so that it actually runs.
(ReportLogTests.test_results_in_array):

git-svn-id: https://svn.webkit.org/repository/webkit/trunk@129555 268f45cc-cd09-0410-ab3c-d52691b4dbfc

ChangeLog
Websites/webkit-perf.appspot.com/app.yaml
Websites/webkit-perf.appspot.com/models.py
Websites/webkit-perf.appspot.com/models_unittest.py

index 983fd8e..7831bc4 100644 (file)
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,31 @@
+2012-09-25  Ryosuke Niwa  <rniwa@webkit.org>
+
+        Perf-o-matic should store "values" and support array'ed input
+        https://bugs.webkit.org/show_bug.cgi?id=97601
+
+        Reviewed by Dirk Pranke.
+
+        Support new JSON format where the outermost structure is an array instead of a dictionary and results may
+        contain "values". This change will let us remove some code from run-perf-tests.
+
+        Old: {"webkit-revision": 123456, "results": {"test": {"avg": 123}}
+        New: [{"webkit-revision": 123456, "results": {"test": {"avg": 123, values: [122, 123, 124]}}}]
+
+        * Websites/webkit-perf.appspot.com/app.yaml: Incremented the version number.
+        * Websites/webkit-perf.appspot.com/models.py:
+        (TestResult): Added values property.
+        (TestResult.get_or_insert_from_parsed_json): Pass in "values" to the constructor if the value is present.
+        (ReportLog.get_value): Use the first item in the array if self._parsed uses the new format.
+        (ReportLog.results_are_well_formed): Verifies that items in "values" are floats convertible. Also verify that
+        if the JSON uses new format, there is exactly one set of results. In theory, we could support multiple results
+        but we don't do that now for its complexity.
+        * Websites/webkit-perf.appspot.com/models_unittest.py:
+        (TestResultTests.test_get_or_insert_stat_value): Make sure values is present and is an empty list.
+        (TestResultTests.test_get_or_insert_stat_value_with_values): Added.
+        (ReportLogTests.test_results_are_well_formed):
+        (ReportLogTests.test_chromium_revision): Renamed from chromium_revision so that it actually runs.
+        (ReportLogTests.test_results_in_array):
+
 2012-09-25  Laszlo Gombos  <l.gombos@samsung.com>
 
         [EFL] Update minimal required versions for dependencies
index a72d863..9c4280b 100644 (file)
@@ -1,5 +1,5 @@
 application: webkit-perf
-version: 19
+version: 20
 runtime: python27
 api_version: 1
 threadsafe: false
index 45ed797..6db24b9 100644 (file)
@@ -202,6 +202,7 @@ class TestResult(db.Model):
     valueStdev = db.FloatProperty()
     valueMin = db.FloatProperty()
     valueMax = db.FloatProperty()
+    values = db.ListProperty(float)
 
     @staticmethod
     def key_name(build, test_name):
@@ -222,7 +223,8 @@ class TestResult(db.Model):
 
         return cls.get_or_insert(key_name, name=test_name, build=build, value=float(result['avg']),
             valueMedian=_float_or_none(result, 'median'), valueStdev=_float_or_none(result, 'stdev'),
-            valueMin=_float_or_none(result, 'min'), valueMax=_float_or_none(result, 'max'))
+            valueMin=_float_or_none(result, 'min'), valueMax=_float_or_none(result, 'max'),
+            values=result.get('values', []))
 
     def replace_to_change_test_name(self, new_name):
         clone = TestResult(key_name=TestResult.key_name(self.build, new_name), name=new_name, build=self.build,
@@ -247,9 +249,12 @@ class ReportLog(db.Model):
         return self._parsed
 
     def get_value(self, keyName):
-        if not self._parsed_payload():
+        parsed = self._parsed_payload()
+        if not parsed:
             return None
-        return self._parsed.get(keyName)
+        if isinstance(parsed, list):
+            parsed = parsed[0]
+        return parsed.get(keyName)
 
     def results(self):
         return self.get_value('results')
@@ -265,13 +270,22 @@ class ReportLog(db.Model):
             except ValueError:
                 return False
 
+        if isinstance(self._parsed_payload(), list) and len(self._parsed_payload()) != 1:
+            return False
+
         if not isinstance(self.results(), dict):
             return False
 
         for testResult in self.results().values():
             if isinstance(testResult, dict):
                 for key, value in testResult.iteritems():
-                    if key != "unit" and not _is_float_convertible(value):
+                    if key == "values":
+                        if not isinstance(value, list):
+                            return False
+                        for item in value:
+                            if not _is_float_convertible(item):
+                                return False
+                    elif key != "unit" and not _is_float_convertible(value):
                         return False
                 if 'avg' not in testResult:
                     return False
index 2305986..5d56940 100644 (file)
@@ -350,6 +350,14 @@ class TestResultTests(DataStoreTestsBase):
         self.assertEqual(result.valueStdev, 3.25)
         self.assertEqual(result.valueMin, 30.5)
         self.assertEqual(result.valueMax, 45)
+        self.assertEqual(result.values, [])
+
+    def test_get_or_insert_stat_value_with_values(self):
+        branch, platform, builder = _create_some_builder()
+        build = _create_build(branch, platform, builder)
+        result = TestResult.get_or_insert_from_parsed_json('some-test', build,
+            {"avg": 40, "median": "40.1", "stdev": 3.25, "min": 30.5, "max": 45, "values": [1.0, 2.0, 3.0]})
+        self.assertEqual(result.values, [1.0, 2.0, 3.0])
 
     def test_replace_to_change_test_name(self):
         branch, platform, builder = _create_some_builder()
@@ -451,6 +459,12 @@ class ReportLogTests(DataStoreTestsBase):
         assert_results_are_well_formed('{"results": {"test": {"avg": 456, "median": "hello"}}}', False)
         assert_results_are_well_formed('{"results": {"test": {"avg": 456, "median": 789}}}', True)
         assert_results_are_well_formed('{"results": {"test": {"avg": 456, "unit": "bytes"}}}', True)
+        assert_results_are_well_formed('{"results": {"test": {"avg": 456, "unit": "bytes", "values": [1.0, 2.0, 3.0]}}}', True)
+
+        assert_results_are_well_formed('[]', False)
+        assert_results_are_well_formed('[{"results": {"test": 123}}]', True)
+        assert_results_are_well_formed('[{"results": {"test": 123}}, {"results": {"test": 123}}]', False)
+        assert_results_are_well_formed('[{"results": {"test": {"avg": 456, "unit": "bytes", "values": [1.0, 2.0, 3.0]}}}]', True)
 
     def test_builder(self):
         log = self._create_log_with_payload('{"key": "value"}')
@@ -500,12 +514,19 @@ class ReportLogTests(DataStoreTestsBase):
         log = self._create_log_with_payload('{"webkit-revision": 123}')
         self.assertEqual(log.webkit_revision(), 123)
 
-    def chromium_revision(self):
+    def test_chromium_revision(self):
         log = self._create_log_with_payload('{"chromium-revision": 123}')
-        self.assertEqual(log.webkit_revision(), 123)
+        self.assertEqual(log.chromium_revision(), 123)
 
         log = self._create_log_with_payload('{"key": "value"}')
-        self.assertEqual(log.webkit_revision(), None)
+        self.assertEqual(log.chromium_revision(), None)
+
+    def test_results_in_array(self):
+        platform = Platform.create_if_possible("some-platform", "Some Platform")
+        log = self._create_log_with_payload('[{"platform": "some-platform", "build-number": 123, "webkit-revision": 456}]')
+        self.assertEqual(log.platform().key(), platform.key())
+        self.assertEqual(log.build_number(), 123)
+        self.assertEqual(log.webkit_revision(), 456)
 
 
 class PersistentCacheTests(DataStoreTestsBase):