REGRESSION (r216694 - 216712): Performance test Speedometer/Full.html is failing
authorrniwa@webkit.org <rniwa@webkit.org@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Fri, 19 May 2017 03:05:07 +0000 (03:05 +0000)
committerrniwa@webkit.org <rniwa@webkit.org@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Fri, 19 May 2017 03:05:07 +0000 (03:05 +0000)
https://bugs.webkit.org/show_bug.cgi?id=172077

Reviewed by Chris Dumez.

PerformanceTests:

Rename Full.html to index.html to match the convention of other benchmarks.

* Speedometer/index.html: Renamed from PerformanceTests/Speedometer/Full.html.

Tools:

Fixed perftest.py by updating the lines to ignore in Speedometer.

Also start report the results as "Speedometer" instead of "DoYouEvenBench/Full.html" to perf.webkit.org.

* Scripts/webkitpy/performance_tests/perftest.py:
(PerfTest._lines_to_ignore_in_parser_result): Updated for Speedometer 2.0.
* Scripts/webkitpy/performance_tests/perftestsrunner.py:
(PerfTestsRunner._collect_tests): Use "Speedometer" instead of "Speedometer/index.html" as the test name.
(_generate_results_dict): Removed the workaround to keep reporting Speedometer as DoYouEvenBench.
* Scripts/webkitpy/performance_tests/perftestsrunner_unittest.py:
(MainTest.test_collect_tests_with_index_html_and_resources): Added.

git-svn-id: https://svn.webkit.org/repository/webkit/trunk@217089 268f45cc-cd09-0410-ab3c-d52691b4dbfc

PerformanceTests/ChangeLog
PerformanceTests/Speedometer/index.html [moved from PerformanceTests/Speedometer/Full.html with 100% similarity]
Tools/ChangeLog
Tools/Scripts/webkitpy/performance_tests/perftest.py
Tools/Scripts/webkitpy/performance_tests/perftestsrunner.py
Tools/Scripts/webkitpy/performance_tests/perftestsrunner_unittest.py

index a9b741a..9874522 100644 (file)
@@ -1,3 +1,14 @@
+2017-05-18  Ryosuke Niwa  <rniwa@webkit.org>
+
+        REGRESSION (r216694 - 216712): Performance test Speedometer/Full.html is failing
+        https://bugs.webkit.org/show_bug.cgi?id=172077
+
+        Reviewed by Chris Dumez.
+
+        Rename Full.html to index.html to match the convention of other benchmarks.
+
+        * Speedometer/index.html: Renamed from PerformanceTests/Speedometer/Full.html.
+
 2017-05-18  Addy Osmani  <addyosmani@gmail.com>
 
         Speedometer: check-in missing React TodoMVC dependency
index 971ec70..1b0c3da 100644 (file)
@@ -1,3 +1,22 @@
+2017-05-18  Ryosuke Niwa  <rniwa@webkit.org>
+
+        REGRESSION (r216694 - 216712): Performance test Speedometer/Full.html is failing
+        https://bugs.webkit.org/show_bug.cgi?id=172077
+
+        Reviewed by Chris Dumez.
+
+        Fixed perftest.py by updating the lines to ignore in Speedometer.
+
+        Also start report the results as "Speedometer" instead of "DoYouEvenBench/Full.html" to perf.webkit.org.
+
+        * Scripts/webkitpy/performance_tests/perftest.py:
+        (PerfTest._lines_to_ignore_in_parser_result): Updated for Speedometer 2.0.
+        * Scripts/webkitpy/performance_tests/perftestsrunner.py:
+        (PerfTestsRunner._collect_tests): Use "Speedometer" instead of "Speedometer/index.html" as the test name.
+        (_generate_results_dict): Removed the workaround to keep reporting Speedometer as DoYouEvenBench.
+        * Scripts/webkitpy/performance_tests/perftestsrunner_unittest.py:
+        (MainTest.test_collect_tests_with_index_html_and_resources): Added.
+
 2017-05-18  Andy Estes  <aestes@apple.com>
 
         ENABLE(APPLE_PAY_DELEGATE) should be NO on macOS Sierra and earlier
index d1cece8..b9d8717 100644 (file)
@@ -251,22 +251,10 @@ class PerfTest(object):
         re.compile(re.escape("""Blocked access to external URL http://www.whatwg.org/specs/web-apps/current-work/""")),
         re.compile(r"CONSOLE MESSAGE: (line \d+: )?Blocked script execution in '[A-Za-z0-9\-\.:]+' because the document's frame is sandboxed and the 'allow-scripts' permission is not set."),
         re.compile(r"CONSOLE MESSAGE: (line \d+: )?Not allowed to load local resource"),
-        # DoYouEvenBench
-        re.compile(re.escape("CONSOLE MESSAGE: line 140: Miss the info bar? Run TodoMVC from a server to avoid a cross-origin error.")),
-        re.compile(re.escape("CONSOLE MESSAGE: line 315: TypeError: Attempted to assign to readonly property.")),
-        re.compile(re.escape("CONSOLE MESSAGE: line 3285: DEBUG: -------------------------------")),
-        re.compile(re.escape("CONSOLE MESSAGE: line 3285: DEBUG: Ember      : 1.3.1")),
-        re.compile(re.escape("CONSOLE MESSAGE: line 3285: DEBUG: Ember Data : 1.0.0-beta.6")),
-        re.compile(re.escape("CONSOLE MESSAGE: line 3285: DEBUG: Handlebars : 1.3.0")),
-        re.compile(re.escape("CONSOLE MESSAGE: line 3285: DEBUG: jQuery     : 2.1.0")),
-        re.compile(re.escape("CONSOLE MESSAGE: line 3285: DEPRECATION: Namespaces should not begin with lowercase")),
-        re.compile(re.escape("processAllNamespaces@app.js:2:40")),
-        re.compile(re.escape("processAllNamespaces@jquery.js:3380:17")),
-        re.compile(re.escape("CONSOLE MESSAGE: line 124: Booting in DEBUG mode")),
-        re.compile(re.escape("CONSOLE MESSAGE: line 125: You can configure event logging with DEBUG.events.logAll()/logNone()/logByName()/logByAction()")),
-        re.compile(re.escape("CONSOLE MESSAGE: line 3285: Ember Views require jQuery 1.7, 1.8, 1.9, 1.10, or 2.0")),
-        re.compile(re.escape("CONSOLE MESSAGE: line 3285: DEPRECATION: Namespaces should not begin with lowercase.")),
-        re.compile(re.escape("CONSOLE MESSAGE: line 208: Miss the info bar? Run TodoMVC from a server to avoid a cross-origin error."))
+        # Speedometer 2.0
+        re.compile(r'CONSOLE MESSAGE: line \d+: DEBUG: -------------------------------'),
+        re.compile(r'CONSOLE MESSAGE: line \d+: DEBUG: Ember\s+: (\d\.)+'),
+        re.compile(r'CONSOLE MESSAGE: line \d+: DEBUG: jQuery\s+: (\d\.)+'),
     ]
 
     def _filter_output(self, output):
index eb04571..6db845d 100644 (file)
@@ -179,6 +179,8 @@ class PerfTestsRunner(object):
             relative_path = filesystem.relpath(path, self._base_path).replace('\\', '/')
             if self._options.use_skipped_list and self._port.skips_perf_test(relative_path) and filesystem.normpath(relative_path) not in paths:
                 continue
+            if relative_path.endswith('/index.html'):
+                relative_path = relative_path[0:-len('/index.html')]
             test = PerfTestFactory.create_perf_test(self._port, relative_path, path, test_runner_count=test_runner_count)
             tests.append(test)
 
@@ -294,10 +296,6 @@ class PerfTestsRunner(object):
                 url = view_source_url('PerformanceTests/' + '/'.join(path[0:i + 1]))
                 test_name = path[i]
 
-                # FIXME: This is a temporary workaround for the fact perf dashboard doesn't support renaming tests.
-                if test_name == 'Speedometer':
-                    test_name = 'DoYouEvenBench'
-
                 tests.setdefault(test_name, {'url': url})
                 current_test = tests[test_name]
                 if is_last_token:
index 36d3dde..e209c65 100644 (file)
@@ -80,6 +80,14 @@ class MainTest(unittest.TestCase):
         port.host.filesystem.chdir(runner._port.perf_tests_dir()[:runner._port.perf_tests_dir().rfind(runner._host.filesystem.sep)])
         self.assertItemsEqual(self._collect_tests_and_sort_test_name(runner), ['test1.html', 'test2.html'])
 
+    def test_collect_tests_with_index_html_and_resources(self):
+        runner, port = self.create_runner()
+        self._add_file(runner, 'Speedometer', 'index.html', 'test content')
+        self._add_file(runner, 'Speedometer/resources', 'resource.html', 'resource content')
+        tests = runner._collect_tests()
+        self.assertEqual(len(tests), 1)
+        self.assertEqual(tests[0].test_name(), 'Speedometer')
+
     def test_collect_tests_with_skipped_list(self):
         runner, port = self.create_runner()