Enable SVG page loading performance tests
authorrniwa@webkit.org <rniwa@webkit.org@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Fri, 13 Apr 2012 18:16:26 +0000 (18:16 +0000)
committerrniwa@webkit.org <rniwa@webkit.org@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Fri, 13 Apr 2012 18:16:26 +0000 (18:16 +0000)
https://bugs.webkit.org/show_bug.cgi?id=83861

Reviewed by Adam Barth.

PerformanceTests:

Unskip PageLoad directory and skip tests that load too fast (less than 100ms) and are unstable,
and the ones that take too long to load (more than 20s).

* Skipped:

Tools:

Add support for page loading tests. These are tests without any test running harnesses, and we use
DriverOutput's test_timeout to measure the loading time.

Extracted PerfTest.run and PerfTest.run_failed from PerfTestRunner._run_single_test.

* Scripts/webkitpy/performance_tests/perftest.py:
(PerfTest.run):
(PerfTest):
(PerfTest.run_failed):
(PerfTest.parse_output):
(PerfTest.output_statistics):
(ChromiumStylePerfTest.parse_output):
(PageLoadingPerfTest): Added.
(PageLoadingPerfTest.__init__):
(PageLoadingPerfTest.run): Run the driver 20 times to get the statistics for the loading time.
* Scripts/webkitpy/performance_tests/perftest_unittest.py: Added.
(MockPrinter):
(MockPrinter.__init__):
(MockPrinter.write):
(MainTest):
(MainTest.test_parse_output):
(MainTest.test_parse_output_with_failing_line):
(TestPageLoadingPerfTest):
(TestPageLoadingPerfTest.assertWritten):
(TestPageLoadingPerfTest.MockDriver):
(TestPageLoadingPerfTest.MockDriver.__init__):
(TestPageLoadingPerfTest.MockDriver.run_test):
(TestPageLoadingPerfTest.test_run):
(TestPageLoadingPerfTest.test_run_with_bad_output):
* Scripts/webkitpy/performance_tests/perftestsrunner.py:
(PerfTestsRunner):
(PerfTestsRunner._collect_tests._is_test_file): Now recognizes svg tests.
(PerfTestsRunner._collect_tests):
(PerfTestsRunner._run_tests_set):
(PerfTestsRunner._run_single_test):
* Scripts/webkitpy/performance_tests/perftestsrunner_unittest.py:
(test_collect_tests_with_page_load_svg):
(test_collect_tests_with_page_load_svg.add_file):

git-svn-id: https://svn.webkit.org/repository/webkit/trunk@114149 268f45cc-cd09-0410-ab3c-d52691b4dbfc

PerformanceTests/ChangeLog
PerformanceTests/Skipped
Tools/ChangeLog
Tools/Scripts/webkitpy/performance_tests/perftest.py
Tools/Scripts/webkitpy/performance_tests/perftest_unittest.py [new file with mode: 0755]
Tools/Scripts/webkitpy/performance_tests/perftestsrunner.py
Tools/Scripts/webkitpy/performance_tests/perftestsrunner_unittest.py

index b499ff4e5b84c10f6c56b07ac2d99b7b9c9068c9..c5e80d1b0673b2cb6c052ec6ce42ad52429dd15b 100644 (file)
@@ -1,3 +1,15 @@
+2012-04-12  Ryosuke Niwa  <rniwa@webkit.org>
+
+        Enable SVG page loading performance tests
+        https://bugs.webkit.org/show_bug.cgi?id=83861
+
+        Reviewed by Adam Barth.
+
+        Unskip PageLoad directory and skip tests that load too fast (less than 100ms) and are unstable,
+        and the ones that take too long to load (more than 20s).
+
+        * Skipped:
+
 2012-04-04  Ilya Tikhonovsky  <loislo@chromium.org>
 
         Web Inspector: Unreviewed. temporary disable broken inspector tests.
index 08ad7e6f78f4c33a6e4ff62936629088a2c25e45..c3134afbf8cc892c23ff9aec913b5d89305c1905 100644 (file)
@@ -1,8 +1,17 @@
 # Not enabled by default on some ports
 Mutation
 
+# Unstable SVG page loading tests
+PageLoad/svg/files/33041-Samurai.svg
+PageLoad/svg/files/Sierpinski_carpet_6.svg # Takes too long to load
+PageLoad/svg/files/az-lizard_benji_park_01.svg
+PageLoad/svg/files/france.svg
+PageLoad/svg/files/francobollo_gnome_ezechi_02.svg
+PageLoad/svg/files/gearflowers.svg
+PageLoad/svg/files/hereGear4.svg
+PageLoad/svg/files/mtsthelens.svg # Takes 22s on MacBookPro
+
 # Do not conform to WebKit or Chromium perf test styles
-PageLoad
 SunSpider
 XSSAuditor
 
index c5d42c1a563af35662c54088e2a2d801692ebeaf..d8033270a02cc62b640414088fa0cb60690bb647 100644 (file)
@@ -1,3 +1,49 @@
+2012-04-12  Ryosuke Niwa  <rniwa@webkit.org>
+
+        Enable SVG page loading performance tests
+        https://bugs.webkit.org/show_bug.cgi?id=83861
+
+        Reviewed by Adam Barth.
+
+        Add support for page loading tests. These are tests without any test running harnesses, and we use
+        DriverOutput's test_timeout to measure the loading time.
+
+        Extracted PerfTest.run and PerfTest.run_failed from PerfTestRunner._run_single_test.
+
+        * Scripts/webkitpy/performance_tests/perftest.py:
+        (PerfTest.run):
+        (PerfTest):
+        (PerfTest.run_failed):
+        (PerfTest.parse_output):
+        (PerfTest.output_statistics):
+        (ChromiumStylePerfTest.parse_output):
+        (PageLoadingPerfTest): Added.
+        (PageLoadingPerfTest.__init__):
+        (PageLoadingPerfTest.run): Run the driver 20 times to get the statistics for the loading time.
+        * Scripts/webkitpy/performance_tests/perftest_unittest.py: Added.
+        (MockPrinter):
+        (MockPrinter.__init__):
+        (MockPrinter.write):
+        (MainTest):
+        (MainTest.test_parse_output):
+        (MainTest.test_parse_output_with_failing_line):
+        (TestPageLoadingPerfTest):
+        (TestPageLoadingPerfTest.assertWritten):
+        (TestPageLoadingPerfTest.MockDriver):
+        (TestPageLoadingPerfTest.MockDriver.__init__):
+        (TestPageLoadingPerfTest.MockDriver.run_test):
+        (TestPageLoadingPerfTest.test_run):
+        (TestPageLoadingPerfTest.test_run_with_bad_output):
+        * Scripts/webkitpy/performance_tests/perftestsrunner.py:
+        (PerfTestsRunner):
+        (PerfTestsRunner._collect_tests._is_test_file): Now recognizes svg tests.
+        (PerfTestsRunner._collect_tests):
+        (PerfTestsRunner._run_tests_set):
+        (PerfTestsRunner._run_single_test):
+        * Scripts/webkitpy/performance_tests/perftestsrunner_unittest.py:
+        (test_collect_tests_with_page_load_svg):
+        (test_collect_tests_with_page_load_svg.add_file):
+
 2012-04-10  Brady Eidson  <beidson@apple.com>
 
         <rdar://problem/11176921> and https://bugs.webkit.org/show_bug.cgi?id=83600
index 89adcf675997fe3ddd902d5c23564e5ca6070bb5..e3f69ac0c2bdee79a18f6f806dfd7b7e44f9b9c5 100644 (file)
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
+import math
 import re
 
+from webkitpy.layout_tests.port.driver import DriverInput
+
 
 class PerfTest(object):
     def __init__(self, test_name, dirname, path_or_url):
@@ -46,6 +49,27 @@ class PerfTest(object):
     def path_or_url(self):
         return self._path_or_url
 
+    def run(self, driver, timeout_ms, printer, buildbot_output):
+        output = driver.run_test(DriverInput(self.path_or_url(), timeout_ms, None, False))
+        if self.run_failed(output, printer):
+            return None
+        return self.parse_output(output, printer, buildbot_output)
+
+    def run_failed(self, output, printer):
+        if output.text == None or output.error:
+            pass
+        elif output.timeout:
+            printer.write('timeout: %s' % self.test_name())
+        elif output.crash:
+            printer.write('crash: %s' % self.test_name())
+        else:
+            return False
+
+        if output.error:
+            printer.write('error: %s\n%s' % (self.test_name(), output.error))
+
+        return True
+
     _lines_to_ignore_in_parser_result = [
         re.compile(r'^Running \d+ times$'),
         re.compile(r'^Ignoring warm-up '),
@@ -56,6 +80,8 @@ class PerfTest(object):
         re.compile(re.escape("""frame "<!--framePath //<!--frame0-->-->" - has 1 onunload handler(s)""")),
         re.compile(re.escape("""frame "<!--framePath //<!--frame0-->/<!--frame0-->-->" - has 1 onunload handler(s)"""))]
 
+    _statistics_keys = ['avg', 'median', 'stdev', 'min', 'max']
+
     def _should_ignore_line_in_parser_test_result(self, line):
         if not line:
             return True
@@ -68,8 +94,7 @@ class PerfTest(object):
         got_a_result = False
         test_failed = False
         results = {}
-        keys = ['avg', 'median', 'stdev', 'min', 'max']
-        score_regex = re.compile(r'^(?P<key>' + r'|'.join(keys) + r')\s+(?P<value>[0-9\.]+)\s*(?P<unit>.*)')
+        score_regex = re.compile(r'^(?P<key>' + r'|'.join(self._statistics_keys) + r')\s+(?P<value>[0-9\.]+)\s*(?P<unit>.*)')
         unit = "ms"
 
         for line in re.split('\n', output.text):
@@ -84,17 +109,21 @@ class PerfTest(object):
                 test_failed = True
                 printer.write("%s" % line)
 
-        if test_failed or set(keys) != set(results.keys()):
+        if test_failed or set(self._statistics_keys) != set(results.keys()):
             return None
 
         results['unit'] = unit
 
         test_name = re.sub(r'\.\w+$', '', self._test_name)
-        buildbot_output.write('RESULT %s= %s %s\n' % (test_name.replace('/', ': '), results['avg'], unit))
-        buildbot_output.write(', '.join(['%s= %s %s' % (key, results[key], unit) for key in keys[1:]]) + '\n')
+        self.output_statistics(test_name, results, buildbot_output)
 
         return {test_name: results}
 
+    def output_statistics(self, test_name, results, buildbot_output):
+        unit = results['unit']
+        buildbot_output.write('RESULT %s= %s %s\n' % (test_name.replace('/', ': '), results['avg'], unit))
+        buildbot_output.write(', '.join(['%s= %s %s' % (key, results[key], unit) for key in self._statistics_keys[1:]]) + '\n')
+
 
 class ChromiumStylePerfTest(PerfTest):
     _chromium_style_result_regex = re.compile(r'^RESULT\s+(?P<name>[^=]+)\s*=\s+(?P<value>\d+(\.\d+)?)\s*(?P<unit>\w+)$')
@@ -116,3 +145,41 @@ class ChromiumStylePerfTest(PerfTest):
                 test_failed = True
                 printer.write("%s" % line)
         return results if results and not test_failed else None
+
+
+class PageLoadingPerfTest(PerfTest):
+    def __init__(self, test_name, dirname, path_or_url):
+        super(PageLoadingPerfTest, self).__init__(test_name, dirname, path_or_url)
+
+    def run(self, driver, timeout_ms, printer, buildbot_output):
+        test_times = []
+
+        for i in range(0, 2):
+            output = driver.run_test(DriverInput(self.path_or_url(), timeout_ms, None, False))
+            if self.run_failed(output, printer):
+                return None
+            if i == 0:
+                continue
+            test_times.append(output.test_time * 1000)
+
+        test_times = sorted(test_times)
+
+        # Compute the mean and variance using a numerically stable algorithm.
+        squareSum = 0
+        mean = 0
+        valueSum = sum(test_times)
+        for i, time in enumerate(test_times):
+            delta = time - mean
+            sweep = i + 1.0
+            mean += delta / sweep
+            squareSum += delta * delta * (i / sweep)
+
+        middle = int(len(test_times) / 2)
+        results = {'avg': mean,
+            'min': min(test_times),
+            'max': max(test_times),
+            'median': test_times[middle] if len(test_times) % 2 else (test_times[middle - 1] + test_times[middle]) / 2,
+            'stdev': math.sqrt(squareSum),
+            'unit': 'ms'}
+        self.output_statistics(self.test_name(), results, buildbot_output)
+        return {self.test_name(): results}
diff --git a/Tools/Scripts/webkitpy/performance_tests/perftest_unittest.py b/Tools/Scripts/webkitpy/performance_tests/perftest_unittest.py
new file mode 100755 (executable)
index 0000000..ade7d03
--- /dev/null
@@ -0,0 +1,121 @@
+#!/usr/bin/python
+# Copyright (C) 2012 Google Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import StringIO
+import math
+import unittest
+
+from webkitpy.layout_tests.port.driver import DriverOutput
+from webkitpy.performance_tests.perftest import ChromiumStylePerfTest
+from webkitpy.performance_tests.perftest import PageLoadingPerfTest
+from webkitpy.performance_tests.perftest import PerfTest
+
+
+class MockPrinter(object):
+    def __init__(self):
+        self.written_lines = []
+
+    def write(self, line):
+        self.written_lines.append(line)
+
+
+class MainTest(unittest.TestCase):
+    def test_parse_output(self):
+        printer = MockPrinter()
+        buildbot_output = StringIO.StringIO()
+        output = DriverOutput('\n'.join([
+            'Running 20 times',
+            'Ignoring warm-up run (1115)',
+            '',
+            'avg 1100',
+            'median 1101',
+            'stdev 11',
+            'min 1080',
+            'max 1120']), image=None, image_hash=None, audio=None)
+        test = PerfTest('some-test', 'some-dir/some-test', '/path/some-dir/some-test')
+        self.assertEqual(test.parse_output(output, printer, buildbot_output),
+            {'some-test': {'avg': 1100.0, 'median': 1101.0, 'min': 1080.0, 'max': 1120.0, 'stdev': 11.0, 'unit': 'ms'}})
+        self.assertEqual(printer.written_lines, [])
+
+    def test_parse_output_with_failing_line(self):
+        printer = MockPrinter()
+        buildbot_output = StringIO.StringIO()
+        output = DriverOutput('\n'.join([
+            'Running 20 times',
+            'Ignoring warm-up run (1115)',
+            '',
+            'some-unrecognizable-line',
+            '',
+            'avg 1100',
+            'median 1101',
+            'stdev 11',
+            'min 1080',
+            'max 1120']), image=None, image_hash=None, audio=None)
+        test = PerfTest('some-test', 'some-dir/some-test', '/path/some-dir/some-test')
+        self.assertEqual(test.parse_output(output, printer, buildbot_output), None)
+        self.assertEqual(printer.written_lines, ['some-unrecognizable-line'])
+
+
+class TestPageLoadingPerfTest(unittest.TestCase):
+    def assertWritten(self, stream, contents):
+        self.assertEquals(stream.buflist, contents)
+
+    class MockDriver(object):
+        def __init__(self, values):
+            self._values = values
+            self._index = 0
+
+        def run_test(self, input):
+            value = self._values[self._index]
+            self._index += 1
+            if isinstance(value, str):
+                return DriverOutput('some output', image=None, image_hash=None, audio=None, error=value)
+            else:
+                return DriverOutput('some output', image=None, image_hash=None, audio=None, test_time=self._values[self._index - 1])
+
+    def test_run(self):
+        printer = MockPrinter()
+        buildbot_output = StringIO.StringIO()
+        test = PageLoadingPerfTest('some-test', 'some-dir/some-test', '/path/some-dir/some-test')
+        driver = TestPageLoadingPerfTest.MockDriver([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20])
+        self.assertEqual(test.run(driver, None, printer, buildbot_output),
+            {'some-test': {'max': 20000, 'avg': 10500.0, 'median': 10500, 'stdev': math.sqrt(665 * 1000 * 1000), 'min': 1000, 'unit': 'ms'}})
+        self.assertEqual(printer.written_lines, [])
+        self.assertWritten(buildbot_output, ['RESULT some-test= 10500.0 ms\n', 'median= 10500 ms, stdev= 25787.5939165 ms, min= 1000 ms, max= 20000 ms\n'])
+
+    def test_run_with_bad_output(self):
+        printer = MockPrinter()
+        buildbot_output = StringIO.StringIO()
+        test = PageLoadingPerfTest('some-test', 'some-dir/some-test', '/path/some-dir/some-test')
+        driver = TestPageLoadingPerfTest.MockDriver([1, 2, 3, 4, 5, 6, 7, 'some error', 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20])
+        self.assertEqual(test.run(driver, None, printer, buildbot_output), None)
+        self.assertEqual(printer.written_lines, ['error: some-test\nsome error'])
+
+if __name__ == '__main__':
+    unittest.main()
index ae62458d99da7592a852fac7d3a86eb628996602..61ce1d91a35dd4454a1d92a8e17dc85a2886134f 100644 (file)
@@ -39,16 +39,18 @@ import time
 from webkitpy.common import find_files
 from webkitpy.common.host import Host
 from webkitpy.common.net.file_uploader import FileUploader
-from webkitpy.layout_tests.port.driver import DriverInput
 from webkitpy.layout_tests.views import printing
 from webkitpy.performance_tests.perftest import ChromiumStylePerfTest
+from webkitpy.performance_tests.perftest import PageLoadingPerfTest
 from webkitpy.performance_tests.perftest import PerfTest
 
+
 _log = logging.getLogger(__name__)
 
 
 class PerfTestsRunner(object):
-    _test_directories_for_chromium_style_tests = ['inspector']
+    _pattern_for_chromium_style_tests = re.compile('^inspector/')
+    _pattern_for_page_loading_tests = re.compile('^PageLoad/')
     _default_branch = 'webkit-trunk'
     _EXIT_CODE_BAD_BUILD = -1
     _EXIT_CODE_BAD_JSON = -2
@@ -112,7 +114,7 @@ class PerfTestsRunner(object):
         """Return the list of tests found."""
 
         def _is_test_file(filesystem, dirname, filename):
-            return filename.endswith('.html')
+            return filesystem.splitext(filename)[1] in ['.html', '.svg']
 
         filesystem = self._host.filesystem
 
@@ -132,8 +134,10 @@ class PerfTestsRunner(object):
                 continue
             test_name = relative_path.replace('\\', '/')
             dirname = filesystem.dirname(path)
-            if self._host.filesystem.dirname(relative_path) in self._test_directories_for_chromium_style_tests:
+            if self._pattern_for_chromium_style_tests.match(relative_path):
                 tests.append(ChromiumStylePerfTest(test_name, dirname, path))
+            elif self._pattern_for_page_loading_tests.match(relative_path):
+                tests.append(PageLoadingPerfTest(test_name, dirname, path))
             else:
                 tests.append(PerfTest(test_name, dirname, path))
 
@@ -255,42 +259,10 @@ class PerfTestsRunner(object):
 
         return unexpected
 
-    _inspector_result_regex = re.compile(r'^RESULT\s+(?P<name>[^=]+)\s*=\s+(?P<value>\d+(\.\d+)?)\s*(?P<unit>\w+)$')
-
-    def _process_chromium_style_test_result(self, test, output):
-        test_failed = False
-        got_a_result = False
-        for line in re.split('\n', output.text):
-            resultLine = self._inspector_result_regex.match(line)
-            if resultLine:
-                # FIXME: Store the unit
-                self._results[resultLine.group('name').replace(' ', '')] = float(resultLine.group('value'))
-                self._buildbot_output.write("%s\n" % line)
-                got_a_result = True
-            elif not len(line) == 0:
-                test_failed = True
-                self._printer.write("%s" % line)
-        return test_failed or not got_a_result
-
     def _run_single_test(self, test, driver):
         start_time = time.time()
 
-        output = driver.run_test(DriverInput(test.path_or_url(), self._options.time_out_ms, None, False))
-        new_results = None
-
-        if output.text == None:
-            pass
-        elif output.timeout:
-            self._printer.write('timeout: %s' % test.test_name())
-        elif output.crash:
-            self._printer.write('crash: %s' % test.test_name())
-        else:
-            new_results = test.parse_output(output, self._printer, self._buildbot_output)
-
-        if len(output.error):
-            self._printer.write('error:\n%s' % output.error)
-            new_results = None
-
+        new_results = test.run(driver, self._options.time_out_ms, self._printer, self._buildbot_output)
         if new_results:
             self._results.update(new_results)
         else:
index 6f04ed8d08942154292a1d13a6fa7ef00faf7714..b92655561f06be3a30eeca1fb41b91207f55bdbc 100755 (executable)
@@ -375,6 +375,19 @@ max 1120
         runner._port.skipped_perf_tests = lambda: ['inspector/unsupported_test1.html', 'unsupported']
         self.assertEqual(self._collect_tests_and_sort_test_name(runner), ['inspector/test1.html', 'inspector/test2.html'])
 
+    def test_collect_tests_with_page_load_svg(self):
+        runner = self.create_runner()
+
+        def add_file(dirname, filename, content=True):
+            dirname = runner._host.filesystem.join(runner._base_path, dirname) if dirname else runner._base_path
+            runner._host.filesystem.maybe_make_directory(dirname)
+            runner._host.filesystem.files[runner._host.filesystem.join(dirname, filename)] = content
+
+        add_file('PageLoad', 'some-svg-test.svg')
+        tests = runner._collect_tests()
+        self.assertEqual(len(tests), 1)
+        self.assertEqual(tests[0].__class__.__name__, 'PageLoadingPerfTest')
+
     def test_parse_args(self):
         runner = self.create_runner()
         options, args = PerfTestsRunner._parse_args([