Force GC between PageLoad tests.
authorpdr@google.com <pdr@google.com@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Thu, 4 Oct 2012 06:29:45 +0000 (06:29 +0000)
committerpdr@google.com <pdr@google.com@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Thu, 4 Oct 2012 06:29:45 +0000 (06:29 +0000)
https://bugs.webkit.org/show_bug.cgi?id=98203

Reviewed by Ryosuke Niwa.

Previously, our PageLoad PerfTests had multi-modal distributions,
typically with a small cluster at 1-2x the median. This turned out
to be caused by not garbage collecting between tests!

This patch adds a new file, force-gc.html, and loads this file between
PageLoad tests to force a GC. I manually verified that this cleans up
our perf test outliers.

PerformanceTests:

* resources/force-gc.html: Added.

Tools:

* Scripts/webkitpy/performance_tests/perftest.py:
(PageLoadingPerfTest.__init__):
(PageLoadingPerfTest):
(PageLoadingPerfTest.run_single):

    This function now loads two pages: one to force a gc and
    then the test to run.

* Scripts/webkitpy/performance_tests/perftest_unittest.py:

    Modified several existing tests to show that the force-gc file
    is loaded.

(MockPort):
(MockPort.__init__):
(MockPort.perf_tests_dir):
(TestPageLoadingPerfTest.MockDriver.__init__):
(TestPageLoadingPerfTest.MockDriver.run_test):
(TestPageLoadingPerfTest.test_run):
(TestPageLoadingPerfTest.test_run_with_bad_output):
(TestReplayPerfTest.ReplayTestPort):
(TestReplayPerfTest.ReplayTestPort.__init__):
(TestReplayPerfTest.test_run_single.run_test):
(TestReplayPerfTest.test_run_single):
(TestReplayPerfTest.test_run_single_fails_when_output_has_error):
(TestPerfTestFactory.test_regular_test):
(TestPerfTestFactory.test_inspector_test):
(TestPerfTestFactory.test_page_loading_test):

git-svn-id: https://svn.webkit.org/repository/webkit/trunk@130366 268f45cc-cd09-0410-ab3c-d52691b4dbfc

PerformanceTests/ChangeLog
PerformanceTests/resources/force-gc.html [new file with mode: 0644]
Tools/ChangeLog
Tools/Scripts/webkitpy/performance_tests/perftest.py
Tools/Scripts/webkitpy/performance_tests/perftest_unittest.py

index d5e27d7..b7da049 100644 (file)
@@ -1,3 +1,20 @@
+2012-10-03  Philip Rogers  <pdr@google.com>
+
+        Force GC between PageLoad tests.
+        https://bugs.webkit.org/show_bug.cgi?id=98203
+
+        Reviewed by Ryosuke Niwa.
+
+        Previously, our PageLoad PerfTests had multi-modal distributions,
+        typically with a small cluster at 1-2x the median. This turned out
+        to be caused by not garbage collecting between tests!
+
+        This patch adds a new file, force-gc.html, and loads this file between
+        PageLoad tests to force a GC. I manually verified that this cleans up
+        our perf test outliers.
+
+        * resources/force-gc.html: Added.
+
 2012-10-03  Julien Chaffraix  <jchaffraix@webkit.org>
 
         Add a performance test for subtree detaching
diff --git a/PerformanceTests/resources/force-gc.html b/PerformanceTests/resources/force-gc.html
new file mode 100644 (file)
index 0000000..2053755
--- /dev/null
@@ -0,0 +1,6 @@
+<!DOCTYPE HTML>
+<!-- This page forces a GC in DRT for https://bugs.webkit.org/show_bug.cgi?id=98203. -->
+<html>
+<body onload="window.GCController.collect();">
+</body>
+</html>
index 3ff717c..9ec2279 100644 (file)
@@ -1,3 +1,47 @@
+2012-10-03  Philip Rogers  <pdr@google.com>
+
+        Force GC between PageLoad tests.
+        https://bugs.webkit.org/show_bug.cgi?id=98203
+
+        Reviewed by Ryosuke Niwa.
+
+        Previously, our PageLoad PerfTests had multi-modal distributions,
+        typically with a small cluster at 1-2x the median. This turned out
+        to be caused by not garbage collecting between tests!
+
+        This patch adds a new file, force-gc.html, and loads this file between
+        PageLoad tests to force a GC. I manually verified that this cleans up
+        our perf test outliers.
+
+        * Scripts/webkitpy/performance_tests/perftest.py:
+        (PageLoadingPerfTest.__init__):
+        (PageLoadingPerfTest):
+        (PageLoadingPerfTest.run_single):
+
+            This function now loads two pages: one to force a gc and
+            then the test to run.
+
+        * Scripts/webkitpy/performance_tests/perftest_unittest.py:
+
+            Modified several existing tests to show that the force-gc file
+            is loaded.
+
+        (MockPort):
+        (MockPort.__init__):
+        (MockPort.perf_tests_dir):
+        (TestPageLoadingPerfTest.MockDriver.__init__):
+        (TestPageLoadingPerfTest.MockDriver.run_test):
+        (TestPageLoadingPerfTest.test_run):
+        (TestPageLoadingPerfTest.test_run_with_bad_output):
+        (TestReplayPerfTest.ReplayTestPort):
+        (TestReplayPerfTest.ReplayTestPort.__init__):
+        (TestReplayPerfTest.test_run_single.run_test):
+        (TestReplayPerfTest.test_run_single):
+        (TestReplayPerfTest.test_run_single_fails_when_output_has_error):
+        (TestPerfTestFactory.test_regular_test):
+        (TestPerfTestFactory.test_inspector_test):
+        (TestPerfTestFactory.test_page_loading_test):
+
 2012-10-03  Christophe Dumez  <christophe.dumez@intel.com>
 
         [EFL] Enable use of X11 in DumpRenderTree / WebKitTestRunner
index 762e231..360fe4c 100644 (file)
@@ -202,8 +202,16 @@ class ChromiumStylePerfTest(PerfTest):
 
 
 class PageLoadingPerfTest(PerfTest):
+    _FORCE_GC_FILE = 'resources/force-gc.html'
+
     def __init__(self, port, test_name, path_or_url):
         super(PageLoadingPerfTest, self).__init__(port, test_name, path_or_url)
+        self.force_gc_test = self._port.host.filesystem.join(self._port.perf_tests_dir(), self._FORCE_GC_FILE)
+
+    def run_single(self, driver, path_or_url, time_out_ms, should_run_pixel_test=False):
+        # Force GC to prevent pageload noise. See https://bugs.webkit.org/show_bug.cgi?id=98203
+        super(PageLoadingPerfTest, self).run_single(driver, self.force_gc_test, time_out_ms, False)
+        return super(PageLoadingPerfTest, self).run_single(driver, path_or_url, time_out_ms, should_run_pixel_test)
 
     def run(self, driver, time_out_ms):
         test_times = []
index 6922ba9..4ac7e6c 100755 (executable)
@@ -43,6 +43,10 @@ from webkitpy.performance_tests.perftest import PerfTestFactory
 from webkitpy.performance_tests.perftest import ReplayPerfTest
 
 
+class MockPort(TestPort):
+    def __init__(self, custom_run_test=None):
+        super(MockPort, self).__init__(host=MockHost(), custom_run_test=custom_run_test)
+
 class MainTest(unittest.TestCase):
     def test_parse_output(self):
         output = DriverOutput('\n'.join([
@@ -98,11 +102,14 @@ class MainTest(unittest.TestCase):
 
 class TestPageLoadingPerfTest(unittest.TestCase):
     class MockDriver(object):
-        def __init__(self, values):
+        def __init__(self, values, test):
             self._values = values
             self._index = 0
+            self._test = test
 
         def run_test(self, input, stop_when_done):
+            if input.test_name == self._test.force_gc_test:
+                return
             value = self._values[self._index]
             self._index += 1
             if isinstance(value, str):
@@ -111,8 +118,9 @@ class TestPageLoadingPerfTest(unittest.TestCase):
                 return DriverOutput('some output', image=None, image_hash=None, audio=None, test_time=self._values[self._index - 1])
 
     def test_run(self):
-        test = PageLoadingPerfTest(None, 'some-test', '/path/some-dir/some-test')
-        driver = TestPageLoadingPerfTest.MockDriver(range(1, 21))
+        port = MockPort()
+        test = PageLoadingPerfTest(port, 'some-test', '/path/some-dir/some-test')
+        driver = TestPageLoadingPerfTest.MockDriver(range(1, 21), test)
         output_capture = OutputCapture()
         output_capture.capture_output()
         try:
@@ -129,8 +137,9 @@ class TestPageLoadingPerfTest(unittest.TestCase):
         output_capture = OutputCapture()
         output_capture.capture_output()
         try:
-            test = PageLoadingPerfTest(None, 'some-test', '/path/some-dir/some-test')
-            driver = TestPageLoadingPerfTest.MockDriver([1, 2, 3, 4, 5, 6, 7, 'some error', 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20])
+            port = MockPort()
+            test = PageLoadingPerfTest(port, 'some-test', '/path/some-dir/some-test')
+            driver = TestPageLoadingPerfTest.MockDriver([1, 2, 3, 4, 5, 6, 7, 'some error', 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], test)
             self.assertEqual(test.run(driver, None), None)
         finally:
             actual_stdout, actual_stderr, actual_logs = output_capture.restore_output()
@@ -141,7 +150,7 @@ class TestPageLoadingPerfTest(unittest.TestCase):
 
 class TestReplayPerfTest(unittest.TestCase):
 
-    class ReplayTestPort(TestPort):
+    class ReplayTestPort(MockPort):
         def __init__(self, custom_run_test=None):
 
             class ReplayTestDriver(TestDriver):
@@ -149,7 +158,7 @@ class TestReplayPerfTest(unittest.TestCase):
                     return custom_run_test(text_input, stop_when_done) if custom_run_test else None
 
             self._custom_driver_class = ReplayTestDriver
-            super(self.__class__, self).__init__(host=MockHost())
+            super(self.__class__, self).__init__()
 
         def _driver_class(self):
             return self._custom_driver_class
@@ -179,6 +188,9 @@ class TestReplayPerfTest(unittest.TestCase):
         loaded_pages = []
 
         def run_test(test_input, stop_when_done):
+            if test_input.test_name == test.force_gc_test:
+                loaded_pages.append(test_input)
+                return
             if test_input.test_name != "about:blank":
                 self.assertEqual(test_input.test_name, 'http://some-test/')
             loaded_pages.append(test_input)
@@ -196,8 +208,9 @@ class TestReplayPerfTest(unittest.TestCase):
         finally:
             actual_stdout, actual_stderr, actual_logs = output_capture.restore_output()
 
-        self.assertEqual(len(loaded_pages), 1)
-        self.assertEqual(loaded_pages[0].test_name, 'http://some-test/')
+        self.assertEqual(len(loaded_pages), 2)
+        self.assertEqual(loaded_pages[0].test_name, test.force_gc_test)
+        self.assertEqual(loaded_pages[1].test_name, 'http://some-test/')
         self.assertEqual(actual_stdout, '')
         self.assertEqual(actual_stderr, '')
         self.assertEqual(actual_logs, '')
@@ -262,8 +275,9 @@ class TestReplayPerfTest(unittest.TestCase):
         finally:
             actual_stdout, actual_stderr, actual_logs = output_capture.restore_output()
 
-        self.assertEqual(len(loaded_pages), 1)
-        self.assertEqual(loaded_pages[0].test_name, 'http://some-test/')
+        self.assertEqual(len(loaded_pages), 2)
+        self.assertEqual(loaded_pages[0].test_name, test.force_gc_test)
+        self.assertEqual(loaded_pages[1].test_name, 'http://some-test/')
         self.assertEqual(actual_stdout, '')
         self.assertEqual(actual_stderr, '')
         self.assertEqual(actual_logs, 'error: some-test.replay\nsome error\n')
@@ -316,15 +330,15 @@ class TestReplayPerfTest(unittest.TestCase):
 
 class TestPerfTestFactory(unittest.TestCase):
     def test_regular_test(self):
-        test = PerfTestFactory.create_perf_test(None, 'some-dir/some-test', '/path/some-dir/some-test')
+        test = PerfTestFactory.create_perf_test(MockPort(), 'some-dir/some-test', '/path/some-dir/some-test')
         self.assertEqual(test.__class__, PerfTest)
 
     def test_inspector_test(self):
-        test = PerfTestFactory.create_perf_test(None, 'inspector/some-test', '/path/inspector/some-test')
+        test = PerfTestFactory.create_perf_test(MockPort(), 'inspector/some-test', '/path/inspector/some-test')
         self.assertEqual(test.__class__, ChromiumStylePerfTest)
 
     def test_page_loading_test(self):
-        test = PerfTestFactory.create_perf_test(None, 'PageLoad/some-test', '/path/PageLoad/some-test')
+        test = PerfTestFactory.create_perf_test(MockPort(), 'PageLoad/some-test', '/path/PageLoad/some-test')
         self.assertEqual(test.__class__, PageLoadingPerfTest)