Force GC between PageLoad tests.
[WebKit-https.git] / Tools / Scripts / webkitpy / performance_tests / perftest_unittest.py
1 #!/usr/bin/python
2 # Copyright (C) 2012 Google Inc. All rights reserved.
3 #
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
6 # met:
7 #
8 #     * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 #     * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
13 # distribution.
14 #     * Neither the name of Google Inc. nor the names of its
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
17 #
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30 import StringIO
31 import math
32 import unittest
33
34 from webkitpy.common.host_mock import MockHost
35 from webkitpy.common.system.outputcapture import OutputCapture
36 from webkitpy.layout_tests.port.driver import DriverOutput
37 from webkitpy.layout_tests.port.test import TestDriver
38 from webkitpy.layout_tests.port.test import TestPort
39 from webkitpy.performance_tests.perftest import ChromiumStylePerfTest
40 from webkitpy.performance_tests.perftest import PageLoadingPerfTest
41 from webkitpy.performance_tests.perftest import PerfTest
42 from webkitpy.performance_tests.perftest import PerfTestFactory
43 from webkitpy.performance_tests.perftest import ReplayPerfTest
44
45
46 class MockPort(TestPort):
47     def __init__(self, custom_run_test=None):
48         super(MockPort, self).__init__(host=MockHost(), custom_run_test=custom_run_test)
49
50 class MainTest(unittest.TestCase):
51     def test_parse_output(self):
52         output = DriverOutput('\n'.join([
53             'Running 20 times',
54             'Ignoring warm-up run (1115)',
55             '',
56             'Time:',
57             'values 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19 ms',
58             'avg 1100 ms',
59             'median 1101 ms',
60             'stdev 11 ms',
61             'min 1080 ms',
62             'max 1120 ms']), image=None, image_hash=None, audio=None)
63         output_capture = OutputCapture()
64         output_capture.capture_output()
65         try:
66             test = PerfTest(None, 'some-test', '/path/some-dir/some-test')
67             self.assertEqual(test.parse_output(output),
68                 {'some-test': {'avg': 1100.0, 'median': 1101.0, 'min': 1080.0, 'max': 1120.0, 'stdev': 11.0, 'unit': 'ms',
69                     'values': [i for i in range(1, 20)]}})
70         finally:
71             pass
72             actual_stdout, actual_stderr, actual_logs = output_capture.restore_output()
73         self.assertEqual(actual_stdout, '')
74         self.assertEqual(actual_stderr, '')
75         self.assertEqual(actual_logs, 'RESULT some-test= 1100.0 ms\nmedian= 1101.0 ms, stdev= 11.0 ms, min= 1080.0 ms, max= 1120.0 ms\n')
76
77     def test_parse_output_with_failing_line(self):
78         output = DriverOutput('\n'.join([
79             'Running 20 times',
80             'Ignoring warm-up run (1115)',
81             '',
82             'some-unrecognizable-line',
83             '',
84             'Time:'
85             'values 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19 ms',
86             'avg 1100 ms',
87             'median 1101 ms',
88             'stdev 11 ms',
89             'min 1080 ms',
90             'max 1120 ms']), image=None, image_hash=None, audio=None)
91         output_capture = OutputCapture()
92         output_capture.capture_output()
93         try:
94             test = PerfTest(None, 'some-test', '/path/some-dir/some-test')
95             self.assertEqual(test.parse_output(output), None)
96         finally:
97             actual_stdout, actual_stderr, actual_logs = output_capture.restore_output()
98         self.assertEqual(actual_stdout, '')
99         self.assertEqual(actual_stderr, '')
100         self.assertEqual(actual_logs, 'some-unrecognizable-line\n')
101
102
103 class TestPageLoadingPerfTest(unittest.TestCase):
104     class MockDriver(object):
105         def __init__(self, values, test):
106             self._values = values
107             self._index = 0
108             self._test = test
109
110         def run_test(self, input, stop_when_done):
111             if input.test_name == self._test.force_gc_test:
112                 return
113             value = self._values[self._index]
114             self._index += 1
115             if isinstance(value, str):
116                 return DriverOutput('some output', image=None, image_hash=None, audio=None, error=value)
117             else:
118                 return DriverOutput('some output', image=None, image_hash=None, audio=None, test_time=self._values[self._index - 1])
119
120     def test_run(self):
121         port = MockPort()
122         test = PageLoadingPerfTest(port, 'some-test', '/path/some-dir/some-test')
123         driver = TestPageLoadingPerfTest.MockDriver(range(1, 21), test)
124         output_capture = OutputCapture()
125         output_capture.capture_output()
126         try:
127             self.assertEqual(test.run(driver, None),
128                 {'some-test': {'max': 20000, 'avg': 11000.0, 'median': 11000, 'stdev': 5627.314338711378, 'min': 2000, 'unit': 'ms',
129                     'values': [i * 1000 for i in range(2, 21)]}})
130         finally:
131             actual_stdout, actual_stderr, actual_logs = output_capture.restore_output()
132         self.assertEqual(actual_stdout, '')
133         self.assertEqual(actual_stderr, '')
134         self.assertEqual(actual_logs, 'RESULT some-test= 11000.0 ms\nmedian= 11000 ms, stdev= 5627.31433871 ms, min= 2000 ms, max= 20000 ms\n')
135
136     def test_run_with_bad_output(self):
137         output_capture = OutputCapture()
138         output_capture.capture_output()
139         try:
140             port = MockPort()
141             test = PageLoadingPerfTest(port, 'some-test', '/path/some-dir/some-test')
142             driver = TestPageLoadingPerfTest.MockDriver([1, 2, 3, 4, 5, 6, 7, 'some error', 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], test)
143             self.assertEqual(test.run(driver, None), None)
144         finally:
145             actual_stdout, actual_stderr, actual_logs = output_capture.restore_output()
146         self.assertEqual(actual_stdout, '')
147         self.assertEqual(actual_stderr, '')
148         self.assertEqual(actual_logs, 'error: some-test\nsome error\n')
149
150
151 class TestReplayPerfTest(unittest.TestCase):
152
153     class ReplayTestPort(MockPort):
154         def __init__(self, custom_run_test=None):
155
156             class ReplayTestDriver(TestDriver):
157                 def run_test(self, text_input, stop_when_done):
158                     return custom_run_test(text_input, stop_when_done) if custom_run_test else None
159
160             self._custom_driver_class = ReplayTestDriver
161             super(self.__class__, self).__init__()
162
163         def _driver_class(self):
164             return self._custom_driver_class
165
166     class MockReplayServer(object):
167         def __init__(self, wait_until_ready=True):
168             self.wait_until_ready = lambda: wait_until_ready
169
170         def stop(self):
171             pass
172
173     def _add_file(self, port, dirname, filename, content=True):
174         port.host.filesystem.maybe_make_directory(dirname)
175         port.host.filesystem.write_binary_file(port.host.filesystem.join(dirname, filename), content)
176
177     def _setup_test(self, run_test=None):
178         test_port = self.ReplayTestPort(run_test)
179         self._add_file(test_port, '/path/some-dir', 'some-test.replay', 'http://some-test/')
180         test = ReplayPerfTest(test_port, 'some-test.replay', '/path/some-dir/some-test.replay')
181         test._start_replay_server = lambda archive, record: self.__class__.MockReplayServer()
182         return test, test_port
183
184     def test_run_single(self):
185         output_capture = OutputCapture()
186         output_capture.capture_output()
187
188         loaded_pages = []
189
190         def run_test(test_input, stop_when_done):
191             if test_input.test_name == test.force_gc_test:
192                 loaded_pages.append(test_input)
193                 return
194             if test_input.test_name != "about:blank":
195                 self.assertEqual(test_input.test_name, 'http://some-test/')
196             loaded_pages.append(test_input)
197             self._add_file(port, '/path/some-dir', 'some-test.wpr', 'wpr content')
198             return DriverOutput('actual text', 'actual image', 'actual checksum',
199                 audio=None, crash=False, timeout=False, error=False)
200
201         test, port = self._setup_test(run_test)
202         test._archive_path = '/path/some-dir/some-test.wpr'
203         test._url = 'http://some-test/'
204
205         try:
206             driver = port.create_driver(worker_number=1, no_timeout=True)
207             self.assertTrue(test.run_single(driver, '/path/some-dir/some-test.replay', time_out_ms=100))
208         finally:
209             actual_stdout, actual_stderr, actual_logs = output_capture.restore_output()
210
211         self.assertEqual(len(loaded_pages), 2)
212         self.assertEqual(loaded_pages[0].test_name, test.force_gc_test)
213         self.assertEqual(loaded_pages[1].test_name, 'http://some-test/')
214         self.assertEqual(actual_stdout, '')
215         self.assertEqual(actual_stderr, '')
216         self.assertEqual(actual_logs, '')
217         self.assertEqual(port.host.filesystem.read_binary_file('/path/some-dir/some-test-actual.png'), 'actual image')
218
219     def test_run_single_fails_without_webpagereplay(self):
220         output_capture = OutputCapture()
221         output_capture.capture_output()
222
223         test, port = self._setup_test()
224         test._start_replay_server = lambda archive, record: None
225         test._archive_path = '/path/some-dir.wpr'
226         test._url = 'http://some-test/'
227
228         try:
229             driver = port.create_driver(worker_number=1, no_timeout=True)
230             self.assertEqual(test.run_single(driver, '/path/some-dir/some-test.replay', time_out_ms=100), None)
231         finally:
232             actual_stdout, actual_stderr, actual_logs = output_capture.restore_output()
233         self.assertEqual(actual_stdout, '')
234         self.assertEqual(actual_stderr, '')
235         self.assertEqual(actual_logs, "Web page replay didn't start.\n")
236
237     def test_prepare_fails_when_wait_until_ready_fails(self):
238         output_capture = OutputCapture()
239         output_capture.capture_output()
240
241         test, port = self._setup_test()
242         test._start_replay_server = lambda archive, record: self.__class__.MockReplayServer(wait_until_ready=False)
243         test._archive_path = '/path/some-dir.wpr'
244         test._url = 'http://some-test/'
245
246         try:
247             driver = port.create_driver(worker_number=1, no_timeout=True)
248             self.assertEqual(test.run_single(driver, '/path/some-dir/some-test.replay', time_out_ms=100), None)
249         finally:
250             actual_stdout, actual_stderr, actual_logs = output_capture.restore_output()
251
252         self.assertEqual(actual_stdout, '')
253         self.assertEqual(actual_stderr, '')
254         self.assertEqual(actual_logs, "Web page replay didn't start.\n")
255
256     def test_run_single_fails_when_output_has_error(self):
257         output_capture = OutputCapture()
258         output_capture.capture_output()
259
260         loaded_pages = []
261
262         def run_test(test_input, stop_when_done):
263             loaded_pages.append(test_input)
264             self._add_file(port, '/path/some-dir', 'some-test.wpr', 'wpr content')
265             return DriverOutput('actual text', 'actual image', 'actual checksum',
266                 audio=None, crash=False, timeout=False, error='some error')
267
268         test, port = self._setup_test(run_test)
269         test._archive_path = '/path/some-dir.wpr'
270         test._url = 'http://some-test/'
271
272         try:
273             driver = port.create_driver(worker_number=1, no_timeout=True)
274             self.assertEqual(test.run_single(driver, '/path/some-dir/some-test.replay', time_out_ms=100), None)
275         finally:
276             actual_stdout, actual_stderr, actual_logs = output_capture.restore_output()
277
278         self.assertEqual(len(loaded_pages), 2)
279         self.assertEqual(loaded_pages[0].test_name, test.force_gc_test)
280         self.assertEqual(loaded_pages[1].test_name, 'http://some-test/')
281         self.assertEqual(actual_stdout, '')
282         self.assertEqual(actual_stderr, '')
283         self.assertEqual(actual_logs, 'error: some-test.replay\nsome error\n')
284
285     def test_prepare(self):
286         output_capture = OutputCapture()
287         output_capture.capture_output()
288
289         def run_test(test_input, stop_when_done):
290             self._add_file(port, '/path/some-dir', 'some-test.wpr', 'wpr content')
291             return DriverOutput('actual text', 'actual image', 'actual checksum',
292                 audio=None, crash=False, timeout=False, error=False)
293
294         test, port = self._setup_test(run_test)
295
296         try:
297             self.assertEqual(test.prepare(time_out_ms=100), True)
298         finally:
299             actual_stdout, actual_stderr, actual_logs = output_capture.restore_output()
300
301         self.assertEqual(actual_stdout, '')
302         self.assertEqual(actual_stderr, '')
303         self.assertEqual(actual_logs, 'Preparing replay for some-test.replay\nPrepared replay for some-test.replay\n')
304         self.assertEqual(port.host.filesystem.read_binary_file('/path/some-dir/some-test-expected.png'), 'actual image')
305
306     def test_prepare_calls_run_single(self):
307         output_capture = OutputCapture()
308         output_capture.capture_output()
309         called = [False]
310
311         def run_single(driver, url, time_out_ms, record):
312             self.assertTrue(record)
313             self.assertEqual(url, '/path/some-dir/some-test.wpr')
314             called[0] = True
315             return False
316
317         test, port = self._setup_test()
318         test.run_single = run_single
319
320         try:
321             self.assertEqual(test.prepare(time_out_ms=100), False)
322         finally:
323             actual_stdout, actual_stderr, actual_logs = output_capture.restore_output()
324         self.assertTrue(called[0])
325         self.assertEqual(test._archive_path, '/path/some-dir/some-test.wpr')
326         self.assertEqual(test._url, 'http://some-test/')
327         self.assertEqual(actual_stdout, '')
328         self.assertEqual(actual_stderr, '')
329         self.assertEqual(actual_logs, "Preparing replay for some-test.replay\nFailed to prepare a replay for some-test.replay\n")
330
331 class TestPerfTestFactory(unittest.TestCase):
332     def test_regular_test(self):
333         test = PerfTestFactory.create_perf_test(MockPort(), 'some-dir/some-test', '/path/some-dir/some-test')
334         self.assertEqual(test.__class__, PerfTest)
335
336     def test_inspector_test(self):
337         test = PerfTestFactory.create_perf_test(MockPort(), 'inspector/some-test', '/path/inspector/some-test')
338         self.assertEqual(test.__class__, ChromiumStylePerfTest)
339
340     def test_page_loading_test(self):
341         test = PerfTestFactory.create_perf_test(MockPort(), 'PageLoad/some-test', '/path/PageLoad/some-test')
342         self.assertEqual(test.__class__, PageLoadingPerfTest)
343
344
345 if __name__ == '__main__':
346     unittest.main()