Add public page loading performance tests using web-page-replay
[WebKit-https.git] / Tools / Scripts / webkitpy / performance_tests / perftest_unittest.py
1 #!/usr/bin/python
2 # Copyright (C) 2012 Google Inc. All rights reserved.
3 #
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
6 # met:
7 #
8 #     * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 #     * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
13 # distribution.
14 #     * Neither the name of Google Inc. nor the names of its
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
17 #
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30 import StringIO
31 import math
32 import unittest
33
34 from webkitpy.common.host_mock import MockHost
35 from webkitpy.common.system.outputcapture import OutputCapture
36 from webkitpy.layout_tests.port.driver import DriverOutput
37 from webkitpy.layout_tests.port.test import TestDriver
38 from webkitpy.layout_tests.port.test import TestPort
39 from webkitpy.performance_tests.perftest import ChromiumStylePerfTest
40 from webkitpy.performance_tests.perftest import PageLoadingPerfTest
41 from webkitpy.performance_tests.perftest import PerfTest
42 from webkitpy.performance_tests.perftest import PerfTestFactory
43 from webkitpy.performance_tests.perftest import ReplayPerfTest
44
45
46 class MainTest(unittest.TestCase):
47     def test_parse_output(self):
48         output = DriverOutput('\n'.join([
49             'Running 20 times',
50             'Ignoring warm-up run (1115)',
51             '',
52             'avg 1100',
53             'median 1101',
54             'stdev 11',
55             'min 1080',
56             'max 1120']), image=None, image_hash=None, audio=None)
57         output_capture = OutputCapture()
58         output_capture.capture_output()
59         try:
60             test = PerfTest(None, 'some-test', '/path/some-dir/some-test')
61             self.assertEqual(test.parse_output(output),
62                 {'some-test': {'avg': 1100.0, 'median': 1101.0, 'min': 1080.0, 'max': 1120.0, 'stdev': 11.0, 'unit': 'ms'}})
63         finally:
64             actual_stdout, actual_stderr, actual_logs = output_capture.restore_output()
65         self.assertEqual(actual_stdout, '')
66         self.assertEqual(actual_stderr, '')
67         self.assertEqual(actual_logs, 'RESULT some-test= 1100.0 ms\nmedian= 1101.0 ms, stdev= 11.0 ms, min= 1080.0 ms, max= 1120.0 ms\n')
68
69     def test_parse_output_with_failing_line(self):
70         output = DriverOutput('\n'.join([
71             'Running 20 times',
72             'Ignoring warm-up run (1115)',
73             '',
74             'some-unrecognizable-line',
75             '',
76             'avg 1100',
77             'median 1101',
78             'stdev 11',
79             'min 1080',
80             'max 1120']), image=None, image_hash=None, audio=None)
81         output_capture = OutputCapture()
82         output_capture.capture_output()
83         try:
84             test = PerfTest(None, 'some-test', '/path/some-dir/some-test')
85             self.assertEqual(test.parse_output(output), None)
86         finally:
87             actual_stdout, actual_stderr, actual_logs = output_capture.restore_output()
88         self.assertEqual(actual_stdout, '')
89         self.assertEqual(actual_stderr, '')
90         self.assertEqual(actual_logs, 'some-unrecognizable-line\n')
91
92
93 class TestPageLoadingPerfTest(unittest.TestCase):
94     class MockDriver(object):
95         def __init__(self, values):
96             self._values = values
97             self._index = 0
98
99         def run_test(self, input):
100             value = self._values[self._index]
101             self._index += 1
102             if isinstance(value, str):
103                 return DriverOutput('some output', image=None, image_hash=None, audio=None, error=value)
104             else:
105                 return DriverOutput('some output', image=None, image_hash=None, audio=None, test_time=self._values[self._index - 1])
106
107     def test_run(self):
108         test = PageLoadingPerfTest(None, 'some-test', '/path/some-dir/some-test')
109         driver = TestPageLoadingPerfTest.MockDriver([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20])
110         output_capture = OutputCapture()
111         output_capture.capture_output()
112         try:
113             self.assertEqual(test.run(driver, None),
114                 {'some-test': {'max': 20000, 'avg': 11000.0, 'median': 11000, 'stdev': math.sqrt(570 * 1000 * 1000), 'min': 2000, 'unit': 'ms'}})
115         finally:
116             actual_stdout, actual_stderr, actual_logs = output_capture.restore_output()
117         self.assertEqual(actual_stdout, '')
118         self.assertEqual(actual_stderr, '')
119         self.assertEqual(actual_logs, 'RESULT some-test= 11000.0 ms\nmedian= 11000 ms, stdev= 23874.6727726 ms, min= 2000 ms, max= 20000 ms\n')
120
121     def test_run_with_bad_output(self):
122         output_capture = OutputCapture()
123         output_capture.capture_output()
124         try:
125             test = PageLoadingPerfTest(None, 'some-test', '/path/some-dir/some-test')
126             driver = TestPageLoadingPerfTest.MockDriver([1, 2, 3, 4, 5, 6, 7, 'some error', 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20])
127             self.assertEqual(test.run(driver, None), None)
128         finally:
129             actual_stdout, actual_stderr, actual_logs = output_capture.restore_output()
130         self.assertEqual(actual_stdout, '')
131         self.assertEqual(actual_stderr, '')
132         self.assertEqual(actual_logs, 'error: some-test\nsome error\n')
133
134
135 class TestReplayPerfTest(unittest.TestCase):
136
137     class ReplayTestPort(TestPort):
138         def __init__(self, custom_run_test=None):
139
140             class ReplayTestDriver(TestDriver):
141                 def run_test(self, text_input):
142                     return custom_run_test(text_input) if custom_run_test else None
143
144             self._custom_driver_class = ReplayTestDriver
145             super(self.__class__, self).__init__(host=MockHost())
146
147         def _driver_class(self):
148             return self._custom_driver_class
149
150     class MockReplayServer(object):
151         def __init__(self, wait_until_ready=True):
152             self.wait_until_ready = lambda: wait_until_ready
153
154         def stop(self):
155             pass
156
157     def _add_file(self, port, dirname, filename, content=True):
158         port.host.filesystem.maybe_make_directory(dirname)
159         port.host.filesystem.files[port.host.filesystem.join(dirname, filename)] = content
160
161     def _setup_test(self, run_test=None):
162         test_port = self.ReplayTestPort(run_test)
163         self._add_file(test_port, '/path/some-dir', 'some-test.replay', 'http://some-test/')
164         test = ReplayPerfTest(test_port, 'some-test.replay', '/path/some-dir/some-test.replay')
165         test._start_replay_server = lambda archive, record: self.__class__.MockReplayServer()
166         return test, test_port
167
168     def test_run_single(self):
169         output_capture = OutputCapture()
170         output_capture.capture_output()
171
172         loaded_pages = []
173
174         def run_test(test_input):
175             if test_input.test_name != "about:blank":
176                 self.assertEqual(test_input.test_name, 'http://some-test/')
177             loaded_pages.append(test_input)
178             self._add_file(port, '/path/some-dir', 'some-test.wpr', 'wpr content')
179             return DriverOutput('actual text', 'actual image', 'actual checksum',
180                 audio=None, crash=False, timeout=False, error=False)
181
182         test, port = self._setup_test(run_test)
183         test._archive_path = '/path/some-dir/some-test.wpr'
184         test._url = 'http://some-test/'
185
186         try:
187             driver = port.create_driver(worker_number=1, no_timeout=True)
188             self.assertTrue(test.run_single(driver, '/path/some-dir/some-test.replay', time_out_ms=100))
189         finally:
190             actual_stdout, actual_stderr, actual_logs = output_capture.restore_output()
191
192         self.assertEqual(len(loaded_pages), 2)
193         self.assertEqual(loaded_pages[0].test_name, 'about:blank')
194         self.assertEqual(loaded_pages[1].test_name, 'http://some-test/')
195         self.assertEqual(actual_stdout, '')
196         self.assertEqual(actual_stderr, '')
197         self.assertEqual(actual_logs, '')
198
199     def test_run_single_fails_without_webpagereplay(self):
200         output_capture = OutputCapture()
201         output_capture.capture_output()
202
203         test, port = self._setup_test()
204         test._start_replay_server = lambda archive, record: None
205         test._archive_path = '/path/some-dir.wpr'
206         test._url = 'http://some-test/'
207
208         try:
209             driver = port.create_driver(worker_number=1, no_timeout=True)
210             self.assertEqual(test.run_single(driver, '/path/some-dir/some-test.replay', time_out_ms=100), None)
211         finally:
212             actual_stdout, actual_stderr, actual_logs = output_capture.restore_output()
213         self.assertEqual(actual_stdout, '')
214         self.assertEqual(actual_stderr, '')
215         self.assertEqual(actual_logs, "Web page replay didn't start.\n")
216
217     def test_prepare_fails_when_wait_until_ready_fails(self):
218         output_capture = OutputCapture()
219         output_capture.capture_output()
220
221         test, port = self._setup_test()
222         test._start_replay_server = lambda archive, record: self.__class__.MockReplayServer(wait_until_ready=False)
223         test._archive_path = '/path/some-dir.wpr'
224         test._url = 'http://some-test/'
225
226         try:
227             driver = port.create_driver(worker_number=1, no_timeout=True)
228             self.assertEqual(test.run_single(driver, '/path/some-dir/some-test.replay', time_out_ms=100), None)
229         finally:
230             actual_stdout, actual_stderr, actual_logs = output_capture.restore_output()
231
232         self.assertEqual(actual_stdout, '')
233         self.assertEqual(actual_stderr, '')
234         self.assertEqual(actual_logs, "Web page replay didn't start.\n")
235
236     def test_run_single_fails_when_output_has_error(self):
237         output_capture = OutputCapture()
238         output_capture.capture_output()
239
240         loaded_pages = []
241
242         def run_test(test_input):
243             loaded_pages.append(test_input)
244             self._add_file(port, '/path/some-dir', 'some-test.wpr', 'wpr content')
245             return DriverOutput('actual text', 'actual image', 'actual checksum',
246                 audio=None, crash=False, timeout=False, error='some error')
247
248         test, port = self._setup_test(run_test)
249         test._archive_path = '/path/some-dir.wpr'
250         test._url = 'http://some-test/'
251
252         try:
253             driver = port.create_driver(worker_number=1, no_timeout=True)
254             self.assertEqual(test.run_single(driver, '/path/some-dir/some-test.replay', time_out_ms=100), None)
255         finally:
256             actual_stdout, actual_stderr, actual_logs = output_capture.restore_output()
257
258         self.assertEqual(len(loaded_pages), 2)
259         self.assertEqual(loaded_pages[0].test_name, 'about:blank')
260         self.assertEqual(loaded_pages[1].test_name, 'http://some-test/')
261         self.assertEqual(actual_stdout, '')
262         self.assertEqual(actual_stderr, '')
263         self.assertEqual(actual_logs, 'error: some-test.replay\nsome error\n')
264
265     def test_prepare(self):
266         output_capture = OutputCapture()
267         output_capture.capture_output()
268
269         def run_test(test_input):
270             self._add_file(port, '/path/some-dir', 'some-test.wpr', 'wpr content')
271             return DriverOutput('actual text', 'actual image', 'actual checksum',
272                 audio=None, crash=False, timeout=False, error=False)
273
274         test, port = self._setup_test(run_test)
275
276         try:
277             self.assertEqual(test.prepare(time_out_ms=100), True)
278         finally:
279             actual_stdout, actual_stderr, actual_logs = output_capture.restore_output()
280
281         self.assertEqual(actual_stdout, '')
282         self.assertEqual(actual_stderr, '')
283         self.assertEqual(actual_logs, 'Preparing replay for some-test.replay\nPrepared replay for some-test.replay\n')
284
285     def test_prepare_calls_run_single(self):
286         output_capture = OutputCapture()
287         output_capture.capture_output()
288         called = [False]
289
290         def run_single(driver, url, time_out_ms, record):
291             self.assertTrue(record)
292             self.assertEqual(url, 'http://some-test/')
293             called[0] = True
294             return False
295
296         test, port = self._setup_test()
297         test.run_single = run_single
298
299         try:
300             self.assertEqual(test.prepare(time_out_ms=100), False)
301         finally:
302             actual_stdout, actual_stderr, actual_logs = output_capture.restore_output()
303         self.assertTrue(called[0])
304         self.assertEqual(test._archive_path, '/path/some-dir/some-test.wpr')
305         self.assertEqual(test._url, 'http://some-test/')
306         self.assertEqual(actual_stdout, '')
307         self.assertEqual(actual_stderr, '')
308         self.assertEqual(actual_logs, "Preparing replay for some-test.replay\nFailed to prepare a replay for some-test.replay\n")
309
310 class TestPerfTestFactory(unittest.TestCase):
311     def test_regular_test(self):
312         test = PerfTestFactory.create_perf_test(None, 'some-dir/some-test', '/path/some-dir/some-test')
313         self.assertEqual(test.__class__, PerfTest)
314
315     def test_inspector_test(self):
316         test = PerfTestFactory.create_perf_test(None, 'inspector/some-test', '/path/inspector/some-test')
317         self.assertEqual(test.__class__, ChromiumStylePerfTest)
318
319     def test_page_loading_test(self):
320         test = PerfTestFactory.create_perf_test(None, 'PageLoad/some-test', '/path/PageLoad/some-test')
321         self.assertEqual(test.__class__, PageLoadingPerfTest)
322
323
324 if __name__ == '__main__':
325     unittest.main()