c5775945da5a78f8676203419bc1b52404b4a53a
[WebKit-https.git] / Tools / Scripts / webkitpy / performance_tests / perftest_unittest.py
1 # Copyright (C) 2012 Google Inc. All rights reserved.
2 #
3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions are
5 # met:
6 #
7 #     * Redistributions of source code must retain the above copyright
8 # notice, this list of conditions and the following disclaimer.
9 #     * Redistributions in binary form must reproduce the above
10 # copyright notice, this list of conditions and the following disclaimer
11 # in the documentation and/or other materials provided with the
12 # distribution.
13 #     * Neither the name of Google Inc. nor the names of its
14 # contributors may be used to endorse or promote products derived from
15 # this software without specific prior written permission.
16 #
17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29 import StringIO
30 import json
31 import math
32 import unittest
33
34 from webkitpy.common.host_mock import MockHost
35 from webkitpy.common.system.outputcapture import OutputCapture
36 from webkitpy.layout_tests.port.driver import DriverOutput
37 from webkitpy.layout_tests.port.test import TestDriver
38 from webkitpy.layout_tests.port.test import TestPort
39 from webkitpy.performance_tests.perftest import ChromiumStylePerfTest
40 from webkitpy.performance_tests.perftest import PageLoadingPerfTest
41 from webkitpy.performance_tests.perftest import PerfTest
42 from webkitpy.performance_tests.perftest import PerfTestFactory
43 from webkitpy.performance_tests.perftest import ReplayPerfTest
44
45
46 class MockPort(TestPort):
47     def __init__(self, custom_run_test=None):
48         super(MockPort, self).__init__(host=MockHost(), custom_run_test=custom_run_test)
49
50 class MainTest(unittest.TestCase):
51     def test_compute_statistics(self):
52         def compute_statistics(values):
53             statistics = PerfTest.compute_statistics(map(lambda x: float(x), values))
54             return json.loads(json.dumps(statistics))
55
56         statistics = compute_statistics([10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 20, 19, 18, 17, 16, 15, 14, 13, 12, 11])
57         self.assertEqual(sorted(statistics.keys()), ['avg', 'max', 'median', 'min', 'stdev'])
58         self.assertEqual(statistics['avg'], 10.5)
59         self.assertEqual(statistics['min'], 1)
60         self.assertEqual(statistics['max'], 20)
61         self.assertEqual(statistics['median'], 10.5)
62         self.assertEqual(compute_statistics([8, 9, 10, 11, 12])['avg'], 10)
63         self.assertEqual(compute_statistics([8, 9, 10, 11, 12] * 4)['avg'], 10)
64         self.assertEqual(compute_statistics([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19])['avg'], 10)
65         self.assertEqual(PerfTest.compute_statistics([1, 5, 2, 8, 7])['median'], 5)
66         self.assertEqual(PerfTest.compute_statistics([1, 6, 2, 8, 7, 2])['median'], 4)
67         self.assertAlmostEqual(statistics['stdev'], math.sqrt(35))
68         self.assertAlmostEqual(compute_statistics([1, 2, 3, 4, 5, 6])['stdev'], math.sqrt(3.5))
69         self.assertAlmostEqual(compute_statistics([4, 2, 5, 8, 6])['stdev'], math.sqrt(5))
70
71     def _assert_results_are_correct(self, test, output):
72         test._filter_output(output)
73         parsed_results = test.parse_output(output)
74         self.assertEqual(parsed_results.keys(), ['some-test'])
75         some_test_results = parsed_results['some-test']
76         self.assertEqual(sorted(some_test_results.keys()), ['avg', 'max', 'median', 'min', 'stdev', 'unit', 'values'])
77         self.assertEqual(some_test_results['values'], [1080, 1120, 1095, 1101, 1104])
78         self.assertEqual(some_test_results['min'], 1080)
79         self.assertEqual(some_test_results['max'], 1120)
80         self.assertEqual(some_test_results['avg'], 1100)
81         self.assertEqual(some_test_results['median'], 1101)
82         self.assertAlmostEqual(some_test_results['stdev'], 14.50862, places=5)
83         self.assertEqual(some_test_results['unit'], 'ms')
84
85     def test_parse_output(self):
86         output = DriverOutput("""
87 Running 20 times
88 Ignoring warm-up run (1115)
89
90 Time:
91 values 1080, 1120, 1095, 1101, 1104 ms
92 avg 1100 ms
93 median 1101 ms
94 stdev 14.50862 ms
95 min 1080 ms
96 max 1120 ms
97 """, image=None, image_hash=None, audio=None)
98         output_capture = OutputCapture()
99         output_capture.capture_output()
100         try:
101             test = PerfTest(MockPort(), 'some-test', '/path/some-dir/some-test')
102             self._assert_results_are_correct(test, output)
103         finally:
104             actual_stdout, actual_stderr, actual_logs = output_capture.restore_output()
105         self.assertEqual(actual_stdout, '')
106         self.assertEqual(actual_stderr, '')
107         self.assertEqual(actual_logs, '')
108
109     def test_parse_output_with_failing_line(self):
110         output = DriverOutput("""
111 Running 20 times
112 Ignoring warm-up run (1115)
113
114 some-unrecognizable-line
115
116 Time:
117 values 1080, 1120, 1095, 1101, 1104 ms
118 avg 1100 ms
119 median 1101 ms
120 stdev 14.50862 ms
121 min 1080 ms
122 max 1120 ms
123 """, image=None, image_hash=None, audio=None)
124         output_capture = OutputCapture()
125         output_capture.capture_output()
126         try:
127             test = PerfTest(MockPort(), 'some-test', '/path/some-dir/some-test')
128             test._filter_output(output)
129             self.assertEqual(test.parse_output(output), None)
130         finally:
131             actual_stdout, actual_stderr, actual_logs = output_capture.restore_output()
132         self.assertEqual(actual_stdout, '')
133         self.assertEqual(actual_stderr, '')
134         self.assertEqual(actual_logs, 'ERROR: some-unrecognizable-line\n')
135
136     def test_parse_output_with_description(self):
137         output = DriverOutput("""
138 Description: this is a test description.
139
140 Running 20 times
141 Ignoring warm-up run (1115)
142
143 Time:
144 values 1080, 1120, 1095, 1101, 1104 ms
145 avg 1100 ms
146 median 1101 ms
147 stdev 14.50862 ms
148 min 1080 ms
149 max 1120 ms""", image=None, image_hash=None, audio=None)
150         test = PerfTest(MockPort(), 'some-test', '/path/some-dir/some-test')
151         self._assert_results_are_correct(test, output)
152         self.assertEqual(test.description(), 'this is a test description.')
153
154     def test_ignored_stderr_lines(self):
155         test = PerfTest(MockPort(), 'some-test', '/path/some-dir/some-test')
156         ignored_lines = [
157             "Unknown option: --foo-bar",
158             "[WARNING:proxy_service.cc] bad moon a-rising",
159             "[INFO:SkFontHost_android.cpp(1158)] Use Test Config File Main /data/local/tmp/drt/android_main_fonts.xml, Fallback /data/local/tmp/drt/android_fallback_fonts.xml, Font Dir /data/local/tmp/drt/fonts/",
160         ]
161         for line in ignored_lines:
162             self.assertTrue(test._should_ignore_line_in_stderr(line))
163
164         non_ignored_lines = [
165             "Should not be ignored",
166             "[WARNING:chrome.cc] Something went wrong",
167             "[ERROR:main.cc] The sky has fallen",
168         ]
169         for line in non_ignored_lines:
170             self.assertFalse(test._should_ignore_line_in_stderr(line))
171
172     def test_parse_output_with_subtests(self):
173         output = DriverOutput("""
174 Running 20 times
175 some test: [1, 2, 3, 4, 5]
176 other test = else: [6, 7, 8, 9, 10]
177 Ignoring warm-up run (1115)
178
179 Time:
180 values 1080, 1120, 1095, 1101, 1104 ms
181 avg 1100 ms
182 median 1101 ms
183 stdev 14.50862 ms
184 min 1080 ms
185 max 1120 ms
186 """, image=None, image_hash=None, audio=None)
187         output_capture = OutputCapture()
188         output_capture.capture_output()
189         try:
190             test = PerfTest(MockPort(), 'some-test', '/path/some-dir/some-test')
191             self._assert_results_are_correct(test, output)
192         finally:
193             actual_stdout, actual_stderr, actual_logs = output_capture.restore_output()
194         self.assertEqual(actual_stdout, '')
195         self.assertEqual(actual_stderr, '')
196         self.assertEqual(actual_logs, '')
197
198
199 class TestPageLoadingPerfTest(unittest.TestCase):
200     class MockDriver(object):
201         def __init__(self, values, test, measurements=None):
202             self._values = values
203             self._index = 0
204             self._test = test
205             self._measurements = measurements
206
207         def run_test(self, input, stop_when_done):
208             if input.test_name == self._test.force_gc_test:
209                 return
210             value = self._values[self._index]
211             self._index += 1
212             if isinstance(value, str):
213                 return DriverOutput('some output', image=None, image_hash=None, audio=None, error=value)
214             else:
215                 return DriverOutput('some output', image=None, image_hash=None, audio=None, test_time=self._values[self._index - 1], measurements=self._measurements)
216
217     def test_run(self):
218         port = MockPort()
219         test = PageLoadingPerfTest(port, 'some-test', '/path/some-dir/some-test')
220         driver = TestPageLoadingPerfTest.MockDriver(range(1, 21), test)
221         output_capture = OutputCapture()
222         output_capture.capture_output()
223         try:
224             self.assertEqual(test._run_with_driver(driver, None),
225                 {'some-test': {'max': 20000, 'avg': 11000.0, 'median': 11000, 'stdev': 5627.314338711378, 'min': 2000, 'unit': 'ms',
226                     'values': [float(i * 1000) for i in range(2, 21)]}})
227         finally:
228             actual_stdout, actual_stderr, actual_logs = output_capture.restore_output()
229         self.assertEqual(actual_stdout, '')
230         self.assertEqual(actual_stderr, '')
231         self.assertEqual(actual_logs, 'RESULT some-test= 11000 ms\nmedian= 11000 ms, stdev= 5627.31433871 ms, min= 2000 ms, max= 20000 ms\n')
232
233     def test_run_with_memory_output(self):
234         port = MockPort()
235         test = PageLoadingPerfTest(port, 'some-test', '/path/some-dir/some-test')
236         memory_results = {'Malloc': 10, 'JSHeap': 5}
237         self.maxDiff = None
238         driver = TestPageLoadingPerfTest.MockDriver(range(1, 21), test, memory_results)
239         output_capture = OutputCapture()
240         output_capture.capture_output()
241         try:
242             self.assertEqual(test._run_with_driver(driver, None),
243                 {'some-test': {'max': 20000, 'avg': 11000.0, 'median': 11000, 'stdev': 5627.314338711378, 'min': 2000, 'unit': 'ms',
244                     'values': [float(i * 1000) for i in range(2, 21)]},
245                  'some-test:Malloc': {'max': 10, 'avg': 10.0, 'median': 10, 'min': 10, 'stdev': 0.0, 'unit': 'bytes',
246                     'values': [float(10)] * 19},
247                  'some-test:JSHeap': {'max': 5, 'avg': 5.0, 'median': 5, 'min': 5, 'stdev': 0.0, 'unit': 'bytes',
248                     'values': [float(5)] * 19}})
249         finally:
250             actual_stdout, actual_stderr, actual_logs = output_capture.restore_output()
251         self.assertEqual(actual_stdout, '')
252         self.assertEqual(actual_stderr, '')
253         self.assertEqual(actual_logs, 'RESULT some-test= 11000 ms\nmedian= 11000 ms, stdev= 5627.31433871 ms, min= 2000 ms, max= 20000 ms\n'
254             + 'RESULT some-test: Malloc= 10 bytes\nmedian= 10 bytes, stdev= 0.0 bytes, min= 10 bytes, max= 10 bytes\n'
255             + 'RESULT some-test: JSHeap= 5 bytes\nmedian= 5 bytes, stdev= 0.0 bytes, min= 5 bytes, max= 5 bytes\n')
256
257     def test_run_with_bad_output(self):
258         output_capture = OutputCapture()
259         output_capture.capture_output()
260         try:
261             port = MockPort()
262             test = PageLoadingPerfTest(port, 'some-test', '/path/some-dir/some-test')
263             driver = TestPageLoadingPerfTest.MockDriver([1, 2, 3, 4, 5, 6, 7, 'some error', 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], test)
264             self.assertEqual(test._run_with_driver(driver, None), None)
265         finally:
266             actual_stdout, actual_stderr, actual_logs = output_capture.restore_output()
267         self.assertEqual(actual_stdout, '')
268         self.assertEqual(actual_stderr, '')
269         self.assertEqual(actual_logs, 'error: some-test\nsome error\n')
270
271
272 class TestReplayPerfTest(unittest.TestCase):
273
274     class ReplayTestPort(MockPort):
275         def __init__(self, custom_run_test=None):
276
277             class ReplayTestDriver(TestDriver):
278                 def run_test(self, text_input, stop_when_done):
279                     return custom_run_test(text_input, stop_when_done) if custom_run_test else None
280
281             self._custom_driver_class = ReplayTestDriver
282             super(self.__class__, self).__init__()
283
284         def _driver_class(self):
285             return self._custom_driver_class
286
287     class MockReplayServer(object):
288         def __init__(self, wait_until_ready=True):
289             self.wait_until_ready = lambda: wait_until_ready
290
291         def stop(self):
292             pass
293
294     def _add_file(self, port, dirname, filename, content=True):
295         port.host.filesystem.maybe_make_directory(dirname)
296         port.host.filesystem.write_binary_file(port.host.filesystem.join(dirname, filename), content)
297
298     def _setup_test(self, run_test=None):
299         test_port = self.ReplayTestPort(run_test)
300         self._add_file(test_port, '/path/some-dir', 'some-test.replay', 'http://some-test/')
301         test = ReplayPerfTest(test_port, 'some-test.replay', '/path/some-dir/some-test.replay')
302         test._start_replay_server = lambda archive, record: self.__class__.MockReplayServer()
303         return test, test_port
304
305     def test_run_single(self):
306         output_capture = OutputCapture()
307         output_capture.capture_output()
308
309         loaded_pages = []
310
311         def run_test(test_input, stop_when_done):
312             if test_input.test_name == test.force_gc_test:
313                 loaded_pages.append(test_input)
314                 return
315             if test_input.test_name != "about:blank":
316                 self.assertEqual(test_input.test_name, 'http://some-test/')
317             loaded_pages.append(test_input)
318             self._add_file(port, '/path/some-dir', 'some-test.wpr', 'wpr content')
319             return DriverOutput('actual text', 'actual image', 'actual checksum',
320                 audio=None, crash=False, timeout=False, error=False)
321
322         test, port = self._setup_test(run_test)
323         test._archive_path = '/path/some-dir/some-test.wpr'
324         test._url = 'http://some-test/'
325
326         try:
327             driver = port.create_driver(worker_number=1, no_timeout=True)
328             self.assertTrue(test.run_single(driver, '/path/some-dir/some-test.replay', time_out_ms=100))
329         finally:
330             actual_stdout, actual_stderr, actual_logs = output_capture.restore_output()
331
332         self.assertEqual(len(loaded_pages), 2)
333         self.assertEqual(loaded_pages[0].test_name, test.force_gc_test)
334         self.assertEqual(loaded_pages[1].test_name, 'http://some-test/')
335         self.assertEqual(actual_stdout, '')
336         self.assertEqual(actual_stderr, '')
337         self.assertEqual(actual_logs, '')
338         self.assertEqual(port.host.filesystem.read_binary_file('/path/some-dir/some-test-actual.png'), 'actual image')
339
340     def test_run_single_fails_without_webpagereplay(self):
341         output_capture = OutputCapture()
342         output_capture.capture_output()
343
344         test, port = self._setup_test()
345         test._start_replay_server = lambda archive, record: None
346         test._archive_path = '/path/some-dir.wpr'
347         test._url = 'http://some-test/'
348
349         try:
350             driver = port.create_driver(worker_number=1, no_timeout=True)
351             self.assertEqual(test.run_single(driver, '/path/some-dir/some-test.replay', time_out_ms=100), None)
352         finally:
353             actual_stdout, actual_stderr, actual_logs = output_capture.restore_output()
354         self.assertEqual(actual_stdout, '')
355         self.assertEqual(actual_stderr, '')
356         self.assertEqual(actual_logs, "Web page replay didn't start.\n")
357
358     def test_prepare_fails_when_wait_until_ready_fails(self):
359         output_capture = OutputCapture()
360         output_capture.capture_output()
361
362         test, port = self._setup_test()
363         test._start_replay_server = lambda archive, record: self.__class__.MockReplayServer(wait_until_ready=False)
364         test._archive_path = '/path/some-dir.wpr'
365         test._url = 'http://some-test/'
366
367         try:
368             driver = port.create_driver(worker_number=1, no_timeout=True)
369             self.assertEqual(test.run_single(driver, '/path/some-dir/some-test.replay', time_out_ms=100), None)
370         finally:
371             actual_stdout, actual_stderr, actual_logs = output_capture.restore_output()
372
373         self.assertEqual(actual_stdout, '')
374         self.assertEqual(actual_stderr, '')
375         self.assertEqual(actual_logs, "Web page replay didn't start.\n")
376
377     def test_run_single_fails_when_output_has_error(self):
378         output_capture = OutputCapture()
379         output_capture.capture_output()
380
381         loaded_pages = []
382
383         def run_test(test_input, stop_when_done):
384             loaded_pages.append(test_input)
385             self._add_file(port, '/path/some-dir', 'some-test.wpr', 'wpr content')
386             return DriverOutput('actual text', 'actual image', 'actual checksum',
387                 audio=None, crash=False, timeout=False, error='some error')
388
389         test, port = self._setup_test(run_test)
390         test._archive_path = '/path/some-dir.wpr'
391         test._url = 'http://some-test/'
392
393         try:
394             driver = port.create_driver(worker_number=1, no_timeout=True)
395             self.assertEqual(test.run_single(driver, '/path/some-dir/some-test.replay', time_out_ms=100), None)
396         finally:
397             actual_stdout, actual_stderr, actual_logs = output_capture.restore_output()
398
399         self.assertEqual(len(loaded_pages), 2)
400         self.assertEqual(loaded_pages[0].test_name, test.force_gc_test)
401         self.assertEqual(loaded_pages[1].test_name, 'http://some-test/')
402         self.assertEqual(actual_stdout, '')
403         self.assertEqual(actual_stderr, '')
404         self.assertEqual(actual_logs, 'error: some-test.replay\nsome error\n')
405
406     def test_prepare(self):
407         output_capture = OutputCapture()
408         output_capture.capture_output()
409
410         def run_test(test_input, stop_when_done):
411             self._add_file(port, '/path/some-dir', 'some-test.wpr', 'wpr content')
412             return DriverOutput('actual text', 'actual image', 'actual checksum',
413                 audio=None, crash=False, timeout=False, error=False)
414
415         test, port = self._setup_test(run_test)
416
417         try:
418             self.assertEqual(test.prepare(time_out_ms=100), True)
419         finally:
420             actual_stdout, actual_stderr, actual_logs = output_capture.restore_output()
421
422         self.assertEqual(actual_stdout, '')
423         self.assertEqual(actual_stderr, '')
424         self.assertEqual(actual_logs, 'Preparing replay for some-test.replay\nPrepared replay for some-test.replay\n')
425         self.assertEqual(port.host.filesystem.read_binary_file('/path/some-dir/some-test-expected.png'), 'actual image')
426
427     def test_prepare_calls_run_single(self):
428         output_capture = OutputCapture()
429         output_capture.capture_output()
430         called = [False]
431
432         def run_single(driver, url, time_out_ms, record):
433             self.assertTrue(record)
434             self.assertEqual(url, '/path/some-dir/some-test.wpr')
435             called[0] = True
436             return False
437
438         test, port = self._setup_test()
439         test.run_single = run_single
440
441         try:
442             self.assertEqual(test.prepare(time_out_ms=100), False)
443         finally:
444             actual_stdout, actual_stderr, actual_logs = output_capture.restore_output()
445         self.assertTrue(called[0])
446         self.assertEqual(test._archive_path, '/path/some-dir/some-test.wpr')
447         self.assertEqual(test._url, 'http://some-test/')
448         self.assertEqual(actual_stdout, '')
449         self.assertEqual(actual_stderr, '')
450         self.assertEqual(actual_logs, "Preparing replay for some-test.replay\nFailed to prepare a replay for some-test.replay\n")
451
452 class TestPerfTestFactory(unittest.TestCase):
453     def test_regular_test(self):
454         test = PerfTestFactory.create_perf_test(MockPort(), 'some-dir/some-test', '/path/some-dir/some-test')
455         self.assertEqual(test.__class__, PerfTest)
456
457     def test_inspector_test(self):
458         test = PerfTestFactory.create_perf_test(MockPort(), 'inspector/some-test', '/path/inspector/some-test')
459         self.assertEqual(test.__class__, ChromiumStylePerfTest)