Extract a class to represent a test result to simplify PerfTest.parse_output and...
[WebKit-https.git] / Tools / Scripts / webkitpy / performance_tests / perftest_unittest.py
1 # Copyright (C) 2012 Google Inc. All rights reserved.
2 #
3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions are
5 # met:
6 #
7 #     * Redistributions of source code must retain the above copyright
8 # notice, this list of conditions and the following disclaimer.
9 #     * Redistributions in binary form must reproduce the above
10 # copyright notice, this list of conditions and the following disclaimer
11 # in the documentation and/or other materials provided with the
12 # distribution.
13 #     * Neither the name of Google Inc. nor the names of its
14 # contributors may be used to endorse or promote products derived from
15 # this software without specific prior written permission.
16 #
17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29 import StringIO
30 import json
31 import math
32 import unittest
33
34 from webkitpy.common.host_mock import MockHost
35 from webkitpy.common.system.outputcapture import OutputCapture
36 from webkitpy.layout_tests.port.driver import DriverOutput
37 from webkitpy.layout_tests.port.test import TestDriver
38 from webkitpy.layout_tests.port.test import TestPort
39 from webkitpy.performance_tests.perftest import ChromiumStylePerfTest
40 from webkitpy.performance_tests.perftest import PageLoadingPerfTest
41 from webkitpy.performance_tests.perftest import PerfTest
42 from webkitpy.performance_tests.perftest import PerfTestMetric
43 from webkitpy.performance_tests.perftest import PerfTestFactory
44 from webkitpy.performance_tests.perftest import ReplayPerfTest
45
46
47 class MockPort(TestPort):
48     def __init__(self, custom_run_test=None):
49         super(MockPort, self).__init__(host=MockHost(), custom_run_test=custom_run_test)
50
51
52 class TestPerfTestMetric(unittest.TestCase):
53     def test_init_set_missing_unit(self):
54         self.assertEqual(PerfTestMetric('Time', iterations=[1, 2, 3, 4, 5]).to_dict()['unit'], 'ms')
55         self.assertEqual(PerfTestMetric('Malloc', iterations=[1, 2, 3, 4, 5]).to_dict()['unit'], 'bytes')
56         self.assertEqual(PerfTestMetric('JSHeap', iterations=[1, 2, 3, 4, 5]).to_dict()['unit'], 'bytes')
57
58     def test_legacy_chromium_bot_compatible_test_name(self):
59         self.assertEqual(PerfTestMetric('Time').legacy_chromium_bot_compatible_test_name('test'), 'test')
60         self.assertEqual(PerfTestMetric('Malloc').legacy_chromium_bot_compatible_test_name('test'), 'test:Malloc')
61         self.assertEqual(PerfTestMetric('JSHeap').legacy_chromium_bot_compatible_test_name('test'), 'test:JSHeap')
62         self.assertEqual(PerfTestMetric('FontSize', unit='em').legacy_chromium_bot_compatible_test_name('test'), 'test:FontSize')
63
64     def test_has_values(self):
65         self.assertFalse(PerfTestMetric('Time').has_values())
66         self.assertTrue(PerfTestMetric('Time', iterations=[1]).has_values())
67
68     def test_append(self):
69         metric = PerfTestMetric('Time')
70         metric2 = PerfTestMetric('Time')
71         self.assertFalse(metric.has_values())
72         self.assertFalse(metric2.has_values())
73
74         metric.append(1)
75         self.assertTrue(metric.has_values())
76         self.assertFalse(metric2.has_values())
77         self.assertEqual(metric.to_dict()['values'], [1])
78         metric.append(2)
79         self.assertEqual(metric.to_dict()['values'], [1, 2])
80
81         metric2.append(3)
82         self.assertTrue(metric2.has_values())
83         self.assertEqual(metric.to_dict()['values'], [1, 2])
84         self.assertEqual(metric2.to_dict()['values'], [3])
85
86     def test_compute_statistics(self):
87         def compute_statistics(values):
88             statistics = PerfTestMetric.compute_statistics(map(lambda x: float(x), values))
89             return json.loads(json.dumps(statistics))
90
91         statistics = compute_statistics([10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 20, 19, 18, 17, 16, 15, 14, 13, 12, 11])
92         self.assertEqual(sorted(statistics.keys()), ['avg', 'max', 'median', 'min', 'stdev'])
93         self.assertEqual(statistics['avg'], 10.5)
94         self.assertEqual(statistics['min'], 1)
95         self.assertEqual(statistics['max'], 20)
96         self.assertEqual(statistics['median'], 10.5)
97         self.assertEqual(compute_statistics([8, 9, 10, 11, 12])['avg'], 10)
98         self.assertEqual(compute_statistics([8, 9, 10, 11, 12] * 4)['avg'], 10)
99         self.assertEqual(compute_statistics([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19])['avg'], 10)
100         self.assertEqual(compute_statistics([1, 5, 2, 8, 7])['median'], 5)
101         self.assertEqual(compute_statistics([1, 6, 2, 8, 7, 2])['median'], 4)
102         self.assertAlmostEqual(statistics['stdev'], math.sqrt(35))
103         self.assertAlmostEqual(compute_statistics([1])['stdev'], 0)
104         self.assertAlmostEqual(compute_statistics([1, 2, 3, 4, 5, 6])['stdev'], math.sqrt(3.5))
105         self.assertAlmostEqual(compute_statistics([4, 2, 5, 8, 6])['stdev'], math.sqrt(5))
106
107
108 class TestPerfTest(unittest.TestCase):
109     def _assert_results_are_correct(self, test, output):
110         test._filter_output(output)
111         parsed_results = test.parse_output(output)
112         self.assertEqual(len(parsed_results), 1)
113         some_test_results = parsed_results[0].to_dict()
114         self.assertEqual(sorted(some_test_results.keys()), ['avg', 'max', 'median', 'min', 'stdev', 'unit', 'values'])
115         self.assertEqual(some_test_results['values'], [1080, 1120, 1095, 1101, 1104])
116         self.assertEqual(some_test_results['min'], 1080)
117         self.assertEqual(some_test_results['max'], 1120)
118         self.assertEqual(some_test_results['avg'], 1100)
119         self.assertEqual(some_test_results['median'], 1101)
120         self.assertAlmostEqual(some_test_results['stdev'], 14.50862, places=5)
121         self.assertEqual(some_test_results['unit'], 'ms')
122
123     def test_parse_output(self):
124         output = DriverOutput("""
125 Running 20 times
126 Ignoring warm-up run (1115)
127
128 Time:
129 values 1080, 1120, 1095, 1101, 1104 ms
130 avg 1100 ms
131 median 1101 ms
132 stdev 14.50862 ms
133 min 1080 ms
134 max 1120 ms
135 """, image=None, image_hash=None, audio=None)
136         output_capture = OutputCapture()
137         output_capture.capture_output()
138         try:
139             test = PerfTest(MockPort(), 'some-test', '/path/some-dir/some-test')
140             self._assert_results_are_correct(test, output)
141         finally:
142             actual_stdout, actual_stderr, actual_logs = output_capture.restore_output()
143         self.assertEqual(actual_stdout, '')
144         self.assertEqual(actual_stderr, '')
145         self.assertEqual(actual_logs, '')
146
147     def test_parse_output_with_failing_line(self):
148         output = DriverOutput("""
149 Running 20 times
150 Ignoring warm-up run (1115)
151
152 some-unrecognizable-line
153
154 Time:
155 values 1080, 1120, 1095, 1101, 1104 ms
156 avg 1100 ms
157 median 1101 ms
158 stdev 14.50862 ms
159 min 1080 ms
160 max 1120 ms
161 """, image=None, image_hash=None, audio=None)
162         output_capture = OutputCapture()
163         output_capture.capture_output()
164         try:
165             test = PerfTest(MockPort(), 'some-test', '/path/some-dir/some-test')
166             test._filter_output(output)
167             self.assertEqual(test.parse_output(output), None)
168         finally:
169             actual_stdout, actual_stderr, actual_logs = output_capture.restore_output()
170         self.assertEqual(actual_stdout, '')
171         self.assertEqual(actual_stderr, '')
172         self.assertEqual(actual_logs, 'ERROR: some-unrecognizable-line\n')
173
174     def test_parse_output_with_description(self):
175         output = DriverOutput("""
176 Description: this is a test description.
177
178 Running 20 times
179 Ignoring warm-up run (1115)
180
181 Time:
182 values 1080, 1120, 1095, 1101, 1104 ms
183 avg 1100 ms
184 median 1101 ms
185 stdev 14.50862 ms
186 min 1080 ms
187 max 1120 ms""", image=None, image_hash=None, audio=None)
188         test = PerfTest(MockPort(), 'some-test', '/path/some-dir/some-test')
189         self._assert_results_are_correct(test, output)
190         self.assertEqual(test.description(), 'this is a test description.')
191
192     def test_ignored_stderr_lines(self):
193         test = PerfTest(MockPort(), 'some-test', '/path/some-dir/some-test')
194         ignored_lines = [
195             "Unknown option: --foo-bar",
196             "[WARNING:proxy_service.cc] bad moon a-rising",
197             "[INFO:SkFontHost_android.cpp(1158)] Use Test Config File Main /data/local/tmp/drt/android_main_fonts.xml, Fallback /data/local/tmp/drt/android_fallback_fonts.xml, Font Dir /data/local/tmp/drt/fonts/",
198         ]
199         for line in ignored_lines:
200             self.assertTrue(test._should_ignore_line_in_stderr(line))
201
202         non_ignored_lines = [
203             "Should not be ignored",
204             "[WARNING:chrome.cc] Something went wrong",
205             "[ERROR:main.cc] The sky has fallen",
206         ]
207         for line in non_ignored_lines:
208             self.assertFalse(test._should_ignore_line_in_stderr(line))
209
210     def test_parse_output_with_subtests(self):
211         output = DriverOutput("""
212 Running 20 times
213 some test: [1, 2, 3, 4, 5]
214 other test = else: [6, 7, 8, 9, 10]
215 Ignoring warm-up run (1115)
216
217 Time:
218 values 1080, 1120, 1095, 1101, 1104 ms
219 avg 1100 ms
220 median 1101 ms
221 stdev 14.50862 ms
222 min 1080 ms
223 max 1120 ms
224 """, image=None, image_hash=None, audio=None)
225         output_capture = OutputCapture()
226         output_capture.capture_output()
227         try:
228             test = PerfTest(MockPort(), 'some-test', '/path/some-dir/some-test')
229             self._assert_results_are_correct(test, output)
230         finally:
231             actual_stdout, actual_stderr, actual_logs = output_capture.restore_output()
232         self.assertEqual(actual_stdout, '')
233         self.assertEqual(actual_stderr, '')
234         self.assertEqual(actual_logs, '')
235
236
237 class TestPageLoadingPerfTest(unittest.TestCase):
238     class MockDriver(object):
239         def __init__(self, values, test, measurements=None):
240             self._values = values
241             self._index = 0
242             self._test = test
243             self._measurements = measurements
244
245         def run_test(self, input, stop_when_done):
246             if input.test_name == self._test.force_gc_test:
247                 return
248             value = self._values[self._index]
249             self._index += 1
250             if isinstance(value, str):
251                 return DriverOutput('some output', image=None, image_hash=None, audio=None, error=value)
252             else:
253                 return DriverOutput('some output', image=None, image_hash=None, audio=None, test_time=self._values[self._index - 1], measurements=self._measurements)
254
255     def test_run(self):
256         port = MockPort()
257         test = PageLoadingPerfTest(port, 'some-test', '/path/some-dir/some-test')
258         driver = TestPageLoadingPerfTest.MockDriver(range(1, 21), test)
259         output_capture = OutputCapture()
260         output_capture.capture_output()
261         try:
262             metrics = test._run_with_driver(driver, None)
263         finally:
264             actual_stdout, actual_stderr, actual_logs = output_capture.restore_output()
265
266         self.assertEqual(actual_stdout, '')
267         self.assertEqual(actual_stderr, '')
268         self.assertEqual(actual_logs, '')
269
270         self.assertEqual(len(metrics), 1)
271         self.assertEqual(metrics[0].metric(), 'Time')
272         self.assertEqual(metrics[0].to_dict(), {'max': 20000, 'avg': 11000.0, 'median': 11000, 'stdev': 5627.314338711378, 'min': 2000, 'unit': 'ms',
273             'values': [float(i * 1000) for i in range(2, 21)]})
274
275     def test_run_with_memory_output(self):
276         port = MockPort()
277         test = PageLoadingPerfTest(port, 'some-test', '/path/some-dir/some-test')
278         memory_results = {'Malloc': 10, 'JSHeap': 5}
279         self.maxDiff = None
280         driver = TestPageLoadingPerfTest.MockDriver(range(1, 21), test, memory_results)
281         output_capture = OutputCapture()
282         output_capture.capture_output()
283         try:
284             metrics = test._run_with_driver(driver, None)
285         finally:
286             actual_stdout, actual_stderr, actual_logs = output_capture.restore_output()
287
288         self.assertEqual(actual_stdout, '')
289         self.assertEqual(actual_stderr, '')
290         self.assertEqual(actual_logs, '')
291
292         self.assertEqual(len(metrics), 3)
293         self.assertEqual(metrics[0].metric(), 'Time')
294         self.assertEqual(metrics[0].to_dict(), {'max': 20000, 'avg': 11000.0, 'median': 11000, 'stdev': 5627.314338711378, 'min': 2000, 'unit': 'ms',
295             'values': [float(i * 1000) for i in range(2, 21)]})
296         self.assertEqual(metrics[1].metric(), 'Malloc')
297         self.assertEqual(metrics[1].to_dict(), {'max': 10, 'avg': 10.0, 'median': 10, 'min': 10, 'stdev': 0.0, 'unit': 'bytes',
298             'values': [float(10)] * 19})
299         self.assertEqual(metrics[2].metric(), 'JSHeap')
300         self.assertEqual(metrics[2].to_dict(), {'max': 5, 'avg': 5.0, 'median': 5, 'min': 5, 'stdev': 0.0, 'unit': 'bytes',
301             'values': [float(5)] * 19})
302
303     def test_run_with_bad_output(self):
304         output_capture = OutputCapture()
305         output_capture.capture_output()
306         try:
307             port = MockPort()
308             test = PageLoadingPerfTest(port, 'some-test', '/path/some-dir/some-test')
309             driver = TestPageLoadingPerfTest.MockDriver([1, 2, 3, 4, 5, 6, 7, 'some error', 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], test)
310             self.assertEqual(test._run_with_driver(driver, None), None)
311         finally:
312             actual_stdout, actual_stderr, actual_logs = output_capture.restore_output()
313         self.assertEqual(actual_stdout, '')
314         self.assertEqual(actual_stderr, '')
315         self.assertEqual(actual_logs, 'error: some-test\nsome error\n')
316
317
318 class TestReplayPerfTest(unittest.TestCase):
319
320     class ReplayTestPort(MockPort):
321         def __init__(self, custom_run_test=None):
322
323             class ReplayTestDriver(TestDriver):
324                 def run_test(self, text_input, stop_when_done):
325                     return custom_run_test(text_input, stop_when_done) if custom_run_test else None
326
327             self._custom_driver_class = ReplayTestDriver
328             super(self.__class__, self).__init__()
329
330         def _driver_class(self):
331             return self._custom_driver_class
332
333     class MockReplayServer(object):
334         def __init__(self, wait_until_ready=True):
335             self.wait_until_ready = lambda: wait_until_ready
336
337         def stop(self):
338             pass
339
340     def _add_file(self, port, dirname, filename, content=True):
341         port.host.filesystem.maybe_make_directory(dirname)
342         port.host.filesystem.write_binary_file(port.host.filesystem.join(dirname, filename), content)
343
344     def _setup_test(self, run_test=None):
345         test_port = self.ReplayTestPort(run_test)
346         self._add_file(test_port, '/path/some-dir', 'some-test.replay', 'http://some-test/')
347         test = ReplayPerfTest(test_port, 'some-test.replay', '/path/some-dir/some-test.replay')
348         test._start_replay_server = lambda archive, record: self.__class__.MockReplayServer()
349         return test, test_port
350
351     def test_run_single(self):
352         output_capture = OutputCapture()
353         output_capture.capture_output()
354
355         loaded_pages = []
356
357         def run_test(test_input, stop_when_done):
358             if test_input.test_name == test.force_gc_test:
359                 loaded_pages.append(test_input)
360                 return
361             if test_input.test_name != "about:blank":
362                 self.assertEqual(test_input.test_name, 'http://some-test/')
363             loaded_pages.append(test_input)
364             self._add_file(port, '/path/some-dir', 'some-test.wpr', 'wpr content')
365             return DriverOutput('actual text', 'actual image', 'actual checksum',
366                 audio=None, crash=False, timeout=False, error=False)
367
368         test, port = self._setup_test(run_test)
369         test._archive_path = '/path/some-dir/some-test.wpr'
370         test._url = 'http://some-test/'
371
372         try:
373             driver = port.create_driver(worker_number=1, no_timeout=True)
374             self.assertTrue(test.run_single(driver, '/path/some-dir/some-test.replay', time_out_ms=100))
375         finally:
376             actual_stdout, actual_stderr, actual_logs = output_capture.restore_output()
377
378         self.assertEqual(len(loaded_pages), 2)
379         self.assertEqual(loaded_pages[0].test_name, test.force_gc_test)
380         self.assertEqual(loaded_pages[1].test_name, 'http://some-test/')
381         self.assertEqual(actual_stdout, '')
382         self.assertEqual(actual_stderr, '')
383         self.assertEqual(actual_logs, '')
384         self.assertEqual(port.host.filesystem.read_binary_file('/path/some-dir/some-test-actual.png'), 'actual image')
385
386     def test_run_single_fails_without_webpagereplay(self):
387         output_capture = OutputCapture()
388         output_capture.capture_output()
389
390         test, port = self._setup_test()
391         test._start_replay_server = lambda archive, record: None
392         test._archive_path = '/path/some-dir.wpr'
393         test._url = 'http://some-test/'
394
395         try:
396             driver = port.create_driver(worker_number=1, no_timeout=True)
397             self.assertEqual(test.run_single(driver, '/path/some-dir/some-test.replay', time_out_ms=100), None)
398         finally:
399             actual_stdout, actual_stderr, actual_logs = output_capture.restore_output()
400         self.assertEqual(actual_stdout, '')
401         self.assertEqual(actual_stderr, '')
402         self.assertEqual(actual_logs, "Web page replay didn't start.\n")
403
404     def test_prepare_fails_when_wait_until_ready_fails(self):
405         output_capture = OutputCapture()
406         output_capture.capture_output()
407
408         test, port = self._setup_test()
409         test._start_replay_server = lambda archive, record: self.__class__.MockReplayServer(wait_until_ready=False)
410         test._archive_path = '/path/some-dir.wpr'
411         test._url = 'http://some-test/'
412
413         try:
414             driver = port.create_driver(worker_number=1, no_timeout=True)
415             self.assertEqual(test.run_single(driver, '/path/some-dir/some-test.replay', time_out_ms=100), None)
416         finally:
417             actual_stdout, actual_stderr, actual_logs = output_capture.restore_output()
418
419         self.assertEqual(actual_stdout, '')
420         self.assertEqual(actual_stderr, '')
421         self.assertEqual(actual_logs, "Web page replay didn't start.\n")
422
423     def test_run_single_fails_when_output_has_error(self):
424         output_capture = OutputCapture()
425         output_capture.capture_output()
426
427         loaded_pages = []
428
429         def run_test(test_input, stop_when_done):
430             loaded_pages.append(test_input)
431             self._add_file(port, '/path/some-dir', 'some-test.wpr', 'wpr content')
432             return DriverOutput('actual text', 'actual image', 'actual checksum',
433                 audio=None, crash=False, timeout=False, error='some error')
434
435         test, port = self._setup_test(run_test)
436         test._archive_path = '/path/some-dir.wpr'
437         test._url = 'http://some-test/'
438
439         try:
440             driver = port.create_driver(worker_number=1, no_timeout=True)
441             self.assertEqual(test.run_single(driver, '/path/some-dir/some-test.replay', time_out_ms=100), None)
442         finally:
443             actual_stdout, actual_stderr, actual_logs = output_capture.restore_output()
444
445         self.assertEqual(len(loaded_pages), 2)
446         self.assertEqual(loaded_pages[0].test_name, test.force_gc_test)
447         self.assertEqual(loaded_pages[1].test_name, 'http://some-test/')
448         self.assertEqual(actual_stdout, '')
449         self.assertEqual(actual_stderr, '')
450         self.assertEqual(actual_logs, 'error: some-test.replay\nsome error\n')
451
452     def test_prepare(self):
453         output_capture = OutputCapture()
454         output_capture.capture_output()
455
456         def run_test(test_input, stop_when_done):
457             self._add_file(port, '/path/some-dir', 'some-test.wpr', 'wpr content')
458             return DriverOutput('actual text', 'actual image', 'actual checksum',
459                 audio=None, crash=False, timeout=False, error=False)
460
461         test, port = self._setup_test(run_test)
462
463         try:
464             self.assertEqual(test.prepare(time_out_ms=100), True)
465         finally:
466             actual_stdout, actual_stderr, actual_logs = output_capture.restore_output()
467
468         self.assertEqual(actual_stdout, '')
469         self.assertEqual(actual_stderr, '')
470         self.assertEqual(actual_logs, 'Preparing replay for some-test.replay\nPrepared replay for some-test.replay\n')
471         self.assertEqual(port.host.filesystem.read_binary_file('/path/some-dir/some-test-expected.png'), 'actual image')
472
473     def test_prepare_calls_run_single(self):
474         output_capture = OutputCapture()
475         output_capture.capture_output()
476         called = [False]
477
478         def run_single(driver, url, time_out_ms, record):
479             self.assertTrue(record)
480             self.assertEqual(url, '/path/some-dir/some-test.wpr')
481             called[0] = True
482             return False
483
484         test, port = self._setup_test()
485         test.run_single = run_single
486
487         try:
488             self.assertEqual(test.prepare(time_out_ms=100), False)
489         finally:
490             actual_stdout, actual_stderr, actual_logs = output_capture.restore_output()
491         self.assertTrue(called[0])
492         self.assertEqual(test._archive_path, '/path/some-dir/some-test.wpr')
493         self.assertEqual(test._url, 'http://some-test/')
494         self.assertEqual(actual_stdout, '')
495         self.assertEqual(actual_stderr, '')
496         self.assertEqual(actual_logs, "Preparing replay for some-test.replay\nFailed to prepare a replay for some-test.replay\n")
497
498 class TestPerfTestFactory(unittest.TestCase):
499     def test_regular_test(self):
500         test = PerfTestFactory.create_perf_test(MockPort(), 'some-dir/some-test', '/path/some-dir/some-test')
501         self.assertEqual(test.__class__, PerfTest)
502
503     def test_inspector_test(self):
504         test = PerfTestFactory.create_perf_test(MockPort(), 'inspector/some-test', '/path/inspector/some-test')
505         self.assertEqual(test.__class__, ChromiumStylePerfTest)