6f04ed8d08942154292a1d13a6fa7ef00faf7714
[WebKit-https.git] / Tools / Scripts / webkitpy / performance_tests / perftestsrunner_unittest.py
1 #!/usr/bin/python
2 # Copyright (C) 2012 Google Inc. All rights reserved.
3 #
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
6 # met:
7 #
8 #     * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 #     * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
13 # distribution.
14 #     * Neither the name of Google Inc. nor the names of its
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
17 #
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30 """Unit tests for run_perf_tests."""
31
32 import StringIO
33 import json
34 import unittest
35
36 from webkitpy.common.host_mock import MockHost
37 from webkitpy.common.system.filesystem_mock import MockFileSystem
38 from webkitpy.common.system.outputcapture import OutputCapture
39 from webkitpy.layout_tests.port.driver import DriverInput, DriverOutput
40 from webkitpy.layout_tests.port.test import TestPort
41 from webkitpy.layout_tests.views import printing
42 from webkitpy.performance_tests.perftest import ChromiumStylePerfTest
43 from webkitpy.performance_tests.perftest import PerfTest
44 from webkitpy.performance_tests.perftestsrunner import PerfTestsRunner
45
46
47 class MainTest(unittest.TestCase):
48     def assertWritten(self, stream, contents):
49         self.assertEquals(stream.buflist, contents)
50
51     class TestDriver:
52         def run_test(self, driver_input):
53             text = ''
54             timeout = False
55             crash = False
56             if driver_input.test_name.endswith('pass.html'):
57                 text = 'RESULT group_name: test_name= 42 ms'
58             elif driver_input.test_name.endswith('timeout.html'):
59                 timeout = True
60             elif driver_input.test_name.endswith('failed.html'):
61                 text = None
62             elif driver_input.test_name.endswith('tonguey.html'):
63                 text = 'we are not expecting an output from perf tests but RESULT blablabla'
64             elif driver_input.test_name.endswith('crash.html'):
65                 crash = True
66             elif driver_input.test_name.endswith('event-target-wrapper.html'):
67                 text = """Running 20 times
68 Ignoring warm-up run (1502)
69 1504
70 1505
71 1510
72 1504
73 1507
74 1509
75 1510
76 1487
77 1488
78 1472
79 1472
80 1488
81 1473
82 1472
83 1475
84 1487
85 1486
86 1486
87 1475
88 1471
89
90 avg 1489.05
91 median 1487
92 stdev 14.46
93 min 1471
94 max 1510
95 """
96             elif driver_input.test_name.endswith('some-parser.html'):
97                 text = """Running 20 times
98 Ignoring warm-up run (1115)
99
100 avg 1100
101 median 1101
102 stdev 11
103 min 1080
104 max 1120
105 """
106             return DriverOutput(text, '', '', '', crash=crash, timeout=timeout)
107
108         def start(self):
109             """do nothing"""
110
111         def stop(self):
112             """do nothing"""
113
114     def create_runner(self, buildbot_output=None, args=[], regular_output=None, driver_class=TestDriver):
115         buildbot_output = buildbot_output or StringIO.StringIO()
116         regular_output = regular_output or StringIO.StringIO()
117
118         options, parsed_args = PerfTestsRunner._parse_args(args)
119         test_port = TestPort(host=MockHost(), options=options)
120         test_port.create_driver = lambda worker_number=None, no_timeout=False: driver_class()
121
122         runner = PerfTestsRunner(regular_output, buildbot_output, args=args, port=test_port)
123         runner._host.filesystem.maybe_make_directory(runner._base_path, 'inspector')
124         runner._host.filesystem.maybe_make_directory(runner._base_path, 'Bindings')
125         runner._host.filesystem.maybe_make_directory(runner._base_path, 'Parser')
126         return runner
127
128     def run_test(self, test_name):
129         runner = self.create_runner()
130         driver = MainTest.TestDriver()
131         return runner._run_single_test(ChromiumStylePerfTest(test_name, 'some-dir',
132             runner._host.filesystem.join('some-dir', test_name)), driver)
133
134     def test_run_passing_test(self):
135         self.assertTrue(self.run_test('pass.html'))
136
137     def test_run_silent_test(self):
138         self.assertFalse(self.run_test('silent.html'))
139
140     def test_run_failed_test(self):
141         self.assertFalse(self.run_test('failed.html'))
142
143     def test_run_tonguey_test(self):
144         self.assertFalse(self.run_test('tonguey.html'))
145
146     def test_run_timeout_test(self):
147         self.assertFalse(self.run_test('timeout.html'))
148
149     def test_run_crash_test(self):
150         self.assertFalse(self.run_test('crash.html'))
151
152     def _tests_for_runner(self, runner, test_names):
153         filesystem = runner._host.filesystem
154         tests = []
155         for test in test_names:
156             path = filesystem.join(runner._base_path, test)
157             dirname = filesystem.dirname(path)
158             if test.startswith('inspector/'):
159                 tests.append(ChromiumStylePerfTest(test, dirname, path))
160             else:
161                 tests.append(PerfTest(test, dirname, path))
162         return tests
163
164     def test_run_test_set(self):
165         buildbot_output = StringIO.StringIO()
166         runner = self.create_runner(buildbot_output)
167         tests = self._tests_for_runner(runner, ['inspector/pass.html', 'inspector/silent.html', 'inspector/failed.html',
168             'inspector/tonguey.html', 'inspector/timeout.html', 'inspector/crash.html'])
169         unexpected_result_count = runner._run_tests_set(tests, runner._port)
170         self.assertEqual(unexpected_result_count, len(tests) - 1)
171         self.assertWritten(buildbot_output, ['RESULT group_name: test_name= 42 ms\n'])
172
173     def test_run_test_set_kills_drt_per_run(self):
174
175         class TestDriverWithStopCount(MainTest.TestDriver):
176             stop_count = 0
177
178             def stop(self):
179                 TestDriverWithStopCount.stop_count += 1
180
181         buildbot_output = StringIO.StringIO()
182         runner = self.create_runner(buildbot_output, driver_class=TestDriverWithStopCount)
183         tests = self._tests_for_runner(runner, ['inspector/pass.html', 'inspector/silent.html', 'inspector/failed.html',
184             'inspector/tonguey.html', 'inspector/timeout.html', 'inspector/crash.html'])
185
186         unexpected_result_count = runner._run_tests_set(tests, runner._port)
187         self.assertEqual(TestDriverWithStopCount.stop_count, 6)
188
189     def test_run_test_pause_before_testing(self):
190         class TestDriverWithStartCount(MainTest.TestDriver):
191             start_count = 0
192
193             def start(self):
194                 TestDriverWithStartCount.start_count += 1
195
196         buildbot_output = StringIO.StringIO()
197         regular_output = StringIO.StringIO()
198         runner = self.create_runner(buildbot_output, args=["--pause-before-testing"], regular_output=regular_output, driver_class=TestDriverWithStartCount)
199         tests = self._tests_for_runner(runner, ['inspector/pass.html'])
200
201         try:
202             output = OutputCapture()
203             output.capture_output()
204             unexpected_result_count = runner._run_tests_set(tests, runner._port)
205             self.assertEqual(TestDriverWithStartCount.start_count, 1)
206         finally:
207             _, stderr, _ = output.restore_output()
208             self.assertEqual(stderr, "Ready to run test?\n")
209             self.assertTrue("Running inspector/pass.html (1 of 1)" in regular_output.getvalue())
210
211     def test_run_test_set_for_parser_tests(self):
212         buildbot_output = StringIO.StringIO()
213         runner = self.create_runner(buildbot_output)
214         tests = self._tests_for_runner(runner, ['Bindings/event-target-wrapper.html', 'Parser/some-parser.html'])
215         unexpected_result_count = runner._run_tests_set(tests, runner._port)
216         self.assertEqual(unexpected_result_count, 0)
217         self.assertWritten(buildbot_output, ['RESULT Bindings: event-target-wrapper= 1489.05 ms\n',
218                                              'median= 1487.0 ms, stdev= 14.46 ms, min= 1471.0 ms, max= 1510.0 ms\n',
219                                              'RESULT Parser: some-parser= 1100.0 ms\n',
220                                              'median= 1101.0 ms, stdev= 11.0 ms, min= 1080.0 ms, max= 1120.0 ms\n'])
221
222     def test_run_test_set_with_json_output(self):
223         buildbot_output = StringIO.StringIO()
224         runner = self.create_runner(buildbot_output, args=['--output-json-path=/mock-checkout/output.json'])
225         runner._host.filesystem.files[runner._base_path + '/inspector/pass.html'] = True
226         runner._host.filesystem.files[runner._base_path + '/Bindings/event-target-wrapper.html'] = True
227         runner._timestamp = 123456789
228         self.assertEqual(runner.run(), 0)
229         self.assertWritten(buildbot_output, ['RESULT Bindings: event-target-wrapper= 1489.05 ms\n',
230                                              'median= 1487.0 ms, stdev= 14.46 ms, min= 1471.0 ms, max= 1510.0 ms\n',
231                                              'RESULT group_name: test_name= 42 ms\n'])
232
233         self.assertEqual(json.loads(runner._host.filesystem.files['/mock-checkout/output.json']), {
234             "timestamp": 123456789, "results":
235             {"Bindings/event-target-wrapper": {"max": 1510, "avg": 1489.05, "median": 1487, "min": 1471, "stdev": 14.46, "unit": "ms"},
236             "inspector/pass.html:group_name:test_name": 42},
237             "webkit-revision": 5678})
238
239     def test_run_test_set_with_json_source(self):
240         buildbot_output = StringIO.StringIO()
241         runner = self.create_runner(buildbot_output, args=['--output-json-path=/mock-checkout/output.json',
242             '--source-json-path=/mock-checkout/source.json'])
243         runner._host.filesystem.files['/mock-checkout/source.json'] = '{"key": "value"}'
244         runner._host.filesystem.files[runner._base_path + '/inspector/pass.html'] = True
245         runner._host.filesystem.files[runner._base_path + '/Bindings/event-target-wrapper.html'] = True
246         runner._timestamp = 123456789
247         self.assertEqual(runner.run(), 0)
248         self.assertWritten(buildbot_output, ['RESULT Bindings: event-target-wrapper= 1489.05 ms\n',
249                                              'median= 1487.0 ms, stdev= 14.46 ms, min= 1471.0 ms, max= 1510.0 ms\n',
250                                              'RESULT group_name: test_name= 42 ms\n'])
251
252         self.assertEqual(json.loads(runner._host.filesystem.files['/mock-checkout/output.json']), {
253             "timestamp": 123456789, "results":
254             {"Bindings/event-target-wrapper": {"max": 1510, "avg": 1489.05, "median": 1487, "min": 1471, "stdev": 14.46, "unit": "ms"},
255             "inspector/pass.html:group_name:test_name": 42},
256             "webkit-revision": 5678,
257             "key": "value"})
258
259     def test_run_test_set_with_multiple_repositories(self):
260         buildbot_output = StringIO.StringIO()
261         runner = self.create_runner(buildbot_output, args=['--output-json-path=/mock-checkout/output.json'])
262         runner._host.filesystem.files[runner._base_path + '/inspector/pass.html'] = True
263         runner._timestamp = 123456789
264         runner._port.repository_paths = lambda: [('webkit', '/mock-checkout'), ('some', '/mock-checkout/some')]
265         self.assertEqual(runner.run(), 0)
266
267         self.assertEqual(json.loads(runner._host.filesystem.files['/mock-checkout/output.json']), {
268             "timestamp": 123456789, "results": {"inspector/pass.html:group_name:test_name": 42.0}, "webkit-revision": 5678, "some-revision": 5678})
269
270     def test_run_with_upload_json(self):
271         runner = self.create_runner(args=['--output-json-path=/mock-checkout/output.json',
272             '--test-results-server', 'some.host', '--platform', 'platform1', '--builder-name', 'builder1', '--build-number', '123'])
273         upload_json_is_called = [False]
274         upload_json_returns_true = True
275
276         def mock_upload_json(hostname, json_path):
277             self.assertEqual(hostname, 'some.host')
278             self.assertEqual(json_path, '/mock-checkout/output.json')
279             upload_json_is_called[0] = True
280             return upload_json_returns_true
281
282         runner._upload_json = mock_upload_json
283         runner._host.filesystem.files['/mock-checkout/source.json'] = '{"key": "value"}'
284         runner._host.filesystem.files[runner._base_path + '/inspector/pass.html'] = True
285         runner._host.filesystem.files[runner._base_path + '/Bindings/event-target-wrapper.html'] = True
286         runner._timestamp = 123456789
287         self.assertEqual(runner.run(), 0)
288         self.assertEqual(upload_json_is_called[0], True)
289         generated_json = json.loads(runner._host.filesystem.files['/mock-checkout/output.json'])
290         self.assertEqual(generated_json['platform'], 'platform1')
291         self.assertEqual(generated_json['builder-name'], 'builder1')
292         self.assertEqual(generated_json['build-number'], 123)
293         upload_json_returns_true = False
294
295         runner = self.create_runner(args=['--output-json-path=/mock-checkout/output.json',
296             '--test-results-server', 'some.host', '--platform', 'platform1', '--builder-name', 'builder1', '--build-number', '123'])
297         runner._upload_json = mock_upload_json
298         self.assertEqual(runner.run(), -3)
299
300     def test_upload_json(self):
301         regular_output = StringIO.StringIO()
302         runner = self.create_runner(regular_output=regular_output)
303         runner._host.filesystem.files['/mock-checkout/some.json'] = 'some content'
304
305         called = []
306         upload_single_text_file_throws = False
307         upload_single_text_file_return_value = StringIO.StringIO('OK')
308
309         class MockFileUploader:
310             def __init__(mock, url, timeout):
311                 self.assertEqual(url, 'https://some.host/api/test/report')
312                 self.assertTrue(isinstance(timeout, int) and timeout)
313                 called.append('FileUploader')
314
315             def upload_single_text_file(mock, filesystem, content_type, filename):
316                 self.assertEqual(filesystem, runner._host.filesystem)
317                 self.assertEqual(content_type, 'application/json')
318                 self.assertEqual(filename, 'some.json')
319                 called.append('upload_single_text_file')
320                 if upload_single_text_file_throws:
321                     raise "Some exception"
322                 return upload_single_text_file_return_value
323
324         runner._upload_json('some.host', 'some.json', MockFileUploader)
325         self.assertEqual(called, ['FileUploader', 'upload_single_text_file'])
326
327         output = OutputCapture()
328         output.capture_output()
329         upload_single_text_file_return_value = StringIO.StringIO('Some error')
330         runner._upload_json('some.host', 'some.json', MockFileUploader)
331         _, _, logs = output.restore_output()
332         self.assertEqual(logs, 'Uploaded JSON but got a bad response:\nSome error\n')
333
334         # Throwing an exception upload_single_text_file shouldn't blow up _upload_json
335         called = []
336         upload_single_text_file_throws = True
337         runner._upload_json('some.host', 'some.json', MockFileUploader)
338         self.assertEqual(called, ['FileUploader', 'upload_single_text_file'])
339
340     def test_collect_tests(self):
341         runner = self.create_runner()
342         filename = runner._host.filesystem.join(runner._base_path, 'inspector', 'a_file.html')
343         runner._host.filesystem.files[filename] = 'a content'
344         tests = runner._collect_tests()
345         self.assertEqual(len(tests), 1)
346
347     def _collect_tests_and_sort_test_name(self, runner):
348         return sorted([test.test_name() for test in runner._collect_tests()])
349
350     def test_collect_tests(self):
351         runner = self.create_runner(args=['PerformanceTests/test1.html', 'test2.html'])
352
353         def add_file(filename):
354             runner._host.filesystem.files[runner._host.filesystem.join(runner._base_path, filename)] = 'some content'
355
356         add_file('test1.html')
357         add_file('test2.html')
358         add_file('test3.html')
359         runner._host.filesystem.chdir(runner._port.perf_tests_dir()[:runner._port.perf_tests_dir().rfind(runner._host.filesystem.sep)])
360         self.assertEqual(self._collect_tests_and_sort_test_name(runner), ['test1.html', 'test2.html'])
361
362     def test_collect_tests_with_skipped_list(self):
363         runner = self.create_runner()
364
365         def add_file(dirname, filename, content=True):
366             dirname = runner._host.filesystem.join(runner._base_path, dirname) if dirname else runner._base_path
367             runner._host.filesystem.maybe_make_directory(dirname)
368             runner._host.filesystem.files[runner._host.filesystem.join(dirname, filename)] = content
369
370         add_file('inspector', 'test1.html')
371         add_file('inspector', 'unsupported_test1.html')
372         add_file('inspector', 'test2.html')
373         add_file('inspector/resources', 'resource_file.html')
374         add_file('unsupported', 'unsupported_test2.html')
375         runner._port.skipped_perf_tests = lambda: ['inspector/unsupported_test1.html', 'unsupported']
376         self.assertEqual(self._collect_tests_and_sort_test_name(runner), ['inspector/test1.html', 'inspector/test2.html'])
377
378     def test_parse_args(self):
379         runner = self.create_runner()
380         options, args = PerfTestsRunner._parse_args([
381                 '--verbose',
382                 '--build-directory=folder42',
383                 '--platform=platform42',
384                 '--builder-name', 'webkit-mac-1',
385                 '--build-number=56',
386                 '--time-out-ms=42',
387                 '--output-json-path=a/output.json',
388                 '--source-json-path=a/source.json',
389                 '--test-results-server=somehost',
390                 '--debug', 'an_arg'])
391         self.assertEqual(options.build, True)
392         self.assertEqual(options.verbose, True)
393         self.assertEqual(options.help_printing, None)
394         self.assertEqual(options.build_directory, 'folder42')
395         self.assertEqual(options.platform, 'platform42')
396         self.assertEqual(options.builder_name, 'webkit-mac-1')
397         self.assertEqual(options.build_number, '56')
398         self.assertEqual(options.time_out_ms, '42')
399         self.assertEqual(options.configuration, 'Debug')
400         self.assertEqual(options.print_options, None)
401         self.assertEqual(options.output_json_path, 'a/output.json')
402         self.assertEqual(options.source_json_path, 'a/source.json')
403         self.assertEqual(options.test_results_server, 'somehost')
404
405
406 if __name__ == '__main__':
407     unittest.main()