58bd377d689002ca4d17584d1eb175c15b4ff11b
[WebKit-https.git] / Tools / Scripts / webkitpy / performance_tests / perftestsrunner_unittest.py
1 #!/usr/bin/python
2 # Copyright (C) 2012 Google Inc. All rights reserved.
3 #
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
6 # met:
7 #
8 #     * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 #     * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
13 # distribution.
14 #     * Neither the name of Google Inc. nor the names of its
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
17 #
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30 """Unit tests for run_perf_tests."""
31
32 import StringIO
33 import json
34 import unittest
35
36 from webkitpy.common import array_stream
37 from webkitpy.common.host_mock import MockHost
38 from webkitpy.common.system.filesystem_mock import MockFileSystem
39 from webkitpy.common.system.outputcapture import OutputCapture
40 from webkitpy.layout_tests.port.driver import DriverInput, DriverOutput
41 from webkitpy.layout_tests.port.test import TestPort
42 from webkitpy.layout_tests.views import printing
43 from webkitpy.performance_tests.perftestsrunner import PerfTestsRunner
44
45
46 class MainTest(unittest.TestCase):
47     class TestDriver:
48         def run_test(self, driver_input):
49             text = ''
50             timeout = False
51             crash = False
52             if driver_input.test_name.endswith('pass.html'):
53                 text = 'RESULT group_name: test_name= 42 ms'
54             elif driver_input.test_name.endswith('timeout.html'):
55                 timeout = True
56             elif driver_input.test_name.endswith('failed.html'):
57                 text = None
58             elif driver_input.test_name.endswith('tonguey.html'):
59                 text = 'we are not expecting an output from perf tests but RESULT blablabla'
60             elif driver_input.test_name.endswith('crash.html'):
61                 crash = True
62             elif driver_input.test_name.endswith('event-target-wrapper.html'):
63                 text = """Running 20 times
64 Ignoring warm-up run (1502)
65 1504
66 1505
67 1510
68 1504
69 1507
70 1509
71 1510
72 1487
73 1488
74 1472
75 1472
76 1488
77 1473
78 1472
79 1475
80 1487
81 1486
82 1486
83 1475
84 1471
85
86 avg 1489.05
87 median 1487
88 stdev 14.46
89 min 1471
90 max 1510
91 """
92             elif driver_input.test_name.endswith('some-parser.html'):
93                 text = """Running 20 times
94 Ignoring warm-up run (1115)
95
96 avg 1100
97 median 1101
98 stdev 11
99 min 1080
100 max 1120
101 """
102             return DriverOutput(text, '', '', '', crash=crash, timeout=timeout)
103
104         def stop(self):
105             """do nothing"""
106
107     def create_runner(self, buildbot_output=None, args=[], regular_output=None, driver_class=TestDriver):
108         buildbot_output = buildbot_output or array_stream.ArrayStream()
109         regular_output = regular_output or array_stream.ArrayStream()
110
111         options, parsed_args = PerfTestsRunner._parse_args(args)
112         test_port = TestPort(host=MockHost(), options=options)
113         test_port.create_driver = lambda worker_number=None, no_timeout=False: driver_class()
114
115         runner = PerfTestsRunner(regular_output, buildbot_output, args=args, port=test_port)
116         runner._host.filesystem.maybe_make_directory(runner._base_path, 'inspector')
117         runner._host.filesystem.maybe_make_directory(runner._base_path, 'Bindings')
118         runner._host.filesystem.maybe_make_directory(runner._base_path, 'Parser')
119         return runner
120
121     def run_test(self, test_name):
122         runner = self.create_runner()
123         driver = MainTest.TestDriver()
124         return runner._run_single_test(test_name, driver, is_chromium_style=True)
125
126     def test_run_passing_test(self):
127         self.assertTrue(self.run_test('pass.html'))
128
129     def test_run_silent_test(self):
130         self.assertFalse(self.run_test('silent.html'))
131
132     def test_run_failed_test(self):
133         self.assertFalse(self.run_test('failed.html'))
134
135     def test_run_tonguey_test(self):
136         self.assertFalse(self.run_test('tonguey.html'))
137
138     def test_run_timeout_test(self):
139         self.assertFalse(self.run_test('timeout.html'))
140
141     def test_run_crash_test(self):
142         self.assertFalse(self.run_test('crash.html'))
143
144     def test_run_test_set(self):
145         buildbot_output = array_stream.ArrayStream()
146         runner = self.create_runner(buildbot_output)
147         dirname = runner._base_path + '/inspector/'
148         tests = [dirname + 'pass.html', dirname + 'silent.html', dirname + 'failed.html',
149             dirname + 'tonguey.html', dirname + 'timeout.html', dirname + 'crash.html']
150         unexpected_result_count = runner._run_tests_set(tests, runner._port)
151         self.assertEqual(unexpected_result_count, len(tests) - 1)
152         self.assertEqual(len(buildbot_output.get()), 1)
153         self.assertEqual(buildbot_output.get()[0], 'RESULT group_name: test_name= 42 ms\n')
154
155     def test_run_test_set_kills_drt_per_run(self):
156
157         class TestDriverWithStopCount(MainTest.TestDriver):
158             stop_count = 0
159
160             def __init__(self):
161                 TestDriverWithStopCount.sotp_count = 0
162
163             def stop(self):
164                 TestDriverWithStopCount.stop_count += 1
165
166         buildbot_output = array_stream.ArrayStream()
167         runner = self.create_runner(buildbot_output, driver_class=TestDriverWithStopCount)
168
169         dirname = runner._base_path + '/inspector/'
170         tests = [dirname + 'pass.html', dirname + 'silent.html', dirname + 'failed.html',
171             dirname + 'tonguey.html', dirname + 'timeout.html', dirname + 'crash.html']
172
173         unexpected_result_count = runner._run_tests_set(tests, runner._port)
174         self.assertEqual(TestDriverWithStopCount.stop_count, 6)
175
176     def test_run_test_set_for_parser_tests(self):
177         buildbot_output = array_stream.ArrayStream()
178         runner = self.create_runner(buildbot_output)
179         tests = [runner._base_path + '/Bindings/event-target-wrapper.html', runner._base_path + '/Parser/some-parser.html']
180         unexpected_result_count = runner._run_tests_set(tests, runner._port)
181         self.assertEqual(unexpected_result_count, 0)
182         self.assertEqual(buildbot_output.get()[0], 'RESULT Bindings: event-target-wrapper= 1489.05 ms\n')
183         self.assertEqual(buildbot_output.get()[1], 'median= 1487.0 ms, stdev= 14.46 ms, min= 1471.0 ms, max= 1510.0 ms\n')
184         self.assertEqual(buildbot_output.get()[2], 'RESULT Parser: some-parser= 1100.0 ms\n')
185         self.assertEqual(buildbot_output.get()[3], 'median= 1101.0 ms, stdev= 11.0 ms, min= 1080.0 ms, max= 1120.0 ms\n')
186
187     def test_run_test_set_with_json_output(self):
188         buildbot_output = array_stream.ArrayStream()
189         runner = self.create_runner(buildbot_output, args=['--output-json-path=/mock-checkout/output.json'])
190         runner._host.filesystem.files[runner._base_path + '/inspector/pass.html'] = True
191         runner._host.filesystem.files[runner._base_path + '/Bindings/event-target-wrapper.html'] = True
192         runner._timestamp = 123456789
193         self.assertEqual(runner.run(), 0)
194         self.assertEqual(len(buildbot_output.get()), 3)
195         self.assertEqual(buildbot_output.get()[0], 'RESULT Bindings: event-target-wrapper= 1489.05 ms\n')
196         self.assertEqual(buildbot_output.get()[1], 'median= 1487.0 ms, stdev= 14.46 ms, min= 1471.0 ms, max= 1510.0 ms\n')
197         self.assertEqual(buildbot_output.get()[2], 'RESULT group_name: test_name= 42 ms\n')
198
199         self.assertEqual(json.loads(runner._host.filesystem.files['/mock-checkout/output.json']), {
200             "timestamp": 123456789, "results":
201             {"Bindings/event-target-wrapper": {"max": 1510, "avg": 1489.05, "median": 1487, "min": 1471, "stdev": 14.46},
202             "group_name:test_name": 42},
203             "revision": 1234})
204
205     def test_run_test_set_with_json_source(self):
206         buildbot_output = array_stream.ArrayStream()
207         runner = self.create_runner(buildbot_output, args=['--output-json-path=/mock-checkout/output.json',
208             '--source-json-path=/mock-checkout/source.json'])
209         runner._host.filesystem.files['/mock-checkout/source.json'] = '{"key": "value"}'
210         runner._host.filesystem.files[runner._base_path + '/inspector/pass.html'] = True
211         runner._host.filesystem.files[runner._base_path + '/Bindings/event-target-wrapper.html'] = True
212         runner._timestamp = 123456789
213         self.assertEqual(runner.run(), 0)
214         self.assertEqual(len(buildbot_output.get()), 3)
215         self.assertEqual(buildbot_output.get()[0], 'RESULT Bindings: event-target-wrapper= 1489.05 ms\n')
216         self.assertEqual(buildbot_output.get()[1], 'median= 1487.0 ms, stdev= 14.46 ms, min= 1471.0 ms, max= 1510.0 ms\n')
217         self.assertEqual(buildbot_output.get()[2], 'RESULT group_name: test_name= 42 ms\n')
218
219         self.assertEqual(json.loads(runner._host.filesystem.files['/mock-checkout/output.json']), {
220             "timestamp": 123456789, "results":
221             {"Bindings/event-target-wrapper": {"max": 1510, "avg": 1489.05, "median": 1487, "min": 1471, "stdev": 14.46},
222             "group_name:test_name": 42},
223             "revision": 1234,
224             "key": "value"})
225
226     def test_run_with_upload_json(self):
227         runner = self.create_runner(args=['--output-json-path=/mock-checkout/output.json',
228             '--test-results-server', 'some.host', '--platform', 'platform1', '--builder-name', 'builder1', '--build-number', '123'])
229         upload_json_is_called = [False]
230         upload_json_returns_true = True
231
232         def mock_upload_json(hostname, json_path):
233             self.assertEqual(hostname, 'some.host')
234             self.assertEqual(json_path, '/mock-checkout/output.json')
235             upload_json_is_called[0] = True
236             return upload_json_returns_true
237
238         runner._upload_json = mock_upload_json
239         runner._host.filesystem.files['/mock-checkout/source.json'] = '{"key": "value"}'
240         runner._host.filesystem.files[runner._base_path + '/inspector/pass.html'] = True
241         runner._host.filesystem.files[runner._base_path + '/Bindings/event-target-wrapper.html'] = True
242         runner._timestamp = 123456789
243         self.assertEqual(runner.run(), 0)
244         self.assertEqual(upload_json_is_called[0], True)
245         generated_json = json.loads(runner._host.filesystem.files['/mock-checkout/output.json'])
246         self.assertEqual(generated_json['platform'], 'platform1')
247         self.assertEqual(generated_json['builder-name'], 'builder1')
248         self.assertEqual(generated_json['build-number'], 123)
249         upload_json_returns_true = False
250         self.assertEqual(runner.run(), -3)
251
252     def test_upload_json(self):
253         regular_output = array_stream.ArrayStream()
254         runner = self.create_runner(regular_output=regular_output)
255         runner._host.filesystem.files['/mock-checkout/some.json'] = 'some content'
256
257         called = []
258         upload_single_text_file_throws = False
259         upload_single_text_file_return_value = StringIO.StringIO('OK')
260
261         class MockFileUploader:
262             def __init__(mock, url, timeout):
263                 self.assertEqual(url, 'https://some.host/api/test/report')
264                 self.assertTrue(isinstance(timeout, int) and timeout)
265                 called.append('FileUploader')
266
267             def upload_single_text_file(mock, filesystem, content_type, filename):
268                 self.assertEqual(filesystem, runner._host.filesystem)
269                 self.assertEqual(content_type, 'application/json')
270                 self.assertEqual(filename, 'some.json')
271                 called.append('upload_single_text_file')
272                 if upload_single_text_file_throws:
273                     raise "Some exception"
274                 return upload_single_text_file_return_value
275
276         runner._upload_json('some.host', 'some.json', MockFileUploader)
277         self.assertEqual(called, ['FileUploader', 'upload_single_text_file'])
278
279         output = OutputCapture()
280         output.capture_output()
281         upload_single_text_file_return_value = StringIO.StringIO('Some error')
282         runner._upload_json('some.host', 'some.json', MockFileUploader)
283         _, _, logs = output.restore_output()
284         self.assertEqual(logs, 'Uploaded JSON but got a bad response:\nSome error\n')
285
286         # Throwing an exception upload_single_text_file shouldn't blow up _upload_json
287         called = []
288         upload_single_text_file_throws = True
289         runner._upload_json('some.host', 'some.json', MockFileUploader)
290         self.assertEqual(called, ['FileUploader', 'upload_single_text_file'])
291
292     def test_collect_tests(self):
293         runner = self.create_runner()
294         filename = runner._host.filesystem.join(runner._base_path, 'inspector', 'a_file.html')
295         runner._host.filesystem.files[filename] = 'a content'
296         tests = runner._collect_tests()
297         self.assertEqual(len(tests), 1)
298
299     def test_collect_tests_with_skipped_list(self):
300         runner = self.create_runner()
301
302         def add_file(dirname, filename, content=True):
303             dirname = runner._host.filesystem.join(runner._base_path, dirname) if dirname else runner._base_path
304             runner._host.filesystem.maybe_make_directory(dirname)
305             runner._host.filesystem.files[runner._host.filesystem.join(dirname, filename)] = content
306
307         add_file('inspector', 'test1.html')
308         add_file('inspector', 'unsupported_test1.html')
309         add_file('inspector', 'test2.html')
310         add_file('inspector/resources', 'resource_file.html')
311         add_file('unsupported', 'unsupported_test2.html')
312         runner._port.skipped_perf_tests = lambda: ['inspector/unsupported_test1.html', 'unsupported']
313         tests = [runner._port.relative_perf_test_filename(test) for test in runner._collect_tests()]
314         self.assertEqual(sorted(tests), ['inspector/test1.html', 'inspector/test2.html'])
315
316     def test_parse_args(self):
317         runner = self.create_runner()
318         options, args = PerfTestsRunner._parse_args([
319                 '--verbose',
320                 '--build-directory=folder42',
321                 '--platform=platform42',
322                 '--builder-name', 'webkit-mac-1',
323                 '--build-number=56',
324                 '--time-out-ms=42',
325                 '--output-json-path=a/output.json',
326                 '--source-json-path=a/source.json',
327                 '--test-results-server=somehost',
328                 '--debug', 'an_arg'])
329         self.assertEqual(options.build, True)
330         self.assertEqual(options.verbose, True)
331         self.assertEqual(options.help_printing, None)
332         self.assertEqual(options.build_directory, 'folder42')
333         self.assertEqual(options.platform, 'platform42')
334         self.assertEqual(options.builder_name, 'webkit-mac-1')
335         self.assertEqual(options.build_number, '56')
336         self.assertEqual(options.time_out_ms, '42')
337         self.assertEqual(options.configuration, 'Debug')
338         self.assertEqual(options.print_options, None)
339         self.assertEqual(options.output_json_path, 'a/output.json')
340         self.assertEqual(options.source_json_path, 'a/source.json')
341         self.assertEqual(options.test_results_server, 'somehost')
342
343
344 if __name__ == '__main__':
345     unittest.main()