0996c8e23c9fda738043a322d2d765727f59be72
[WebKit-https.git] / Tools / Scripts / webkitpy / performance_tests / perftestsrunner_unittest.py
1 #!/usr/bin/python
2 # Copyright (C) 2011 Google Inc. All rights reserved.
3 #
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
6 # met:
7 #
8 #     * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 #     * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
13 # distribution.
14 #     * Neither the name of Google Inc. nor the names of its
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
17 #
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30 """Unit tests for run_perf_tests."""
31
32 import StringIO
33 import json
34 import unittest
35
36 from webkitpy.common import array_stream
37 from webkitpy.common.host_mock import MockHost
38 from webkitpy.common.system.filesystem_mock import MockFileSystem
39 from webkitpy.common.system.outputcapture import OutputCapture
40 from webkitpy.layout_tests.port.driver import DriverInput, DriverOutput
41 from webkitpy.layout_tests.port.test import TestPort
42 from webkitpy.layout_tests.views import printing
43 from webkitpy.performance_tests.perftestsrunner import PerfTestsRunner
44
45
46 class MainTest(unittest.TestCase):
47     class TestDriver:
48         def run_test(self, driver_input):
49             text = ''
50             timeout = False
51             crash = False
52             if driver_input.test_name.endswith('pass.html'):
53                 text = 'RESULT group_name: test_name= 42 ms'
54             elif driver_input.test_name.endswith('timeout.html'):
55                 timeout = True
56             elif driver_input.test_name.endswith('failed.html'):
57                 text = None
58             elif driver_input.test_name.endswith('tonguey.html'):
59                 text = 'we are not expecting an output from perf tests but RESULT blablabla'
60             elif driver_input.test_name.endswith('crash.html'):
61                 crash = True
62             elif driver_input.test_name.endswith('event-target-wrapper.html'):
63                 text = """Running 20 times
64 Ignoring warm-up run (1502)
65 1504
66 1505
67 1510
68 1504
69 1507
70 1509
71 1510
72 1487
73 1488
74 1472
75 1472
76 1488
77 1473
78 1472
79 1475
80 1487
81 1486
82 1486
83 1475
84 1471
85
86 avg 1489.05
87 median 1487
88 stdev 14.46
89 min 1471
90 max 1510
91 """
92             elif driver_input.test_name.endswith('some-parser.html'):
93                 text = """Running 20 times
94 Ignoring warm-up run (1115)
95
96 avg 1100
97 median 1101
98 stdev 11
99 min 1080
100 max 1120
101 """
102             return DriverOutput(text, '', '', '', crash=crash, timeout=timeout)
103
104         def stop(self):
105             """do nothing"""
106
107     def create_runner(self, buildbot_output=None, args=[], regular_output=None):
108         buildbot_output = buildbot_output or array_stream.ArrayStream()
109         regular_output = regular_output or array_stream.ArrayStream()
110
111         options, parsed_args = PerfTestsRunner._parse_args(args)
112         test_port = TestPort(host=MockHost(), options=options)
113         test_port.create_driver = lambda worker_number=None, no_timeout=False: MainTest.TestDriver()
114
115         runner = PerfTestsRunner(regular_output, buildbot_output, args=args, port=test_port)
116         runner._host.filesystem.maybe_make_directory(runner._base_path, 'inspector')
117         runner._host.filesystem.maybe_make_directory(runner._base_path, 'Bindings')
118         runner._host.filesystem.maybe_make_directory(runner._base_path, 'Parser')
119         return runner
120
121     def run_test(self, test_name):
122         runner = self.create_runner()
123         driver = MainTest.TestDriver()
124         return runner._run_single_test(test_name, driver, is_chromium_style=True)
125
126     def test_run_passing_test(self):
127         test_failed, driver_need_restart = self.run_test('pass.html')
128         self.assertFalse(test_failed)
129         self.assertFalse(driver_need_restart)
130
131     def test_run_silent_test(self):
132         test_failed, driver_need_restart = self.run_test('silent.html')
133         self.assertTrue(test_failed)
134         self.assertFalse(driver_need_restart)
135
136     def test_run_failed_test(self):
137         test_failed, driver_need_restart = self.run_test('failed.html')
138         self.assertTrue(test_failed)
139         self.assertFalse(driver_need_restart)
140
141     def test_run_tonguey_test(self):
142         test_failed, driver_need_restart = self.run_test('tonguey.html')
143         self.assertTrue(test_failed)
144         self.assertFalse(driver_need_restart)
145
146     def test_run_timeout_test(self):
147         test_failed, driver_need_restart = self.run_test('timeout.html')
148         self.assertTrue(test_failed)
149         self.assertTrue(driver_need_restart)
150
151     def test_run_crash_test(self):
152         test_failed, driver_need_restart = self.run_test('crash.html')
153         self.assertTrue(test_failed)
154         self.assertTrue(driver_need_restart)
155
156     def test_run_test_set(self):
157         buildbot_output = array_stream.ArrayStream()
158         runner = self.create_runner(buildbot_output)
159         dirname = runner._base_path + '/inspector/'
160         tests = [dirname + 'pass.html', dirname + 'silent.html', dirname + 'failed.html',
161             dirname + 'tonguey.html', dirname + 'timeout.html', dirname + 'crash.html']
162         unexpected_result_count = runner._run_tests_set(tests, runner._port)
163         self.assertEqual(unexpected_result_count, len(tests) - 1)
164         self.assertEqual(len(buildbot_output.get()), 1)
165         self.assertEqual(buildbot_output.get()[0], 'RESULT group_name: test_name= 42 ms\n')
166
167     def test_run_test_set_for_parser_tests(self):
168         buildbot_output = array_stream.ArrayStream()
169         runner = self.create_runner(buildbot_output)
170         tests = [runner._base_path + '/Bindings/event-target-wrapper.html', runner._base_path + '/Parser/some-parser.html']
171         unexpected_result_count = runner._run_tests_set(tests, runner._port)
172         self.assertEqual(unexpected_result_count, 0)
173         self.assertEqual(buildbot_output.get()[0], 'RESULT Bindings: event-target-wrapper= 1489.05 ms\n')
174         self.assertEqual(buildbot_output.get()[1], 'median= 1487 ms, stdev= 14.46 ms, min= 1471 ms, max= 1510 ms\n')
175         self.assertEqual(buildbot_output.get()[2], 'RESULT Parser: some-parser= 1100 ms\n')
176         self.assertEqual(buildbot_output.get()[3], 'median= 1101 ms, stdev= 11 ms, min= 1080 ms, max= 1120 ms\n')
177
178     def test_run_test_set_with_json_output(self):
179         buildbot_output = array_stream.ArrayStream()
180         runner = self.create_runner(buildbot_output, args=['--output-json-path=/mock-checkout/output.json'])
181         runner._host.filesystem.files[runner._base_path + '/inspector/pass.html'] = True
182         runner._host.filesystem.files[runner._base_path + '/Bindings/event-target-wrapper.html'] = True
183         runner._timestamp = 123456789
184         self.assertEqual(runner.run(), 0)
185         self.assertEqual(len(buildbot_output.get()), 3)
186         self.assertEqual(buildbot_output.get()[0], 'RESULT Bindings: event-target-wrapper= 1489.05 ms\n')
187         self.assertEqual(buildbot_output.get()[1], 'median= 1487 ms, stdev= 14.46 ms, min= 1471 ms, max= 1510 ms\n')
188         self.assertEqual(buildbot_output.get()[2], 'RESULT group_name: test_name= 42 ms\n')
189
190         self.assertEqual(json.loads(runner._host.filesystem.files['/mock-checkout/output.json']), {
191             "timestamp": 123456789, "results":
192             {"event-target-wrapper": {"max": "1510", "avg": "1489.05", "median": "1487", "min": "1471", "stdev": "14.46"},
193             "group_name:test_name": 42},
194             "revision": 1234})
195
196     def test_run_test_set_with_json_source(self):
197         buildbot_output = array_stream.ArrayStream()
198         runner = self.create_runner(buildbot_output, args=['--output-json-path=/mock-checkout/output.json',
199             '--source-json-path=/mock-checkout/source.json'])
200         runner._host.filesystem.files['/mock-checkout/source.json'] = '{"key": "value"}'
201         runner._host.filesystem.files[runner._base_path + '/inspector/pass.html'] = True
202         runner._host.filesystem.files[runner._base_path + '/Bindings/event-target-wrapper.html'] = True
203         runner._timestamp = 123456789
204         self.assertEqual(runner.run(), 0)
205         self.assertEqual(len(buildbot_output.get()), 3)
206         self.assertEqual(buildbot_output.get()[0], 'RESULT Bindings: event-target-wrapper= 1489.05 ms\n')
207         self.assertEqual(buildbot_output.get()[1], 'median= 1487 ms, stdev= 14.46 ms, min= 1471 ms, max= 1510 ms\n')
208         self.assertEqual(buildbot_output.get()[2], 'RESULT group_name: test_name= 42 ms\n')
209
210         self.assertEqual(json.loads(runner._host.filesystem.files['/mock-checkout/output.json']), {
211             "timestamp": 123456789, "results":
212             {"event-target-wrapper": {"max": "1510", "avg": "1489.05", "median": "1487", "min": "1471", "stdev": "14.46"},
213             "group_name:test_name": 42},
214             "revision": 1234,
215             "key": "value"})
216
217     def test_run_with_upload_json(self):
218         runner = self.create_runner(args=['--output-json-path=/mock-checkout/output.json',
219             '--test-results-server', 'some.host', '--platform', 'platform1', '--builder-name', 'builder1', '--build-number', '123'])
220         upload_json_is_called = [False]
221         upload_json_returns_true = True
222
223         def mock_upload_json(hostname, json_path):
224             self.assertEqual(hostname, 'some.host')
225             self.assertEqual(json_path, '/mock-checkout/output.json')
226             upload_json_is_called[0] = True
227             return upload_json_returns_true
228
229         runner._upload_json = mock_upload_json
230         runner._host.filesystem.files['/mock-checkout/source.json'] = '{"key": "value"}'
231         runner._host.filesystem.files[runner._base_path + '/inspector/pass.html'] = True
232         runner._host.filesystem.files[runner._base_path + '/Bindings/event-target-wrapper.html'] = True
233         runner._timestamp = 123456789
234         self.assertEqual(runner.run(), 0)
235         self.assertEqual(upload_json_is_called[0], True)
236         generated_json = json.loads(runner._host.filesystem.files['/mock-checkout/output.json'])
237         self.assertEqual(generated_json['platform'], 'platform1')
238         self.assertEqual(generated_json['builder-name'], 'builder1')
239         self.assertEqual(generated_json['build-number'], 123)
240         upload_json_returns_true = False
241         self.assertEqual(runner.run(), -3)
242
243     def test_upload_json(self):
244         regular_output = array_stream.ArrayStream()
245         runner = self.create_runner(regular_output=regular_output)
246         runner._host.filesystem.files['/mock-checkout/some.json'] = 'some content'
247
248         called = []
249         upload_single_text_file_throws = False
250         upload_single_text_file_return_value = StringIO.StringIO('OK')
251
252         class MockFileUploader:
253             def __init__(mock, url, timeout):
254                 self.assertEqual(url, 'https://some.host/api/test/report')
255                 self.assertTrue(isinstance(timeout, int) and timeout)
256                 called.append('FileUploader')
257
258             def upload_single_text_file(mock, filesystem, content_type, filename):
259                 self.assertEqual(filesystem, runner._host.filesystem)
260                 self.assertEqual(content_type, 'application/json')
261                 self.assertEqual(filename, 'some.json')
262                 called.append('upload_single_text_file')
263                 if upload_single_text_file_throws:
264                     raise "Some exception"
265                 return upload_single_text_file_return_value
266
267         runner._upload_json('some.host', 'some.json', MockFileUploader)
268         self.assertEqual(called, ['FileUploader', 'upload_single_text_file'])
269
270         output = OutputCapture()
271         output.capture_output()
272         upload_single_text_file_return_value = StringIO.StringIO('Some error')
273         runner._upload_json('some.host', 'some.json', MockFileUploader)
274         _, _, logs = output.restore_output()
275         self.assertEqual(logs, 'Uploaded JSON but got a bad response:\nSome error\n')
276
277         # Throwing an exception upload_single_text_file shouldn't blow up _upload_json
278         called = []
279         upload_single_text_file_throws = True
280         runner._upload_json('some.host', 'some.json', MockFileUploader)
281         self.assertEqual(called, ['FileUploader', 'upload_single_text_file'])
282
283     def test_collect_tests(self):
284         runner = self.create_runner()
285         filename = runner._host.filesystem.join(runner._base_path, 'inspector', 'a_file.html')
286         runner._host.filesystem.files[filename] = 'a content'
287         tests = runner._collect_tests()
288         self.assertEqual(len(tests), 1)
289
290     def test_collect_tests_with_skipped_list(self):
291         runner = self.create_runner()
292
293         def add_file(dirname, filename, content=True):
294             dirname = runner._host.filesystem.join(runner._base_path, dirname) if dirname else runner._base_path
295             runner._host.filesystem.maybe_make_directory(dirname)
296             runner._host.filesystem.files[runner._host.filesystem.join(dirname, filename)] = content
297
298         add_file('inspector', 'test1.html')
299         add_file('inspector', 'unsupported_test1.html')
300         add_file('inspector', 'test2.html')
301         add_file('inspector/resources', 'resource_file.html')
302         add_file('unsupported', 'unsupported_test2.html')
303         runner._port.skipped_perf_tests = lambda: ['inspector/unsupported_test1.html', 'unsupported']
304         tests = [runner._port.relative_perf_test_filename(test) for test in runner._collect_tests()]
305         self.assertEqual(sorted(tests), ['inspector/test1.html', 'inspector/test2.html'])
306
307     def test_parse_args(self):
308         runner = self.create_runner()
309         options, args = PerfTestsRunner._parse_args([
310                 '--verbose',
311                 '--build-directory=folder42',
312                 '--platform=platform42',
313                 '--builder-name', 'webkit-mac-1',
314                 '--build-number=56',
315                 '--time-out-ms=42',
316                 '--output-json-path=a/output.json',
317                 '--source-json-path=a/source.json',
318                 '--test-results-server=somehost',
319                 '--debug', 'an_arg'])
320         self.assertEqual(options.build, True)
321         self.assertEqual(options.verbose, True)
322         self.assertEqual(options.help_printing, None)
323         self.assertEqual(options.build_directory, 'folder42')
324         self.assertEqual(options.platform, 'platform42')
325         self.assertEqual(options.builder_name, 'webkit-mac-1')
326         self.assertEqual(options.build_number, '56')
327         self.assertEqual(options.time_out_ms, '42')
328         self.assertEqual(options.configuration, 'Debug')
329         self.assertEqual(options.print_options, None)
330         self.assertEqual(options.output_json_path, 'a/output.json')
331         self.assertEqual(options.source_json_path, 'a/source.json')
332         self.assertEqual(options.test_results_server, 'somehost')
333
334
335 if __name__ == '__main__':
336     unittest.main()