Unreviewed, remove failure expectation for test that is now consistently passing.
[WebKit-https.git] / Tools / Scripts / webkitpy / performance_tests / perftestsrunner_unittest.py
1 # Copyright (C) 2012 Google Inc. All rights reserved.
2 # Copyright (C) 2020 Apple Inc. All rights reserved.
3 #
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
6 # met:
7 #
8 #     * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 #     * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
13 # distribution.
14 #     * Neither the name of Google Inc. nor the names of its
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
17 #
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30 """Unit tests for run_perf_tests."""
31
32 import json
33 import logging
34 import unittest
35
36 from webkitcorepy import BytesIO
37
38 from webkitpy.common.host_mock import MockHost
39 from webkitpy.port.test import TestPort
40 from webkitpy.performance_tests.perftestsrunner import PerfTestsRunner
41
42 from webkitcorepy import OutputCapture
43
44
45 class MainTest(unittest.TestCase):
46     def create_runner(self, args=[]):
47         options, parsed_args = PerfTestsRunner._parse_args(args)
48         test_port = TestPort(host=MockHost(), options=options)
49         runner = PerfTestsRunner(args=args, port=test_port)
50         runner._host.filesystem.maybe_make_directory(runner._base_path, 'inspector')
51         runner._host.filesystem.maybe_make_directory(runner._base_path, 'Bindings')
52         runner._host.filesystem.maybe_make_directory(runner._base_path, 'Parser')
53         return runner, test_port
54
55     def _add_file(self, runner, dirname, filename, content=True):
56         dirname = runner._host.filesystem.join(runner._base_path, dirname) if dirname else runner._base_path
57         runner._host.filesystem.maybe_make_directory(dirname)
58         runner._host.filesystem.files[runner._host.filesystem.join(dirname, filename)] = content
59
60     def test_drt_notimeout(self):
61         runner, port = self.create_runner()
62         self.assertEqual(runner._options.additional_drt_flag, ['--no-timeout'])
63
64     def test_collect_tests(self):
65         runner, port = self.create_runner()
66         self._add_file(runner, 'inspector', 'a_file.html', 'a content')
67         tests = runner._collect_tests()
68         self.assertEqual(len(tests), 1)
69
70     def _collect_tests_and_sort_test_name(self, runner):
71         return sorted([test.test_name() for test in runner._collect_tests()])
72
73     def test_collect_tests_with_multile_files(self):
74         runner, port = self.create_runner(args=['PerformanceTests/test1.html', 'test2.html'])
75
76         def add_file(filename):
77             port.host.filesystem.files[runner._host.filesystem.join(runner._base_path, filename)] = 'some content'
78
79         add_file('test1.html')
80         add_file('test2.html')
81         add_file('test3.html')
82         port.host.filesystem.chdir(runner._port.perf_tests_dir()[:runner._port.perf_tests_dir().rfind(runner._host.filesystem.sep)])
83         self.assertEqual(self._collect_tests_and_sort_test_name(runner), ['test1.html', 'test2.html'])
84
85     def test_collect_tests_with_index_html_and_resources(self):
86         runner, port = self.create_runner()
87         self._add_file(runner, 'Speedometer', 'index.html', 'test content')
88         self._add_file(runner, 'Speedometer/resources', 'resource.html', 'resource content')
89         tests = runner._collect_tests()
90         self.assertEqual(len(tests), 1)
91         self.assertEqual(tests[0].test_name(), 'Speedometer')
92
93     def test_collect_tests_with_skipped_list(self):
94         runner, port = self.create_runner()
95
96         self._add_file(runner, 'inspector', 'test1.html')
97         self._add_file(runner, 'inspector', 'unsupported_test1.html')
98         self._add_file(runner, 'inspector', 'test2.html')
99         self._add_file(runner, 'inspector/resources', 'resource_file.html')
100         self._add_file(runner, 'unsupported', 'unsupported_test2.html')
101         port.skipped_perf_tests = lambda: ['inspector/unsupported_test1.html', 'unsupported']
102         self.assertEqual(self._collect_tests_and_sort_test_name(runner), ['inspector/test1.html', 'inspector/test2.html'])
103
104     def test_collect_tests_with_skipped_list_and_files(self):
105         runner, port = self.create_runner(args=['Suite/Test1.html', 'Suite/SkippedTest1.html', 'SkippedSuite/Test1.html'])
106
107         self._add_file(runner, 'SkippedSuite', 'Test1.html')
108         self._add_file(runner, 'SkippedSuite', 'Test2.html')
109         self._add_file(runner, 'Suite', 'Test1.html')
110         self._add_file(runner, 'Suite', 'Test2.html')
111         self._add_file(runner, 'Suite', 'SkippedTest1.html')
112         self._add_file(runner, 'Suite', 'SkippedTest2.html')
113         port.skipped_perf_tests = lambda: ['Suite/SkippedTest1.html', 'Suite/SkippedTest1.html', 'SkippedSuite']
114         self.assertEqual(self._collect_tests_and_sort_test_name(runner),
115             ['SkippedSuite/Test1.html', 'Suite/SkippedTest1.html', 'Suite/Test1.html'])
116
117     def test_collect_tests_with_ignored_skipped_list(self):
118         runner, port = self.create_runner(args=['--force'])
119
120         self._add_file(runner, 'inspector', 'test1.html')
121         self._add_file(runner, 'inspector', 'unsupported_test1.html')
122         self._add_file(runner, 'inspector', 'test2.html')
123         self._add_file(runner, 'inspector/resources', 'resource_file.html')
124         self._add_file(runner, 'unsupported', 'unsupported_test2.html')
125         port.skipped_perf_tests = lambda: ['inspector/unsupported_test1.html', 'unsupported']
126         self.assertEqual(self._collect_tests_and_sort_test_name(runner), ['inspector/test1.html', 'inspector/test2.html', 'inspector/unsupported_test1.html', 'unsupported/unsupported_test2.html'])
127
128     def test_default_args(self):
129         runner, port = self.create_runner()
130         options, args = PerfTestsRunner._parse_args([])
131         self.assertTrue(options.build)
132         self.assertEqual(options.time_out_ms, 600 * 1000)
133         self.assertEqual(options.additional_drt_flag, [])
134         self.assertTrue(options.generate_results)
135         self.assertTrue(options.show_results)
136         self.assertTrue(options.use_skipped_list)
137         self.assertEqual(options.repeat, 1)
138         self.assertEqual(options.test_runner_count, -1)
139         self.assertEqual(options.no_timeout, False)
140
141     def test_parse_args(self):
142         runner, port = self.create_runner()
143         options, args = PerfTestsRunner._parse_args([
144                 '--build-directory=folder42',
145                 '--platform=platform42',
146                 '--builder-name', 'webkit-mac-1',
147                 '--build-number=56',
148                 '--time-out-ms=42',
149                 '--no-show-results',
150                 '--reset-results',
151                 '--output-json-path=a/output.json',
152                 '--worker-config-json-path=a/source.json',
153                 '--test-results-server=somehost',
154                 '--additional-drt-flag=--enable-threaded-parser',
155                 '--additional-drt-flag=--awesomesauce',
156                 '--repeat=5',
157                 '--test-runner-count=5',
158                 '--debug',
159                 '--no-timeout'])
160         self.assertTrue(options.build)
161         self.assertEqual(options.build_directory, 'folder42')
162         self.assertEqual(options.platform, 'platform42')
163         self.assertEqual(options.builder_name, 'webkit-mac-1')
164         self.assertEqual(options.build_number, '56')
165         self.assertEqual(options.time_out_ms, '42')
166         self.assertEqual(options.configuration, 'Debug')
167         self.assertFalse(options.show_results)
168         self.assertTrue(options.reset_results)
169         self.assertEqual(options.output_json_path, 'a/output.json')
170         self.assertEqual(options.worker_config_json_path, 'a/source.json')
171         self.assertEqual(options.test_results_server, 'somehost')
172         self.assertEqual(options.additional_drt_flag, ['--enable-threaded-parser', '--awesomesauce'])
173         self.assertEqual(options.repeat, 5)
174         self.assertEqual(options.test_runner_count, 5)
175         self.assertEqual(options.no_timeout, True)
176
177     def test_upload_json(self):
178         runner, port = self.create_runner()
179         port.host.filesystem.files['/mock-checkout/some.json'] = 'some content'
180
181         class MockFileUploader:
182             called = []
183             upload_single_text_file_throws = False
184             upload_single_text_file_return_value = None
185
186             @classmethod
187             def reset(cls):
188                 cls.called = []
189                 cls.upload_single_text_file_throws = False
190                 cls.upload_single_text_file_return_value = None
191
192             def __init__(mock, url, timeout):
193                 self.assertEqual(url, 'https://some.host/some/path')
194                 self.assertTrue(isinstance(timeout, int) and timeout)
195                 mock.called.append('FileUploader')
196
197             def upload_single_text_file(mock, filesystem, content_type, filename):
198                 self.assertEqual(filesystem, port.host.filesystem)
199                 self.assertEqual(content_type, 'application/json')
200                 self.assertEqual(filename, 'some.json')
201                 mock.called.append('upload_single_text_file')
202                 if mock.upload_single_text_file_throws:
203                     raise Exception
204                 return mock.upload_single_text_file_return_value
205
206         MockFileUploader.upload_single_text_file_return_value = BytesIO(b'OK')
207         self.assertTrue(runner._upload_json('https://some.host', 'some.json', '/some/path', MockFileUploader))
208         self.assertEqual(MockFileUploader.called, ['FileUploader', 'upload_single_text_file'])
209
210         MockFileUploader.reset()
211         MockFileUploader.upload_single_text_file_return_value = BytesIO(b'OK')
212         self.assertTrue(runner._upload_json('some.host', 'some.json', '/some/path', MockFileUploader))
213
214         MockFileUploader.reset()
215         MockFileUploader.upload_single_text_file_return_value = BytesIO(b'Some error')
216         with OutputCapture(level=logging.INFO) as captured:
217             self.assertFalse(runner._upload_json('https://some.host', 'some.json', '/some/path', MockFileUploader))
218         self.assertEqual(
219             captured.root.log.getvalue(),
220             'Uploaded JSON to https://some.host/some/path but got a bad response:\nSome error\n',
221         )
222
223         # Throwing an exception upload_single_text_file shouldn't blow up _upload_json
224         MockFileUploader.reset()
225         MockFileUploader.upload_single_text_file_throws = True
226         self.assertFalse(runner._upload_json('https://some.host', 'some.json', '/some/path', MockFileUploader))
227         self.assertEqual(MockFileUploader.called, ['FileUploader', 'upload_single_text_file'])
228
229         MockFileUploader.reset()
230         MockFileUploader.upload_single_text_file_return_value = BytesIO(b'{"status": "OK"}')
231         self.assertTrue(runner._upload_json('https://some.host', 'some.json', '/some/path', MockFileUploader))
232         self.assertEqual(MockFileUploader.called, ['FileUploader', 'upload_single_text_file'])
233
234         MockFileUploader.reset()
235         MockFileUploader.upload_single_text_file_return_value = BytesIO(b'{"status": "SomethingHasFailed", "failureStored": false}')
236         with OutputCapture(level=logging.INFO) as captured:
237             self.assertFalse(runner._upload_json('https://some.host', 'some.json', '/some/path', MockFileUploader))
238         serialized_json = json.dumps({'status': 'SomethingHasFailed', 'failureStored': False}, indent=4)
239         self.assertEqual(
240             captured.root.log.getvalue(),
241             'Uploaded JSON to https://some.host/some/path but got an error:\n{}\n'.format(serialized_json),
242         )