run-api-tests: Upload test results
[WebKit-https.git] / Tools / Scripts / webkitpy / api_tests / manager.py
1 # Copyright (C) 2018-2019 Apple Inc. All rights reserved.
2 #
3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions
5 # are met:
6 # 1.  Redistributions of source code must retain the above copyright
7 #    notice, this list of conditions and the following disclaimer.
8 # 2.  Redistributions in binary form must reproduce the above copyright
9 #    notice, this list of conditions and the following disclaimer in the
10 #    documentation and/or other materials provided with the distribution.
11 #
12 # THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND ANY
13 # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
14 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
15 # DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR ANY
16 # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
17 # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
18 # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
19 # ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
20 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
21 # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
22
23 import json
24 import logging
25 import time
26
27 from webkitpy.api_tests.runner import Runner
28 from webkitpy.common.system.executive import ScriptError
29 from webkitpy.results.upload import Upload
30
31 from webkitpy.xcode.simulated_device import DeviceRequest, SimulatedDeviceManager
32
33 _log = logging.getLogger(__name__)
34
35
36 class Manager(object):
37     """A class for managing running API and WTF tests
38     """
39
40     SUCCESS = 0
41     FAILED_BUILD_CHECK = 1
42     FAILED_COLLECT_TESTS = 2
43     FAILED_TESTS = 3
44     FAILED_UPLOAD = 4
45
46     def __init__(self, port, options, stream):
47         self._port = port
48         self.host = port.host
49         self._options = options
50         self._stream = stream
51
52     @staticmethod
53     def _test_list_from_output(output, prefix=''):
54         result = []
55         current_test_suite = None
56         for line in output.split('\n'):
57             striped_line = line.lstrip().rstrip()
58             if not striped_line:
59                 continue
60
61             if striped_line[-1] == '.':
62                 current_test_suite = striped_line[:-1]
63             else:
64                 striped_line = striped_line.lstrip()
65                 if ' ' in striped_line:
66                     continue
67                 val = '{}{}.{}'.format(prefix, current_test_suite, striped_line)
68                 if val not in result:
69                     result.append(val)
70         return result
71
72     @staticmethod
73     def _find_test_subset(superset, arg_filter):
74         result = []
75         for arg in arg_filter:
76             split_arg = arg.split('.')
77             for test in superset:
78                 # Might match <binary>.<suite>.<test> or just <suite>.<test>
79                 split_test = test.split('.')
80                 if len(split_arg) == 1:
81                     if test not in result and (arg == split_test[0] or arg == split_test[1]):
82                         result.append(test)
83                 elif len(split_arg) == 2:
84                     if test not in result and (split_arg == split_test[0:2] or split_arg == split_test[1:3]):
85                         result.append(test)
86                 else:
87                     if arg == test and test not in result:
88                         result.append(test)
89         return result
90
91     def _collect_tests(self, args):
92         available_tests = []
93         specified_binaries = self._binaries_for_arguments(args)
94         for canonicalized_binary, path in self._port.path_to_api_test_binaries().iteritems():
95             if canonicalized_binary not in specified_binaries:
96                 continue
97             try:
98                 output = self.host.executive.run_command(
99                     Runner.command_for_port(self._port, [path, '--gtest_list_tests']),
100                     env=self._port.environment_for_api_tests())
101                 available_tests += Manager._test_list_from_output(output, '{}.'.format(canonicalized_binary))
102             except ScriptError:
103                 _log.error('Failed to list {} tests'.format(canonicalized_binary))
104                 raise
105
106         if len(args) == 0:
107             return sorted(available_tests)
108         return sorted(Manager._find_test_subset(available_tests, args))
109
110     @staticmethod
111     def _print_test_result(stream, test_name, output):
112         stream.writeln('    {}'.format(test_name))
113         has_output = False
114         for line in output.splitlines():
115             stream.writeln('        {}'.format(line))
116             has_output = True
117         if has_output:
118             stream.writeln('')
119         return not has_output
120
121     def _print_tests_result_with_status(self, status, runner):
122         mapping = runner.result_map_by_status(status)
123         if mapping:
124             self._stream.writeln(runner.NAME_FOR_STATUS[status])
125             self._stream.writeln('')
126             need_newline = False
127             for test, output in mapping.iteritems():
128                 need_newline = Manager._print_test_result(self._stream, test, output)
129             if need_newline:
130                 self._stream.writeln('')
131
132     def _initialize_devices(self):
133         if 'simulator' in self._port.port_name:
134             SimulatedDeviceManager.initialize_devices(DeviceRequest(self._port.DEVICE_TYPE, allow_incomplete_match=True), self.host, simulator_ui=False)
135         elif 'device' in self._port.port_name:
136             raise RuntimeError('Running api tests on {} is not supported'.format(self._port.port_name))
137
138     def _binaries_for_arguments(self, args):
139         if self._port.get_option('api_binary'):
140             return self._port.get_option('api_binary')
141
142         binaries = []
143         for arg in args:
144             candidate_binary = arg.split('.')[0]
145             if candidate_binary in binaries:
146                 continue
147             if candidate_binary in self._port.path_to_api_test_binaries():
148                 binaries.append(candidate_binary)
149             else:
150                 # If the user specifies a test-name without a binary, we need to search both binaries
151                 return self._port.path_to_api_test_binaries().keys()
152         return binaries or self._port.path_to_api_test_binaries().keys()
153
154     def run(self, args, json_output=None):
155         if json_output:
156             json_output = self.host.filesystem.abspath(json_output)
157             if not self.host.filesystem.isdir(self.host.filesystem.dirname(json_output)) or self.host.filesystem.isdir(json_output):
158                 raise RuntimeError('Cannot write to {}'.format(json_output))
159
160         start_time = time.time()
161
162         self._stream.write_update('Checking build ...')
163         if not self._port.check_api_test_build(self._binaries_for_arguments(args)):
164             _log.error('Build check failed')
165             return Manager.FAILED_BUILD_CHECK
166
167         self._initialize_devices()
168
169         self._stream.write_update('Collecting tests ...')
170         try:
171             test_names = self._collect_tests(args)
172         except ScriptError:
173             self._stream.writeln('Failed to collect tests')
174             return Manager.FAILED_COLLECT_TESTS
175         self._stream.write_update('Found {} tests'.format(len(test_names)))
176         if len(test_names) == 0:
177             self._stream.writeln('No tests found')
178             return Manager.FAILED_COLLECT_TESTS
179
180         if self._port.get_option('dump'):
181             for test in test_names:
182                 self._stream.writeln(test)
183             return Manager.SUCCESS
184
185         try:
186             _log.info('Running tests')
187             runner = Runner(self._port, self._stream)
188             runner.run(test_names, int(self._options.child_processes) if self._options.child_processes else self._port.default_child_processes())
189         except KeyboardInterrupt:
190             # If we receive a KeyboardInterrupt, print results.
191             self._stream.writeln('')
192
193         end_time = time.time()
194
195         successful = runner.result_map_by_status(runner.STATUS_PASSED)
196         disabled = len(runner.result_map_by_status(runner.STATUS_DISABLED))
197         _log.info('Ran {} tests of {} with {} successful'.format(len(runner.results) - disabled, len(test_names), len(successful)))
198
199         result_dictionary = {
200             'Skipped': [],
201             'Failed': [],
202             'Crashed': [],
203             'Timedout': [],
204         }
205
206         self._stream.writeln('-' * 30)
207         result = Manager.SUCCESS
208         if len(successful) + disabled == len(test_names):
209             self._stream.writeln('All tests successfully passed!')
210             if json_output:
211                 self.host.filesystem.write_text_file(json_output, json.dumps(result_dictionary, indent=4))
212         else:
213             self._stream.writeln('Test suite failed')
214             self._stream.writeln('')
215
216             skipped = []
217             for test in test_names:
218                 if test not in runner.results:
219                     skipped.append(test)
220                     result_dictionary['Skipped'].append({'name': test, 'output': None})
221             if skipped:
222                 self._stream.writeln('Skipped {} tests'.format(len(skipped)))
223                 self._stream.writeln('')
224                 if self._options.verbose:
225                     for test in skipped:
226                         self._stream.writeln('    {}'.format(test))
227
228             self._print_tests_result_with_status(runner.STATUS_FAILED, runner)
229             self._print_tests_result_with_status(runner.STATUS_CRASHED, runner)
230             self._print_tests_result_with_status(runner.STATUS_TIMEOUT, runner)
231
232             for test, result in runner.results.iteritems():
233                 status_to_string = {
234                     runner.STATUS_FAILED: 'Failed',
235                     runner.STATUS_CRASHED: 'Crashed',
236                     runner.STATUS_TIMEOUT: 'Timedout',
237                 }.get(result[0])
238                 if not status_to_string:
239                     continue
240                 result_dictionary[status_to_string].append({'name': test, 'output': result[1]})
241
242             if json_output:
243                 self.host.filesystem.write_text_file(json_output, json.dumps(result_dictionary, indent=4))
244
245             result = Manager.FAILED_TESTS
246
247         if self._options.report_urls:
248             self._stream.writeln('\n')
249             self._stream.write_update('Preparing upload data ...')
250
251             status_to_test_result = {
252                 runner.STATUS_PASSED: None,
253                 runner.STATUS_FAILED: Upload.Expectations.FAIL,
254                 runner.STATUS_CRASHED: Upload.Expectations.CRASH,
255                 runner.STATUS_TIMEOUT: Upload.Expectations.TIMEOUT,
256             }
257             upload = Upload(
258                 suite='api-tests',
259                 configuration=self._port.configuration_for_upload(self._port.target_host(0)),
260                 details=Upload.create_details(options=self._options),
261                 commits=self._port.commits_for_upload(),
262                 run_stats=Upload.create_run_stats(
263                     start_time=start_time,
264                     end_time=end_time,
265                     tests_skipped=len(result_dictionary['Skipped']),
266                 ),
267                 results={test: Upload.create_test_result(actual=status_to_test_result[result[0]])
268                          for test, result in runner.results.iteritems() if result[0] in status_to_test_result},
269             )
270             for url in self._options.report_urls:
271                 self._stream.write_update('Uploading to {} ...'.format(url))
272                 if not upload.upload(url, log_line_func=self._stream.writeln):
273                     result = Manager.FAILED_UPLOAD
274             self._stream.writeln('Uploads completed!')
275
276         return result