REGRESSION (r230998): Bot watcher's dashboard doesn't display number of API test...
[WebKit-https.git] / Tools / Scripts / webkitpy / api_tests / manager.py
1 # Copyright (C) 2018 Apple Inc. All rights reserved.
2 #
3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions
5 # are met:
6 # 1.  Redistributions of source code must retain the above copyright
7 #    notice, this list of conditions and the following disclaimer.
8 # 2.  Redistributions in binary form must reproduce the above copyright
9 #    notice, this list of conditions and the following disclaimer in the
10 #    documentation and/or other materials provided with the distribution.
11 #
12 # THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND ANY
13 # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
14 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
15 # DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR ANY
16 # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
17 # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
18 # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
19 # ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
20 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
21 # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
22
23 import logging
24 import os
25
26 from webkitpy.api_tests.runner import Runner
27 from webkitpy.common.system.executive import ScriptError
28 from webkitpy.xcode.device_type import DeviceType
29 from webkitpy.xcode.simulated_device import DeviceRequest, SimulatedDeviceManager
30
31 _log = logging.getLogger(__name__)
32
33
34 class Manager(object):
35     """A class for managing running API and WTF tests
36     """
37
38     SUCCESS = 0
39     FAILED_BUILD_CHECK = 1
40     FAILED_COLLECT_TESTS = 2
41     FAILED_TESTS = 3
42
43     def __init__(self, port, options, stream):
44         self._port = port
45         self.host = port.host
46         self._options = options
47         self._stream = stream
48
49     @staticmethod
50     def _test_list_from_output(output, prefix=''):
51         result = []
52         current_test_suite = None
53         for line in output.split('\n'):
54             striped_line = line.lstrip().rstrip()
55             if not striped_line:
56                 continue
57
58             if striped_line[-1] == '.':
59                 current_test_suite = striped_line[:-1]
60             else:
61                 striped_line = striped_line.lstrip()
62                 if ' ' in striped_line:
63                     continue
64                 val = '{}{}.{}'.format(prefix, current_test_suite, striped_line)
65                 if val not in result:
66                     result.append(val)
67         return result
68
69     @staticmethod
70     def _find_test_subset(superset, arg_filter):
71         result = []
72         for arg in arg_filter:
73             split_arg = arg.split('.')
74             for test in superset:
75                 # Might match <binary>.<suite>.<test> or just <suite>.<test>
76                 split_test = test.split('.')
77                 if len(split_arg) == 1:
78                     if test not in result and (arg == split_test[0] or arg == split_test[1]):
79                         result.append(test)
80                 elif len(split_arg) == 2:
81                     if test not in result and (split_arg == split_test[0:2] or split_arg == split_test[1:3]):
82                         result.append(test)
83                 else:
84                     if arg == test and test not in result:
85                         result.append(test)
86         return result
87
88     def _collect_tests(self, args):
89         available_tests = []
90         for binary in self._port.path_to_api_test_binaries():
91             stripped_name = os.path.splitext(os.path.basename(binary))[0]
92             try:
93                 output = self.host.executive.run_command(
94                     Runner.command_for_port(self._port, [binary, '--gtest_list_tests']),
95                     env=self._port.environment_for_api_tests())
96                 available_tests += Manager._test_list_from_output(output, '{}.'.format(stripped_name))
97             except ScriptError:
98                 _log.error('Failed to list {} tests'.format(stripped_name))
99                 raise
100
101         if len(args) == 0:
102             return sorted(available_tests)
103         return sorted(Manager._find_test_subset(available_tests, args))
104
105     @staticmethod
106     def _print_test_result(stream, test_name, output):
107         stream.writeln('    {}'.format(test_name))
108         has_output = False
109         for line in output.splitlines():
110             stream.writeln('        {}'.format(line))
111             has_output = True
112         if has_output:
113             stream.writeln('')
114         return not has_output
115
116     def _print_tests_result_with_status(self, status, runner):
117         mapping = runner.result_map_by_status(status)
118         if mapping:
119             self._stream.writeln(runner.NAME_FOR_STATUS[status])
120             self._stream.writeln('')
121             need_newline = False
122             for test, output in mapping.iteritems():
123                 need_newline = Manager._print_test_result(self._stream, test, output)
124             if need_newline:
125                 self._stream.writeln('')
126
127     def _initialize_devices(self):
128         if 'simulator' in self._port.port_name:
129             SimulatedDeviceManager.initialize_devices(DeviceRequest(DeviceType.from_string(self._port.DEFAULT_DEVICE_CLASS), allow_incomplete_match=True), self.host, simulator_ui=False)
130         elif 'device' in self._port.port_name:
131             raise RuntimeError('Running api tests on {} is not supported'.format(self._port.port_name))
132
133     def run(self, args):
134         self._stream.write_update('Checking build ...')
135         if not self._port.check_api_test_build():
136             _log.error('Build check failed')
137             return Manager.FAILED_BUILD_CHECK
138
139         self._initialize_devices()
140
141         self._stream.write_update('Collecting tests ...')
142         try:
143             test_names = self._collect_tests(args)
144         except ScriptError:
145             self._stream.writeln('Failed to collect tests')
146             return Manager.FAILED_COLLECT_TESTS
147         self._stream.write_update('Found {} tests'.format(len(test_names)))
148         if len(test_names) == 0:
149             self._stream.writeln('No tests found')
150             return Manager.FAILED_COLLECT_TESTS
151
152         if self._port.get_option('dump'):
153             for test in test_names:
154                 self._stream.writeln(test)
155             return Manager.SUCCESS
156
157         try:
158             _log.info('Running tests')
159             runner = Runner(self._port, self._stream)
160             runner.run(test_names, int(self._options.child_processes) if self._options.child_processes else self._port.default_child_processes())
161         except KeyboardInterrupt:
162             # If we receive a KeyboardInterrupt, print results.
163             self._stream.writeln('')
164
165         successful = runner.result_map_by_status(runner.STATUS_PASSED)
166         disabled = len(runner.result_map_by_status(runner.STATUS_DISABLED))
167         _log.info('Ran {} tests of {} with {} successful'.format(len(runner.results) - disabled, len(test_names), len(successful)))
168
169         self._stream.writeln('------------------------------')
170         if len(successful) + disabled == len(test_names):
171             self._stream.writeln('All tests successfully passed!')
172             return Manager.SUCCESS
173
174         self._stream.writeln('Test suite failed')
175         self._stream.writeln('')
176
177         skipped = []
178         for test in test_names:
179             if test not in runner.results:
180                 skipped.append(test)
181         if skipped:
182             self._stream.writeln('Skipped {} tests'.format(len(skipped)))
183             self._stream.writeln('')
184             if self._options.verbose:
185                 for test in skipped:
186                     self._stream.writeln('    {}'.format(test))
187
188         self._print_tests_result_with_status(runner.STATUS_FAILED, runner)
189         self._print_tests_result_with_status(runner.STATUS_CRASHED, runner)
190         self._print_tests_result_with_status(runner.STATUS_TIMEOUT, runner)
191
192         return Manager.FAILED_TESTS