run-api-tests: Upload test results
[WebKit-https.git] / Tools / Scripts / webkitpy / port / device_port.py
1 # Copyright (C) 2018-2019 Apple Inc. All rights reserved.
2 #
3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions
5 # are met:
6 # 1.  Redistributions of source code must retain the above copyright
7 #     notice, this list of conditions and the following disclaimer.
8 # 2.  Redistributions in binary form must reproduce the above copyright
9 #     notice, this list of conditions and the following disclaimer in the
10 #     documentation and/or other materials provided with the distribution.
11 #
12 # THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND
13 # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
14 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
15 # DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR
16 # ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
17 # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
18 # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
19 # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
20 # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
21 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
22
23 import logging
24 import traceback
25
26 from webkitpy.common.version_name_map import VersionNameMap, PUBLIC_TABLE, INTERNAL_TABLE
27 from webkitpy.layout_tests.models.test_configuration import TestConfiguration
28 from webkitpy.port.darwin import DarwinPort
29 from webkitpy.port.simulator_process import SimulatorProcess
30 from webkitpy.results.upload import Upload
31 from webkitpy.xcode.device_type import DeviceType
32 from webkitpy.xcode.simulated_device import DeviceRequest, SimulatedDeviceManager
33
34
35 _log = logging.getLogger(__name__)
36
37
38 class DevicePort(DarwinPort):
39
40     DEVICE_MANAGER = None
41     NO_DEVICE_MANAGER = 'No device manager found for port'
42
43     def __init__(self, *args, **kwargs):
44         super(DevicePort, self).__init__(*args, **kwargs)
45         self._test_runner_process_constructor = SimulatorProcess
46         self._printing_cmd_line = False
47
48     def driver_cmd_line_for_logging(self):
49         # Avoid creating/connecting to devices just for command line logging.
50         self._printing_cmd_line = True
51         result = super(DevicePort, self).driver_cmd_line_for_logging()
52         self._printing_cmd_line = False
53         return result
54
55     def _generate_all_test_configurations(self):
56         configurations = []
57         for build_type in self.ALL_BUILD_TYPES:
58             for architecture in self.ARCHITECTURES:
59                 configurations.append(TestConfiguration(version=self.version_name(), architecture=architecture, build_type=build_type))
60         return configurations
61
62     def child_processes(self):
63         return int(self.get_option('child_processes'))
64
65     def driver_name(self):
66         if self.get_option('driver_name'):
67             return self.get_option('driver_name')
68         if self.get_option('webkit_test_runner'):
69             return 'WebKitTestRunnerApp.app'
70         return 'DumpRenderTree.app'
71
72     # A device is the target host for a specific worker number.
73     def target_host(self, worker_number=None):
74         if self._printing_cmd_line or worker_number is None:
75             return self.host
76         if self.DEVICE_MANAGER is None:
77             raise RuntimeError('No device manager for specified port')
78         if self.DEVICE_MANAGER.INITIALIZED_DEVICES is None:
79             raise RuntimeError('No initialized devices for testing')
80         return self.DEVICE_MANAGER.INITIALIZED_DEVICES[worker_number]
81
82     def devices(self):
83         if self.DEVICE_MANAGER is None:
84             return []
85         if self.DEVICE_MANAGER.INITIALIZED_DEVICES is None:
86             return []
87         return self.DEVICE_MANAGER.INITIALIZED_DEVICES
88
89     # Despite their names, these flags do not actually get passed all the way down to webkit-build.
90     def _build_driver_flags(self):
91         return ['--sdk', self.SDK] + (['ARCHS=%s' % self.architecture()] if self.architecture() else [])
92
93     def _install(self):
94         if not self.get_option('install'):
95             _log.debug('Skipping installation')
96             return
97
98         for i in xrange(self.child_processes()):
99             device = self.target_host(i)
100             _log.debug('Installing to {}'.format(device))
101             # Without passing DYLD_LIBRARY_PATH, libWebCoreTestSupport cannot be loaded and DRT/WKTR will crash pre-launch,
102             # leaving a crash log which will be picked up in results. DYLD_FRAMEWORK_PATH is needed to prevent an early crash.
103             if not device.install_app(self._path_to_driver(), {'DYLD_LIBRARY_PATH': self._build_path(), 'DYLD_FRAMEWORK_PATH': self._build_path()}):
104                 raise RuntimeError('Failed to install app {} on device {}'.format(self._path_to_driver(), device.udid))
105             if not device.install_dylibs(self._build_path()):
106                 raise RuntimeError('Failed to install dylibs at {} on device {}'.format(self._build_path(), device.udid))
107
108     def _device_type_with_version(self, device_type=None):
109         device_type = device_type if device_type else self.DEVICE_TYPE
110         return DeviceType(
111             hardware_family=device_type.hardware_family,
112             hardware_type=device_type.hardware_type,
113             software_version=self.device_version(),
114             software_variant=device_type.software_variant,
115         )
116
117     def default_child_processes(self, device_type=None):
118         if not self.DEVICE_MANAGER:
119             raise RuntimeError(self.NO_DEVICE_MANAGER)
120
121         device_type = self._device_type_with_version(device_type)
122         if device_type not in self.DEVICE_TYPE:
123             return 0
124
125         if self.get_option('force'):
126             device_type.hardware_family = None
127             device_type.hardware_type = None
128
129         return self.DEVICE_MANAGER.device_count_for_type(
130             self._device_type_with_version(device_type),
131             host=self.host,
132             use_booted_simulator=not self.get_option('dedicated_simulators', False),
133         )
134
135     def max_child_processes(self, device_type=None):
136         result = self.default_child_processes(device_type=device_type)
137         if result and self.DEVICE_MANAGER == SimulatedDeviceManager:
138             return super(DevicePort, self).max_child_processes(device_type=None)
139         return result
140
141     def supported_device_types(self):
142         types = set()
143         for device in self.DEVICE_MANAGER.available_devices(host=self.host):
144             if self.DEVICE_MANAGER == SimulatedDeviceManager and not device.platform_device.is_booted_or_booting():
145                 continue
146             if device.device_type in self.DEVICE_TYPE:
147                 types.add(device.device_type)
148         if types and not self.get_option('dedicated_simulators', False):
149
150             def sorted_by_default_device_type(type):
151                 try:
152                     return self.DEFAULT_DEVICE_TYPES.index(type)
153                 except ValueError:
154                     return len(self.DEFAULT_DEVICE_TYPES)
155
156             return sorted(types, key=sorted_by_default_device_type)
157
158         return self.DEFAULT_DEVICE_TYPES or [self.DEVICE_TYPE]
159
160     def setup_test_run(self, device_type=None):
161         if not self.DEVICE_MANAGER:
162             raise RuntimeError(self.NO_DEVICE_MANAGER)
163
164         device_type = self._device_type_with_version(device_type)
165         _log.debug('\nCreating devices for {}'.format(device_type))
166
167         request = DeviceRequest(
168             device_type,
169             use_booted_simulator=not self.get_option('dedicated_simulators', False),
170             use_existing_simulator=False,
171             allow_incomplete_match=self.get_option('force'),
172         )
173         self.DEVICE_MANAGER.initialize_devices(
174             [request] * self.child_processes(),
175             self.host,
176             layout_test_dir=self.layout_tests_dir(),
177             pin=self.get_option('pin', None),
178             use_nfs=self.get_option('use_nfs', True),
179             reboot=self.get_option('reboot', False),
180         )
181
182         if not self.devices():
183             raise RuntimeError('No devices are available for testing')
184         if len(self.DEVICE_MANAGER.INITIALIZED_DEVICES) < self.child_processes():
185             raise RuntimeError('To few connected devices for {} processes'.format(self.child_processes()))
186
187         self._install()
188
189         for i in xrange(self.child_processes()):
190             host = self.target_host(i)
191             host.prepare_for_testing(
192                 self.ports_to_forward(),
193                 self.app_identifier_from_bundle(self._path_to_driver()),
194                 self.layout_tests_dir(),
195             )
196             self._crash_logs_to_skip_for_host[host] = host.filesystem.files_under(self.path_to_crash_logs())
197
198     def clean_up_test_run(self):
199         super(DevicePort, self).clean_up_test_run()
200
201         # Best effort to let every device teardown before throwing any exceptions here.
202         # Failure to teardown devices can leave things in a bad state.
203         exception_list = []
204         for i in xrange(self.child_processes()):
205             device = self.target_host(i)
206             if not device:
207                 continue
208             try:
209                 device.finished_testing()
210             except BaseException as e:
211                 trace = traceback.format_exc()
212                 if isinstance(e, Exception):
213                     exception_list.append([e, trace])
214                 else:
215                     exception_list.append([Exception('Exception while tearing down {}'.format(device)), trace])
216
217         if len(exception_list) == 1:
218             raise
219         if len(exception_list) > 1:
220             print('\n')
221             for exception in exception_list:
222                 _log.error('{} raised: {}'.format(exception[0].__class__.__name__, exception[0]))
223                 _log.error(exception[1])
224                 _log.error('--------------------------------------------------')
225
226             raise RuntimeError('Multiple failures when teardown devices')
227
228     def did_spawn_worker(self, worker_number):
229         super(DevicePort, self).did_spawn_worker(worker_number)
230
231         self.target_host(worker_number).release_worker_resources()
232
233     def setup_environ_for_server(self, server_name=None):
234         env = super(DevicePort, self).setup_environ_for_server(server_name)
235         if server_name == self.driver_name() and self.get_option('guard_malloc'):
236             self._append_value_colon_separated(env, 'DYLD_INSERT_LIBRARIES', '/usr/lib/libgmalloc.dylib')
237             self._append_value_colon_separated(env, '__XPC_DYLD_INSERT_LIBRARIES', '/usr/lib/libgmalloc.dylib')
238         env['XML_CATALOG_FILES'] = ''  # work around missing /etc/catalog <rdar://problem/4292995>
239         return env
240
241     def device_version(self):
242         raise NotImplementedError
243
244     def configuration_for_upload(self, host=None):
245         configuration = self.test_configuration()
246
247         device_type = host.device_type if host else self.DEVICE_TYPE
248         model = device_type.hardware_family
249         if model and device_type.hardware_type:
250             model += ' {}'.format(device_type.hardware_type)
251
252         version = self.device_version()
253         version_name = None
254         for table in [INTERNAL_TABLE, PUBLIC_TABLE]:
255             version_name = VersionNameMap.map(self.host.platform).to_name(version, platform=device_type.software_variant.lower(), table=table)
256             if version_name:
257                 break
258
259         return Upload.create_configuration(
260             platform=device_type.software_variant.lower(),
261             is_simulator=self.DEVICE_MANAGER == SimulatedDeviceManager,
262             version=str(version),
263             version_name=version_name,
264             architecture=configuration.architecture,
265             style='guard-malloc' if self.get_option('guard_malloc') else configuration.build_type,
266             model=model,
267             sdk=host.build_version if host else None,
268         )