webkitpy: Implement device type specific expected results (Part 2)
[WebKit-https.git] / Tools / Scripts / webkitpy / port / device_port.py
1 # Copyright (C) 2018 Apple Inc. All rights reserved.
2 #
3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions
5 # are met:
6 # 1.  Redistributions of source code must retain the above copyright
7 #     notice, this list of conditions and the following disclaimer.
8 # 2.  Redistributions in binary form must reproduce the above copyright
9 #     notice, this list of conditions and the following disclaimer in the
10 #     documentation and/or other materials provided with the distribution.
11 #
12 # THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND
13 # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
14 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
15 # DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR
16 # ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
17 # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
18 # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
19 # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
20 # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
21 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
22
23 import logging
24 import traceback
25
26 from webkitpy.layout_tests.models.test_configuration import TestConfiguration
27 from webkitpy.port.darwin import DarwinPort
28 from webkitpy.port.simulator_process import SimulatorProcess
29 from webkitpy.xcode.device_type import DeviceType
30 from webkitpy.xcode.simulated_device import DeviceRequest, SimulatedDeviceManager
31
32
33 _log = logging.getLogger(__name__)
34
35
36 class DevicePort(DarwinPort):
37
38     DEVICE_MANAGER = None
39     NO_DEVICE_MANAGER = 'No device manager found for port'
40
41     def __init__(self, *args, **kwargs):
42         super(DevicePort, self).__init__(*args, **kwargs)
43         self._test_runner_process_constructor = SimulatorProcess
44         self._printing_cmd_line = False
45
46     def driver_cmd_line_for_logging(self):
47         # Avoid creating/connecting to devices just for command line logging.
48         self._printing_cmd_line = True
49         result = super(DevicePort, self).driver_cmd_line_for_logging()
50         self._printing_cmd_line = False
51         return result
52
53     def _generate_all_test_configurations(self):
54         configurations = []
55         for build_type in self.ALL_BUILD_TYPES:
56             for architecture in self.ARCHITECTURES:
57                 configurations.append(TestConfiguration(version=self.version_name(), architecture=architecture, build_type=build_type))
58         return configurations
59
60     def child_processes(self):
61         return int(self.get_option('child_processes'))
62
63     def driver_name(self):
64         if self.get_option('driver_name'):
65             return self.get_option('driver_name')
66         if self.get_option('webkit_test_runner'):
67             return 'WebKitTestRunnerApp.app'
68         return 'DumpRenderTree.app'
69
70     # A device is the target host for a specific worker number.
71     def target_host(self, worker_number=None):
72         if self._printing_cmd_line or worker_number is None:
73             return self.host
74         if self.DEVICE_MANAGER is None:
75             raise RuntimeError('No device manager for specified port')
76         if self.DEVICE_MANAGER.INITIALIZED_DEVICES is None:
77             raise RuntimeError('No initialized devices for testing')
78         return self.DEVICE_MANAGER.INITIALIZED_DEVICES[worker_number]
79
80     def devices(self):
81         if self.DEVICE_MANAGER is None:
82             return []
83         if self.DEVICE_MANAGER.INITIALIZED_DEVICES is None:
84             return []
85         return self.DEVICE_MANAGER.INITIALIZED_DEVICES
86
87     # Despite their names, these flags do not actually get passed all the way down to webkit-build.
88     def _build_driver_flags(self):
89         return ['--sdk', self.SDK] + (['ARCHS=%s' % self.architecture()] if self.architecture() else [])
90
91     def _install(self):
92         if not self.get_option('install'):
93             _log.debug('Skipping installation')
94             return
95
96         for i in xrange(self.child_processes()):
97             device = self.target_host(i)
98             _log.debug('Installing to {}'.format(device))
99             # Without passing DYLD_LIBRARY_PATH, libWebCoreTestSupport cannot be loaded and DRT/WKTR will crash pre-launch,
100             # leaving a crash log which will be picked up in results. DYLD_FRAMEWORK_PATH is needed to prevent an early crash.
101             if not device.install_app(self._path_to_driver(), {'DYLD_LIBRARY_PATH': self._build_path(), 'DYLD_FRAMEWORK_PATH': self._build_path()}):
102                 raise RuntimeError('Failed to install app {} on device {}'.format(self._path_to_driver(), device.udid))
103             if not device.install_dylibs(self._build_path()):
104                 raise RuntimeError('Failed to install dylibs at {} on device {}'.format(self._build_path(), device.udid))
105
106     def _device_type_with_version(self, device_type=None):
107         device_type = device_type if device_type else self.DEVICE_TYPE
108         return DeviceType(
109             hardware_family=device_type.hardware_family,
110             hardware_type=device_type.hardware_type,
111             software_version=self.device_version(),
112             software_variant=device_type.software_variant,
113         )
114
115     def default_child_processes(self, device_type=None):
116         if not self.DEVICE_MANAGER:
117             raise RuntimeError(self.NO_DEVICE_MANAGER)
118
119         device_type = self._device_type_with_version(device_type)
120         if device_type not in self.DEVICE_TYPE:
121             return 0
122
123         if self.get_option('force'):
124             device_type.hardware_family = None
125             device_type.hardware_type = None
126
127         return self.DEVICE_MANAGER.device_count_for_type(
128             self._device_type_with_version(device_type),
129             host=self.host,
130             use_booted_simulator=not self.get_option('dedicated_simulators', False),
131         )
132
133     def max_child_processes(self, device_type=None):
134         result = self.default_child_processes(device_type=device_type)
135         if result and self.DEVICE_MANAGER == SimulatedDeviceManager:
136             return super(DevicePort, self).max_child_processes(device_type=None)
137         return result
138
139     def supported_device_types(self):
140         types = set()
141         for device in self.DEVICE_MANAGER.available_devices(host=self.host):
142             if self.DEVICE_MANAGER == SimulatedDeviceManager and not device.platform_device.is_booted_or_booting():
143                 continue
144             if device.device_type in self.DEVICE_TYPE:
145                 types.add(device.device_type)
146         if types:
147             return list(types)
148         return self.DEFAULT_DEVICE_TYPES or [self.DEVICE_TYPE]
149
150     def setup_test_run(self, device_type=None):
151         if not self.DEVICE_MANAGER:
152             raise RuntimeError(self.NO_DEVICE_MANAGER)
153
154         device_type = self._device_type_with_version(device_type)
155         _log.debug('\nCreating devices for {}'.format(device_type))
156
157         request = DeviceRequest(
158             device_type,
159             use_booted_simulator=not self.get_option('dedicated_simulators', False),
160             use_existing_simulator=False,
161             allow_incomplete_match=self.get_option('force'),
162         )
163         self.DEVICE_MANAGER.initialize_devices(
164             [request] * self.child_processes(),
165             self.host,
166             layout_test_dir=self.layout_tests_dir(),
167             pin=self.get_option('pin', None),
168             use_nfs=self.get_option('use_nfs', True),
169             reboot=self.get_option('reboot', False),
170         )
171
172         if not self.devices():
173             raise RuntimeError('No devices are available for testing')
174         if len(self.DEVICE_MANAGER.INITIALIZED_DEVICES) < self.child_processes():
175             raise RuntimeError('To few connected devices for {} processes'.format(self.child_processes()))
176
177         self._install()
178
179         for i in xrange(self.child_processes()):
180             host = self.target_host(i)
181             host.prepare_for_testing(
182                 self.ports_to_forward(),
183                 self.app_identifier_from_bundle(self._path_to_driver()),
184                 self.layout_tests_dir(),
185             )
186             self._crash_logs_to_skip_for_host[host] = host.filesystem.files_under(self.path_to_crash_logs())
187
188     def clean_up_test_run(self):
189         super(DevicePort, self).clean_up_test_run()
190
191         # Best effort to let every device teardown before throwing any exceptions here.
192         # Failure to teardown devices can leave things in a bad state.
193         exception_list = []
194         for i in xrange(self.child_processes()):
195             device = self.target_host(i)
196             if not device:
197                 continue
198             try:
199                 device.finished_testing()
200             except BaseException as e:
201                 trace = traceback.format_exc()
202                 if isinstance(e, Exception):
203                     exception_list.append([e, trace])
204                 else:
205                     exception_list.append([Exception('Exception while tearing down {}'.format(device)), trace])
206
207         if len(exception_list) == 1:
208             raise
209         if len(exception_list) > 1:
210             print('\n')
211             for exception in exception_list:
212                 _log.error('{} raised: {}'.format(exception[0].__class__.__name__, exception[0]))
213                 _log.error(exception[1])
214                 _log.error('--------------------------------------------------')
215
216             raise RuntimeError('Multiple failures when teardown devices')
217
218     def did_spawn_worker(self, worker_number):
219         super(DevicePort, self).did_spawn_worker(worker_number)
220
221         self.target_host(worker_number).release_worker_resources()
222
223     def setup_environ_for_server(self, server_name=None):
224         env = super(DevicePort, self).setup_environ_for_server(server_name)
225         if server_name == self.driver_name() and self.get_option('guard_malloc'):
226             self._append_value_colon_separated(env, 'DYLD_INSERT_LIBRARIES', '/usr/lib/libgmalloc.dylib')
227             self._append_value_colon_separated(env, '__XPC_DYLD_INSERT_LIBRARIES', '/usr/lib/libgmalloc.dylib')
228         env['XML_CATALOG_FILES'] = ''  # work around missing /etc/catalog <rdar://problem/4292995>
229         return env
230
231     def device_version(self):
232         raise NotImplementedError