7291c9431a522e3b3e86a684b3104b6585d04ff2
[WebKit-https.git] / Tools / Scripts / webkitpy / port / ios.py
1 # Copyright (C) 2014-2017 Apple Inc. All rights reserved.
2 #
3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions
5 # are met:
6 # 1.  Redistributions of source code must retain the above copyright
7 #     notice, this list of conditions and the following disclaimer.
8 # 2.  Redistributions in binary form must reproduce the above copyright
9 #     notice, this list of conditions and the following disclaimer in the
10 #     documentation and/or other materials provided with the distribution.
11 #
12 # THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND
13 # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
14 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
15 # DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR
16 # ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
17 # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
18 # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
19 # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
20 # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
21 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
22
23 import logging
24 import traceback
25
26 from webkitpy.common.memoized import memoized
27 from webkitpy.common.version import Version
28 from webkitpy.common.version_name_map import VersionNameMap, INTERNAL_TABLE
29 from webkitpy.layout_tests.models.test_configuration import TestConfiguration
30 from webkitpy.port.config import apple_additions
31 from webkitpy.port.darwin import DarwinPort
32 from webkitpy.port.simulator_process import SimulatorProcess
33
34 _log = logging.getLogger(__name__)
35
36
37 class IOSPort(DarwinPort):
38     port_name = "ios"
39
40     CURRENT_VERSION = Version(12)
41
42     def __init__(self, host, port_name, **kwargs):
43         super(IOSPort, self).__init__(host, port_name, **kwargs)
44         self._test_runner_process_constructor = SimulatorProcess
45         self._printing_cmd_line = False
46
47     def _device_for_worker_number_map(self):
48         raise NotImplementedError
49
50     def version_name(self):
51         if self._os_version is None:
52             return None
53         return VersionNameMap.map(self.host.platform).to_name(self._os_version, platform=IOSPort.port_name)
54
55     def driver_cmd_line_for_logging(self):
56         # Avoid creating/connecting to devices just for logging the commandline.
57         self._printing_cmd_line = True
58         result = super(IOSPort, self).driver_cmd_line_for_logging()
59         self._printing_cmd_line = False
60         return result
61
62     def driver_name(self):
63         if self.get_option('driver_name'):
64             return self.get_option('driver_name')
65         if self.get_option('webkit_test_runner'):
66             return 'WebKitTestRunnerApp.app'
67         return 'DumpRenderTree.app'
68
69     def _generate_all_test_configurations(self):
70         configurations = []
71         for build_type in self.ALL_BUILD_TYPES:
72             for architecture in self.ARCHITECTURES:
73                 configurations.append(TestConfiguration(version=self.version_name(), architecture=architecture, build_type=build_type))
74         return configurations
75
76     def child_processes(self):
77         return int(self.get_option('child_processes'))
78
79     def _testing_device(self, number):
80         device = self._device_for_worker_number_map()[number]
81         if not device:
82             raise RuntimeError('Device at {} could not be found'.format(number))
83         return device
84
85     # A device is the target host for a specific worker number.
86     def target_host(self, worker_number=None):
87         if self._printing_cmd_line or worker_number is None:
88             return self.host
89         return self._testing_device(worker_number)
90
91     @memoized
92     def default_baseline_search_path(self):
93         wk_string = 'wk1'
94         if self.get_option('webkit_test_runner'):
95             wk_string = 'wk2'
96
97         versions_to_fallback = []
98         if self.ios_version().major == self.CURRENT_VERSION.major:
99             versions_to_fallback = [self.CURRENT_VERSION]
100         elif self.ios_version():
101             temp_version = Version(self.ios_version().major)
102             while temp_version != self.CURRENT_VERSION:
103                 versions_to_fallback.append(Version.from_iterable(temp_version))
104                 if temp_version < self.CURRENT_VERSION:
105                     temp_version.major += 1
106                 else:
107                     temp_version.major -= 1
108
109         expectations = []
110         for version in versions_to_fallback:
111             apple_name = None
112             if apple_additions():
113                 apple_name = VersionNameMap.map(self.host.platform).to_name(version, platform=IOSPort.port_name, table=INTERNAL_TABLE)
114
115             if apple_name:
116                 expectations.append(self._apple_baseline_path('{}-{}-{}'.format(self.port_name, apple_name.lower().replace(' ', ''), wk_string)))
117             expectations.append(self._webkit_baseline_path('{}-{}-{}'.format(self.port_name, version.major, wk_string)))
118             if apple_name:
119                 expectations.append(self._apple_baseline_path('{}-{}'.format(self.port_name, apple_name.lower().replace(' ', ''))))
120             expectations.append(self._webkit_baseline_path('{}-{}'.format(self.port_name, version.major)))
121
122         if apple_additions():
123             expectations.append(self._apple_baseline_path('{}-{}'.format(self.port_name, wk_string)))
124         expectations.append(self._webkit_baseline_path('{}-{}'.format(self.port_name, wk_string)))
125         if apple_additions():
126             expectations.append(self._apple_baseline_path(self.port_name))
127         expectations.append(self._webkit_baseline_path(self.port_name))
128
129         for version in versions_to_fallback:
130             apple_name = None
131             if apple_additions():
132                 apple_name = VersionNameMap.map(self.host.platform).to_name(version, platform=IOSPort.port_name, table=INTERNAL_TABLE)
133             if apple_name:
134                 expectations.append(self._apple_baseline_path('{}-{}'.format(IOSPort.port_name, apple_name.lower().replace(' ', ''))))
135             expectations.append(self._webkit_baseline_path('{}-{}'.format(IOSPort.port_name, version.major)))
136
137         if apple_additions():
138             expectations.append(self._apple_baseline_path('{}-{}'.format(IOSPort.port_name, wk_string)))
139         expectations.append(self._webkit_baseline_path('{}-{}'.format(IOSPort.port_name, wk_string)))
140         if apple_additions():
141             expectations.append(self._apple_baseline_path(IOSPort.port_name))
142         expectations.append(self._webkit_baseline_path(IOSPort.port_name))
143
144         if self.get_option('webkit_test_runner'):
145             expectations.append(self._webkit_baseline_path('wk2'))
146
147         return expectations
148
149     def test_expectations_file_position(self):
150         return 4
151
152     def ios_version(self):
153         raise NotImplementedError
154
155     def _create_devices(self, device_class):
156         raise NotImplementedError
157
158     def setup_test_run(self, device_class=None):
159         self._create_devices(device_class)
160
161         if self.get_option('install'):
162             for i in xrange(self.child_processes()):
163                 device = self.target_host(i)
164                 _log.debug('Installing to {}'.format(device))
165                 # Without passing DYLD_LIBRARY_PATH, libWebCoreTestSupport cannot be loaded and DRT/WKTR will crash pre-launch,
166                 # leaving a crash log which will be picked up in results. DYLD_FRAMEWORK_PATH is needed to prevent an early crash.
167                 if not device.install_app(self._path_to_driver(), {'DYLD_LIBRARY_PATH': self._build_path(), 'DYLD_FRAMEWORK_PATH': self._build_path()}):
168                     raise RuntimeError('Failed to install app {} on device {}'.format(self._path_to_driver(), device.udid))
169                 if not device.install_dylibs(self._build_path()):
170                     raise RuntimeError('Failed to install dylibs at {} on device {}'.format(self._build_path(), device.udid))
171         else:
172             _log.debug('Skipping installation')
173
174         for i in xrange(self.child_processes()):
175             host = self.target_host(i)
176             host.prepare_for_testing(
177                 self.ports_to_forward(),
178                 self.app_identifier_from_bundle(self._path_to_driver()),
179                 self.layout_tests_dir(),
180             )
181             self._crash_logs_to_skip_for_host[host] = host.filesystem.files_under(self.path_to_crash_logs())
182
183     def clean_up_test_run(self):
184         super(IOSPort, self).clean_up_test_run()
185
186         # Best effort to let every device teardown before throwing any exceptions here.
187         # Failure to teardown devices can leave things in a bad state.
188         exception_list = []
189         for i in xrange(self.child_processes()):
190             device = self.target_host(i)
191             try:
192                 if device:
193                     device.finished_testing()
194             except BaseException as e:
195                 trace = traceback.format_exc()
196                 if isinstance(e, Exception):
197                     exception_list.append([e, trace])
198                 else:
199                     exception_list.append([Exception('Exception tearing down {}'.format(device)), trace])
200         if len(exception_list) == 1:
201             raise
202         elif len(exception_list) > 1:
203             print('\n')
204             for exception in exception_list:
205                 _log.error('{} raised: {}'.format(exception[0].__class__.__name__, exception[0]))
206                 _log.error(exception[1])
207                 _log.error('--------------------------------------------------')
208
209             raise RuntimeError('Multiple failures when teardown devices')
210
211     def did_spawn_worker(self, worker_number):
212         super(IOSPort, self).did_spawn_worker(worker_number)
213
214         self.target_host(worker_number).release_worker_resources()