Use simctl instead of LayoutTestRelay
[WebKit.git] / Tools / Scripts / webkitpy / port / ios.py
1 # Copyright (C) 2014-2016 Apple Inc. All rights reserved.
2 #
3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions
5 # are met:
6 # 1.  Redistributions of source code must retain the above copyright
7 #     notice, this list of conditions and the following disclaimer.
8 # 2.  Redistributions in binary form must reproduce the above copyright
9 #     notice, this list of conditions and the following disclaimer in the
10 #     documentation and/or other materials provided with the distribution.
11 #
12 # THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND
13 # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
14 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
15 # DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR
16 # ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
17 # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
18 # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
19 # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
20 # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
21 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
22
23 import itertools
24 import logging
25 import os
26 import re
27 import shutil
28 import subprocess
29 import time
30
31 from webkitpy.common.memoized import memoized
32 from webkitpy.common.system.executive import ScriptError
33 from webkitpy.layout_tests.models.test_configuration import TestConfiguration
34 from webkitpy.port import config as port_config
35 from webkitpy.port import driver, image_diff
36 from webkitpy.port.darwin import DarwinPort
37 from webkitpy.port.simulator_process import SimulatorProcess
38 from webkitpy.xcode.simulator import Simulator, Runtime, DeviceType
39 from webkitpy.common.system.crashlogs import CrashLogs
40
41
42 _log = logging.getLogger(__name__)
43
44
45 class IOSPort(DarwinPort):
46     port_name = "ios"
47
48     ARCHITECTURES = ['armv7', 'armv7s', 'arm64']
49     DEFAULT_ARCHITECTURE = 'arm64'
50     VERSION_FALLBACK_ORDER = ['ios-7', 'ios-8', 'ios-9', 'ios-10']
51
52     @classmethod
53     def determine_full_port_name(cls, host, options, port_name):
54         if port_name == cls.port_name:
55             iphoneos_sdk_version = host.platform.xcode_sdk_version('iphoneos')
56             if not iphoneos_sdk_version:
57                 raise Exception("Please install the iOS SDK.")
58             major_version_number = iphoneos_sdk_version.split('.')[0]
59             port_name = port_name + '-' + major_version_number
60         return port_name
61
62     # Despite their names, these flags do not actually get passed all the way down to webkit-build.
63     def _build_driver_flags(self):
64         return ['--sdk', 'iphoneos'] + (['ARCHS=%s' % self.architecture()] if self.architecture() else [])
65
66     def operating_system(self):
67         return 'ios'
68
69
70 class IOSSimulatorPort(DarwinPort):
71     port_name = "ios-simulator"
72
73     FUTURE_VERSION = 'future'
74     ARCHITECTURES = ['x86_64', 'x86']
75     DEFAULT_ARCHITECTURE = 'x86_64'
76
77     DEFAULT_DEVICE_CLASS = 'iphone'
78     CUSTOM_DEVICE_CLASSES = ['ipad', 'iphone7']
79     SDK = 'iphonesimulator'
80
81     SIMULATOR_BUNDLE_ID = 'com.apple.iphonesimulator'
82     SIMULATOR_DIRECTORY = "/tmp/WebKitTestingSimulators/"
83     LSREGISTER_PATH = "/System/Library/Frameworks/CoreServices.framework/Frameworks/LaunchServices.framework/Versions/Current/Support/lsregister"
84     PROCESS_COUNT_ESTIMATE_PER_SIMULATOR_INSTANCE = 100
85
86     DEVICE_CLASS_MAP = {
87         'x86_64': {
88             'iphone': 'iPhone 5s',
89             'iphone7': 'iPhone 7',
90             'ipad': 'iPad Air'
91         },
92         'x86': {
93             'iphone': 'iPhone 5',
94             'ipad': 'iPad Retina'
95         },
96     }
97
98     def __init__(self, host, port_name, **kwargs):
99         DarwinPort.__init__(self, host, port_name, **kwargs)
100         self._test_runner_process_constructor = SimulatorProcess
101
102         optional_device_class = self.get_option('device_class')
103         self._printing_cmd_line = False
104         self._device_class = optional_device_class if optional_device_class else self.DEFAULT_DEVICE_CLASS
105         _log.debug('IOSSimulatorPort _device_class is %s', self._device_class)
106
107         self._current_device = Simulator(host).current_device()
108         if not self._current_device:
109             self.set_option('dedicated_simulators', True)
110         if not self.get_option('dedicated_simulators'):
111             if self.get_option('child_processes') > 1:
112                 _log.warn('Cannot have more than one child process when using a running simulator.  Setting child_processes to 1.')
113             self.set_option('child_processes', 1)
114
115     def driver_name(self):
116         if self.get_option('driver_name'):
117             return self.get_option('driver_name')
118         if self.get_option('webkit_test_runner'):
119             return 'WebKitTestRunnerApp.app'
120         return 'DumpRenderTree.app'
121
122     def driver_cmd_line_for_logging(self):
123         # Avoid spinning up devices just for logging the commandline.
124         self._printing_cmd_line = True
125         result = super(IOSSimulatorPort, self).driver_cmd_line_for_logging()
126         self._printing_cmd_line = False
127         return result
128
129     @property
130     @memoized
131     def simulator_runtime(self):
132         runtime_identifier = self.get_option('runtime')
133         if runtime_identifier:
134             runtime = Runtime.from_identifier(runtime_identifier)
135         else:
136             runtime = Runtime.from_version_string(self.host.platform.xcode_sdk_version('iphonesimulator'))
137         return runtime
138
139     def simulator_device_type(self):
140         device_type_identifier = self.get_option('device_type')
141         if device_type_identifier:
142             _log.debug('simulator_device_type for device identifier %s', device_type_identifier)
143             device_type = DeviceType.from_identifier(device_type_identifier)
144         else:
145             _log.debug('simulator_device_type for device %s', self._device_class)
146             device_name = self.DEVICE_CLASS_MAP[self.architecture()][self._device_class]
147             if not device_name:
148                 raise Exception('Failed to find device for architecture {} and device class {}'.format(self.architecture()), self._device_class)
149             device_type = DeviceType.from_name(device_name)
150         return device_type
151
152     @memoized
153     def child_processes(self):
154         return int(self.get_option('child_processes'))
155
156     @memoized
157     def default_child_processes(self):
158         """Return the number of Simulators instances to use for this port."""
159         best_child_process_count_for_cpu = self._executive.cpu_count() / 2
160         system_process_count_limit = int(subprocess.check_output(["ulimit", "-u"]).strip())
161         current_process_count = len(subprocess.check_output(["ps", "aux"]).strip().split('\n'))
162         _log.debug('Process limit: %d, current #processes: %d' % (system_process_count_limit, current_process_count))
163         maximum_simulator_count_on_this_system = (system_process_count_limit - current_process_count) // self.PROCESS_COUNT_ESTIMATE_PER_SIMULATOR_INSTANCE
164         # FIXME: We should also take into account the available RAM.
165
166         if (maximum_simulator_count_on_this_system < best_child_process_count_for_cpu):
167             _log.warn("This machine could support %s simulators, but is only configured for %s."
168                 % (best_child_process_count_for_cpu, maximum_simulator_count_on_this_system))
169             _log.warn('Please see <https://trac.webkit.org/wiki/IncreasingKernelLimits>.')
170
171         if maximum_simulator_count_on_this_system == 0:
172             maximum_simulator_count_on_this_system = 1
173
174         return min(maximum_simulator_count_on_this_system, best_child_process_count_for_cpu)
175
176     def _get_crash_log(self, name, pid, stdout, stderr, newer_than, time_fn=time.time, sleep_fn=time.sleep, wait_for_log=True):
177         time_fn = time_fn or time.time
178         sleep_fn = sleep_fn or time.sleep
179
180         # FIXME: We should collect the actual crash log for DumpRenderTree.app because it includes more
181         # information (e.g. exception codes) than is available in the stack trace written to standard error.
182         stderr_lines = []
183         crashed_subprocess_name_and_pid = None  # e.g. ('DumpRenderTree.app', 1234)
184         for line in (stderr or '').splitlines():
185             if not crashed_subprocess_name_and_pid:
186                 match = self.SUBPROCESS_CRASH_REGEX.match(line)
187                 if match:
188                     crashed_subprocess_name_and_pid = (match.group('subprocess_name'), int(match.group('subprocess_pid')))
189                     continue
190             stderr_lines.append(line)
191
192         if crashed_subprocess_name_and_pid:
193             return self._get_crash_log(crashed_subprocess_name_and_pid[0], crashed_subprocess_name_and_pid[1], stdout,
194                 '\n'.join(stderr_lines), newer_than, time_fn, sleep_fn, wait_for_log)
195
196         # App crashed
197         _log.debug('looking for crash log for %s:%s' % (name, str(pid)))
198         crash_log = ''
199         crash_logs = CrashLogs(self.host)
200         now = time_fn()
201         deadline = now + 5 * int(self.get_option('child_processes', 1))
202         while not crash_log and now <= deadline:
203             crash_log = crash_logs.find_newest_log(name, pid, include_errors=True, newer_than=newer_than)
204             if not wait_for_log:
205                 break
206             if not crash_log or not [line for line in crash_log.splitlines() if not line.startswith('ERROR')]:
207                 sleep_fn(0.1)
208                 now = time_fn()
209
210         if not crash_log:
211             return stderr, None
212         return stderr, crash_log
213
214     def _build_driver_flags(self):
215         archs = ['ARCHS=i386'] if self.architecture() == 'x86' else []
216         sdk = ['--sdk', 'iphonesimulator']
217         return archs + sdk
218
219     def _generate_all_test_configurations(self):
220         configurations = []
221         for build_type in self.ALL_BUILD_TYPES:
222             for architecture in self.ARCHITECTURES:
223                 configurations.append(TestConfiguration(version=self._version, architecture=architecture, build_type=build_type))
224         return configurations
225
226     def default_baseline_search_path(self):
227         if self.get_option('webkit_test_runner'):
228             fallback_names = [self._wk2_port_name()] + [self.port_name] + ['wk2']
229         else:
230             fallback_names = [self.port_name + '-wk1'] + [self.port_name]
231
232         return map(self._webkit_baseline_path, fallback_names)
233
234     def _set_device_class(self, device_class):
235         self._device_class = device_class if device_class else self.DEFAULT_DEVICE_CLASS
236
237     def _create_simulators(self):
238         if (self.default_child_processes() < self.child_processes()):
239             _log.warn('You have specified very high value({0}) for --child-processes'.format(self.child_processes()))
240             _log.warn('maximum child-processes which can be supported on this system are: {0}'.format(self.default_child_processes()))
241             _log.warn('This is very likely to fail.')
242
243         if self._using_dedicated_simulators():
244             self._createSimulatorApps()
245
246             for i in xrange(self.child_processes()):
247                 self._create_device(i)
248
249             for i in xrange(self.child_processes()):
250                 device_udid = self._testing_device(i).udid
251                 Simulator.wait_until_device_is_in_state(device_udid, Simulator.DeviceState.SHUTDOWN)
252                 Simulator.reset_device(device_udid)
253         else:
254             assert(self._current_device)
255             if self._current_device.name != self.simulator_device_type().name:
256                 _log.warn("Expected simulator of type '" + self.simulator_device_type().name + "' but found simulator of type '" + self._current_device.name + "'")
257                 _log.warn('The next block of tests may fail due to device mis-match')
258
259     def setup_test_run(self, device_class=None):
260         mac_os_version = self.host.platform.os_version
261
262         self._set_device_class(device_class)
263
264         _log.debug('')
265         _log.debug('setup_test_run for %s', self._device_class)
266
267         self._create_simulators()
268
269         if not self._using_dedicated_simulators():
270             return
271
272         for i in xrange(self.child_processes()):
273             device_udid = self._testing_device(i).udid
274             _log.debug('testing device %s has udid %s', i, device_udid)
275
276             # FIXME: <rdar://problem/20916140> Switch to using CoreSimulator.framework for launching and quitting iOS Simulator
277             self._executive.run_command([
278                 'open', '-g', '-b', self.SIMULATOR_BUNDLE_ID + str(i),
279                 '--args', '-CurrentDeviceUDID', device_udid])
280
281             if mac_os_version in ['elcapitan', 'yosemite', 'mavericks']:
282                 time.sleep(2.5)
283
284         _log.info('Waiting for all iOS Simulators to finish booting.')
285         for i in xrange(self.child_processes()):
286             Simulator.wait_until_device_is_booted(self._testing_device(i).udid)
287
288     def _quit_ios_simulator(self):
289         if not self._using_dedicated_simulators():
290             return
291         _log.debug("_quit_ios_simulator killing all Simulator processes")
292         # FIXME: We should kill only the Simulators we started.
293         subprocess.call(["killall", "-9", "-m", "Simulator"])
294
295     def clean_up_test_run(self):
296         super(IOSSimulatorPort, self).clean_up_test_run()
297         _log.debug("clean_up_test_run")
298         self._quit_ios_simulator()
299         fifos = [path for path in os.listdir('/tmp') if re.search('org.webkit.(DumpRenderTree|WebKitTestRunner).*_(IN|OUT|ERROR)', path)]
300         for fifo in fifos:
301             try:
302                 os.remove(os.path.join('/tmp', fifo))
303             except OSError:
304                 _log.warning('Unable to remove ' + fifo)
305                 pass
306
307         if not self._using_dedicated_simulators():
308             return
309
310         for i in xrange(self.child_processes()):
311             simulator_path = self.get_simulator_path(i)
312             device_udid = self._testing_device(i).udid
313             self._remove_device(i)
314
315             if not os.path.exists(simulator_path):
316                 continue
317             try:
318                 self._executive.run_command([self.LSREGISTER_PATH, "-u", simulator_path])
319
320                 _log.debug('rmtree %s', simulator_path)
321                 self._filesystem.rmtree(simulator_path)
322
323                 logs_path = self._filesystem.join(self._filesystem.expanduser("~"), "Library/Logs/CoreSimulator/", device_udid)
324                 _log.debug('rmtree %s', logs_path)
325                 self._filesystem.rmtree(logs_path)
326
327                 saved_state_path = self._filesystem.join(self._filesystem.expanduser("~"), "Library/Saved Application State/", self.SIMULATOR_BUNDLE_ID + str(i) + ".savedState")
328                 _log.debug('rmtree %s', saved_state_path)
329                 self._filesystem.rmtree(saved_state_path)
330
331             except:
332                 _log.warning('Unable to remove Simulator' + str(i))
333
334     def setup_environ_for_server(self, server_name=None):
335         _log.debug("setup_environ_for_server")
336         env = super(IOSSimulatorPort, self).setup_environ_for_server(server_name)
337         if server_name == self.driver_name():
338             if self.get_option('leaks'):
339                 env['MallocStackLogging'] = '1'
340                 env['__XPC_MallocStackLogging'] = '1'
341                 env['MallocScribble'] = '1'
342                 env['__XPC_MallocScribble'] = '1'
343             if self.get_option('guard_malloc'):
344                 self._append_value_colon_separated(env, 'DYLD_INSERT_LIBRARIES', '/usr/lib/libgmalloc.dylib')
345                 self._append_value_colon_separated(env, '__XPC_DYLD_INSERT_LIBRARIES', '/usr/lib/libgmalloc.dylib')
346         env['XML_CATALOG_FILES'] = ''  # work around missing /etc/catalog <rdar://problem/4292995>
347         return env
348
349     def operating_system(self):
350         return 'ios-simulator'
351
352     def check_sys_deps(self, needs_http):
353         if not self.simulator_runtime.available:
354             _log.error('The iOS Simulator runtime with identifier "{0}" cannot be used because it is unavailable.'.format(self.simulator_runtime.identifier))
355             return False
356         return super(IOSSimulatorPort, self).check_sys_deps(needs_http)
357
358     SUBPROCESS_CRASH_REGEX = re.compile('#CRASHED - (?P<subprocess_name>\S+) \(pid (?P<subprocess_pid>\d+)\)')
359
360     def _using_dedicated_simulators(self):
361         return self.get_option('dedicated_simulators')
362
363     def _create_device(self, number):
364         return Simulator.create_device(number, self.simulator_device_type(), self.simulator_runtime)
365
366     def _remove_device(self, number):
367         Simulator.remove_device(number)
368
369     def _testing_device(self, number):
370         return Simulator.device_number(number)
371
372     # FIXME: This is only exposed so that SimulatorProcess can use it.
373     def device_id_for_worker_number(self, number):
374         if self._printing_cmd_line:
375             return '<dummy id>'
376         if self._using_dedicated_simulators():
377             return self._testing_device(number).udid
378         return self._current_device.udid
379
380     def get_simulator_path(self, suffix=""):
381         return os.path.join(self.SIMULATOR_DIRECTORY, "Simulator" + str(suffix) + ".app")
382
383     def diff_image(self, expected_contents, actual_contents, tolerance=None):
384         if not actual_contents and not expected_contents:
385             return (None, 0, None)
386         if not actual_contents or not expected_contents:
387             return (True, 0, None)
388         if not self._image_differ:
389             self._image_differ = image_diff.IOSSimulatorImageDiffer(self)
390         self.set_option_default('tolerance', 0.1)
391         if tolerance is None:
392             tolerance = self.get_option('tolerance')
393         return self._image_differ.diff_image(expected_contents, actual_contents, tolerance)
394
395     def reset_preferences(self):
396         _log.debug("reset_preferences")
397         self._quit_ios_simulator()
398         # Maybe this should delete all devices that we've created?
399
400     def nm_command(self):
401         return self.xcrun_find('nm')
402
403     @property
404     @memoized
405     def developer_dir(self):
406         return self._executive.run_command(['xcode-select', '--print-path']).rstrip()
407
408     def logging_patterns_to_strip(self):
409         return []
410
411     def stderr_patterns_to_strip(self):
412         return []
413
414     def _createSimulatorApps(self):
415         for i in xrange(self.child_processes()):
416             self._createSimulatorApp(i)
417
418     def _createSimulatorApp(self, suffix):
419         destination = self.get_simulator_path(suffix)
420         _log.info("Creating app:" + destination)
421         if os.path.exists(destination):
422             shutil.rmtree(destination, ignore_errors=True)
423         simulator_app_path = self.developer_dir + "/Applications/Simulator.app"
424         shutil.copytree(simulator_app_path, destination)
425
426         # Update app's package-name inside plist and re-code-sign it
427         plist_path = destination + "/Contents/Info.plist"
428         command = "Set CFBundleIdentifier com.apple.iphonesimulator" + str(suffix)
429         subprocess.check_output(["/usr/libexec/PlistBuddy", "-c", command, plist_path])
430         subprocess.check_output(["install_name_tool", "-add_rpath", self.developer_dir + "/Library/PrivateFrameworks/", destination + "/Contents/MacOS/Simulator"])
431         subprocess.check_output(["install_name_tool", "-add_rpath", self.developer_dir + "/../Frameworks/", destination + "/Contents/MacOS/Simulator"])
432         subprocess.check_output(["codesign", "-fs", "-", destination])
433         subprocess.check_output([self.LSREGISTER_PATH, "-f", destination])