Allow a port to run tests with a custom device setup
[WebKit-https.git] / Tools / Scripts / webkitpy / port / ios.py
1 # Copyright (C) 2014-2016 Apple Inc. All rights reserved.
2 #
3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions
5 # are met:
6 # 1.  Redistributions of source code must retain the above copyright
7 #     notice, this list of conditions and the following disclaimer.
8 # 2.  Redistributions in binary form must reproduce the above copyright
9 #     notice, this list of conditions and the following disclaimer in the
10 #     documentation and/or other materials provided with the distribution.
11 #
12 # THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND
13 # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
14 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
15 # DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR
16 # ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
17 # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
18 # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
19 # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
20 # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
21 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
22
23 import itertools
24 import logging
25 import os
26 import re
27 import shutil
28 import subprocess
29 import time
30
31 from webkitpy.common.memoized import memoized
32 from webkitpy.common.system.crashlogs import CrashLogs
33 from webkitpy.common.system.executive import ScriptError
34 from webkitpy.layout_tests.models.test_configuration import TestConfiguration
35 from webkitpy.port import config as port_config
36 from webkitpy.port import driver, image_diff
37 from webkitpy.port.apple import ApplePort
38 from webkitpy.port.base import Port
39 from webkitpy.xcode.simulator import Simulator, Runtime, DeviceType
40
41
42 _log = logging.getLogger(__name__)
43
44
45 class IOSPort(ApplePort):
46     port_name = "ios"
47
48     ARCHITECTURES = ['armv7', 'armv7s', 'arm64']
49     DEFAULT_ARCHITECTURE = 'arm64'
50     VERSION_FALLBACK_ORDER = ['ios-7', 'ios-8', 'ios-9']
51
52     @classmethod
53     def determine_full_port_name(cls, host, options, port_name):
54         if port_name == cls.port_name:
55             iphoneos_sdk_version = host.platform.xcode_sdk_version('iphoneos')
56             if not iphoneos_sdk_version:
57                 raise Exception("Please install the iOS SDK.")
58             major_version_number = iphoneos_sdk_version.split('.')[0]
59             port_name = port_name + '-' + major_version_number
60         return port_name
61
62     # Despite their names, these flags do not actually get passed all the way down to webkit-build.
63     def _build_driver_flags(self):
64         return ['--sdk', 'iphoneos'] + (['ARCHS=%s' % self.architecture()] if self.architecture() else [])
65
66     def operating_system(self):
67         return 'ios'
68
69
70 class IOSSimulatorPort(ApplePort):
71     port_name = "ios-simulator"
72
73     FUTURE_VERSION = 'future'
74     ARCHITECTURES = ['x86_64', 'x86']
75     DEFAULT_ARCHITECTURE = 'x86_64'
76
77     DEFAULT_DEVICE_CLASS = 'iphone'
78     CUSTOM_DEVICE_CLASSES = ['ipad']
79
80     SIMULATOR_BUNDLE_ID = 'com.apple.iphonesimulator'
81     relay_name = 'LayoutTestRelay'
82     SIMULATOR_DIRECTORY = "/tmp/WebKitTestingSimulators/"
83     LSREGISTER_PATH = "/System/Library/Frameworks/CoreServices.framework/Frameworks/LaunchServices.framework/Versions/Current/Support/lsregister"
84     PROCESS_COUNT_ESTIMATE_PER_SIMULATOR_INSTANCE = 100
85
86     DEVICE_CLASS_MAP = {
87         'x86_64': {
88             'iphone': 'iPhone 5s',
89             'ipad': 'iPad Air'
90         },
91         'x86': {
92             'iphone': 'iPhone 5',
93             'ipad': 'iPad Retina'
94         },
95     }
96
97     def __init__(self, host, port_name, **kwargs):
98         super(IOSSimulatorPort, self).__init__(host, port_name, **kwargs)
99
100         optional_device_class = self.get_option('device_class')
101         self._device_class = optional_device_class if optional_device_class else self.DEFAULT_DEVICE_CLASS
102         _log.debug('IOSSimulatorPort _device_class is %s', self._device_class)
103
104     def driver_name(self):
105         if self.get_option('driver_name'):
106             return self.get_option('driver_name')
107         if self.get_option('webkit_test_runner'):
108             return 'WebKitTestRunnerApp.app'
109         return 'DumpRenderTree.app'
110
111     @property
112     @memoized
113     def simulator_runtime(self):
114         runtime_identifier = self.get_option('runtime')
115         if runtime_identifier:
116             runtime = Runtime.from_identifier(runtime_identifier)
117         else:
118             runtime = Runtime.from_version_string(self.host.platform.xcode_sdk_version('iphonesimulator'))
119         return runtime
120
121     def simulator_device_type(self):
122         device_type_identifier = self.get_option('device_type')
123         if device_type_identifier:
124             _log.debug('simulator_device_type for device identifier %s', device_type_identifier)
125             device_type = DeviceType.from_identifier(device_type_identifier)
126         else:
127             _log.debug('simulator_device_type for device %s', self._device_class)
128             device_name = self.DEVICE_CLASS_MAP[self.architecture()][self._device_class]
129             if not device_name:
130                 raise Exception('Failed to find device for architecture {} and device class {}'.format(self.architecture()), self._device_class)
131             device_type = DeviceType.from_name(device_name)
132         return device_type
133
134     @property
135     @memoized
136     def relay_path(self):
137         if self._root_was_set:
138             path = self._filesystem.abspath(self.get_option('root'))
139         else:
140             mac_config = port_config.Config(self._executive, self._filesystem, 'mac')
141             path = mac_config.build_directory(self.get_option('configuration'))
142         return self._filesystem.join(path, self.relay_name)
143
144     @memoized
145     def child_processes(self):
146         return int(self.get_option('child_processes'))
147
148     @memoized
149     def default_child_processes(self):
150         """Return the number of Simulators instances to use for this port."""
151         best_child_process_count_for_cpu = self._executive.cpu_count() / 2
152         system_process_count_limit = int(subprocess.check_output(["ulimit", "-u"]).strip())
153         current_process_count = len(subprocess.check_output(["ps", "aux"]).strip().split('\n'))
154         _log.info('Process limit: %d, current #processes: %d' % (system_process_count_limit, current_process_count))
155         maximum_simulator_count_on_this_system = (system_process_count_limit - current_process_count) // self.PROCESS_COUNT_ESTIMATE_PER_SIMULATOR_INSTANCE
156         # FIXME: We should also take into account the available RAM.
157
158         if (maximum_simulator_count_on_this_system < best_child_process_count_for_cpu):
159             _log.warn("This machine could support %s simulators, but is only configured for %s."
160                 % (best_child_process_count_for_cpu, maximum_simulator_count_on_this_system))
161             _log.warn('Please see <https://trac.webkit.org/wiki/IncreasingKernelLimits>.')
162
163         if maximum_simulator_count_on_this_system == 0:
164             maximum_simulator_count_on_this_system = 1
165
166         return min(maximum_simulator_count_on_this_system, best_child_process_count_for_cpu)
167
168     def _check_relay(self):
169         if not self._filesystem.exists(self.relay_path):
170             _log.error("%s was not found at %s" % (self.relay_name, self.relay_path))
171             return False
172         return True
173
174     def _check_port_build(self):
175         if not self._root_was_set and self.get_option('build') and not self._build_relay():
176             return False
177         if not self._check_relay():
178             return False
179         return True
180
181     def _build_relay(self):
182         environment = self.host.copy_current_environment()
183         environment.disable_gcc_smartquotes()
184         env = environment.to_dictionary()
185
186         try:
187             # FIXME: We should be passing _arguments_for_configuration(), which respects build configuration and port,
188             # instead of hardcoding --ios-simulator.
189             self._run_script("build-layouttestrelay", args=["--ios-simulator"], env=env)
190         except ScriptError, e:
191             _log.error(e.message_with_output(output_limit=None))
192             return False
193         return True
194
195     def _build_driver(self):
196         built_tool = super(IOSSimulatorPort, self)._build_driver()
197         built_relay = self._build_relay()
198         return built_tool and built_relay
199
200     def _build_driver_flags(self):
201         archs = ['ARCHS=i386'] if self.architecture() == 'x86' else []
202         sdk = ['--sdk', 'iphonesimulator']
203         return archs + sdk
204
205     def _generate_all_test_configurations(self):
206         configurations = []
207         for build_type in self.ALL_BUILD_TYPES:
208             for architecture in self.ARCHITECTURES:
209                 configurations.append(TestConfiguration(version=self._version, architecture=architecture, build_type=build_type))
210         return configurations
211
212     def _driver_class(self):
213         return driver.IOSSimulatorDriver
214
215     def default_baseline_search_path(self):
216         if self.get_option('webkit_test_runner'):
217             fallback_names = [self._wk2_port_name()] + [self.port_name] + ['wk2']
218         else:
219             fallback_names = [self.port_name + '-wk1'] + [self.port_name]
220
221         return map(self._webkit_baseline_path, fallback_names)
222
223     def _port_specific_expectations_files(self):
224         return list(reversed([self._filesystem.join(self._webkit_baseline_path(p), 'TestExpectations') for p in self.baseline_search_path()]))
225
226     def _set_device_class(self, device_class):
227         # Ideally we'd ensure that no simulators are running when this is called.
228         self._device_class = device_class if device_class else self.DEFAULT_DEVICE_CLASS
229
230     def _create_simulators(self):
231         if (self.default_child_processes() < self.child_processes()):
232                 _log.warn("You have specified very high value({0}) for --child-processes".format(self.child_processes()))
233                 _log.warn("maximum child-processes which can be supported on this system are: {0}".format(self.default_child_processes()))
234                 _log.warn("This is very likely to fail.")
235
236         self._createSimulatorApps()
237
238         for i in xrange(self.child_processes()):
239             Simulator.wait_until_device_is_in_state(self.testing_device(i).udid, Simulator.DeviceState.SHUTDOWN)
240             Simulator.reset_device(self.testing_device(i).udid)
241
242     def setup_test_run(self, device_class=None):
243         mac_os_version = self.host.platform.os_version
244
245         self._set_device_class(device_class)
246
247         _log.debug('')
248         _log.debug('setup_test_run for %s', self._device_class)
249
250         self._create_simulators()
251
252         for i in xrange(self.child_processes()):
253             device_udid = self.testing_device(i).udid
254             _log.debug('testing device %s has udid %s', i, device_udid)
255
256             # FIXME: <rdar://problem/20916140> Switch to using CoreSimulator.framework for launching and quitting iOS Simulator
257             self._executive.run_command([
258                 'open', '-g', '-b', self.SIMULATOR_BUNDLE_ID + str(i),
259                 '--args', '-CurrentDeviceUDID', device_udid])
260
261             if mac_os_version in ['elcapitan', 'yosemite', 'mavericks']:
262                 time.sleep(2.5)
263
264         _log.info('Waiting for all iOS Simulators to finish booting.')
265         for i in xrange(self.child_processes()):
266             Simulator.wait_until_device_is_booted(self.testing_device(i).udid)
267
268     def _quit_ios_simulator(self):
269         _log.debug("_quit_ios_simulator")
270         # FIXME: We should kill only the Simulators we started.
271         subprocess.call(["killall", "-9", "-m", "Simulator"])
272
273     def clean_up_test_run(self):
274         super(IOSSimulatorPort, self).clean_up_test_run()
275         _log.debug("clean_up_test_run")
276         self._quit_ios_simulator()
277         fifos = [path for path in os.listdir('/tmp') if re.search('org.webkit.(DumpRenderTree|WebKitTestRunner).*_(IN|OUT|ERROR)', path)]
278         for fifo in fifos:
279             try:
280                 os.remove(os.path.join('/tmp', fifo))
281             except OSError:
282                 _log.warning('Unable to remove ' + fifo)
283                 pass
284
285         for i in xrange(self.child_processes()):
286             simulator_path = self.get_simulator_path(i)
287             device_udid = self.testing_device(i).udid
288             if not os.path.exists(simulator_path):
289                 continue
290             try:
291                 self._executive.run_command([self.LSREGISTER_PATH, "-u", simulator_path])
292
293                 _log.debug('rmtree %s', simulator_path)
294                 self._filesystem.rmtree(simulator_path)
295
296                 logs_path = self._filesystem.join(self._filesystem.expanduser("~"), "Library/Logs/CoreSimulator/", device_udid)
297                 _log.debug('rmtree %s', logs_path)
298                 self._filesystem.rmtree(logs_path)
299
300                 saved_state_path = self._filesystem.join(self._filesystem.expanduser("~"), "Library/Saved Application State/", self.SIMULATOR_BUNDLE_ID + str(i) + ".savedState")
301                 _log.debug('rmtree %s', saved_state_path)
302                 self._filesystem.rmtree(saved_state_path)
303
304                 Simulator().delete_device(device_udid)
305             except:
306                 _log.warning('Unable to remove Simulator' + str(i))
307
308     def setup_environ_for_server(self, server_name=None):
309         _log.debug("setup_environ_for_server")
310         env = super(IOSSimulatorPort, self).setup_environ_for_server(server_name)
311         if server_name == self.driver_name():
312             if self.get_option('leaks'):
313                 env['MallocStackLogging'] = '1'
314                 env['__XPC_MallocStackLogging'] = '1'
315                 env['MallocScribble'] = '1'
316                 env['__XPC_MallocScribble'] = '1'
317             if self.get_option('guard_malloc'):
318                 self._append_value_colon_separated(env, 'DYLD_INSERT_LIBRARIES', '/usr/lib/libgmalloc.dylib')
319                 self._append_value_colon_separated(env, '__XPC_DYLD_INSERT_LIBRARIES', '/usr/lib/libgmalloc.dylib')
320         env['XML_CATALOG_FILES'] = ''  # work around missing /etc/catalog <rdar://problem/4292995>
321         return env
322
323     def operating_system(self):
324         return 'ios-simulator'
325
326     def check_sys_deps(self, needs_http):
327         if not self.simulator_runtime.available:
328             _log.error('The iOS Simulator runtime with identifier "{0}" cannot be used because it is unavailable.'.format(self.simulator_runtime.identifier))
329             return False
330         return super(IOSSimulatorPort, self).check_sys_deps(needs_http)
331
332     SUBPROCESS_CRASH_REGEX = re.compile('#CRASHED - (?P<subprocess_name>\S+) \(pid (?P<subprocess_pid>\d+)\)')
333
334     def _get_crash_log(self, name, pid, stdout, stderr, newer_than, time_fn=time.time, sleep_fn=time.sleep, wait_for_log=True):
335         time_fn = time_fn or time.time
336         sleep_fn = sleep_fn or time.sleep
337
338         # FIXME: We should collect the actual crash log for DumpRenderTree.app because it includes more
339         # information (e.g. exception codes) than is available in the stack trace written to standard error.
340         stderr_lines = []
341         crashed_subprocess_name_and_pid = None  # e.g. ('DumpRenderTree.app', 1234)
342         for line in (stderr or '').splitlines():
343             if not crashed_subprocess_name_and_pid:
344                 match = self.SUBPROCESS_CRASH_REGEX.match(line)
345                 if match:
346                     crashed_subprocess_name_and_pid = (match.group('subprocess_name'), int(match.group('subprocess_pid')))
347                     continue
348             stderr_lines.append(line)
349
350         if crashed_subprocess_name_and_pid:
351             return self._get_crash_log(crashed_subprocess_name_and_pid[0], crashed_subprocess_name_and_pid[1], stdout,
352                 '\n'.join(stderr_lines), newer_than, time_fn, sleep_fn, wait_for_log)
353
354         # LayoutTestRelay crashed
355         _log.debug('looking for crash log for %s:%s' % (name, str(pid)))
356         crash_log = ''
357         crash_logs = CrashLogs(self.host)
358         now = time_fn()
359         deadline = now + 5 * int(self.get_option('child_processes', 1))
360         while not crash_log and now <= deadline:
361             crash_log = crash_logs.find_newest_log(name, pid, include_errors=True, newer_than=newer_than)
362             if not wait_for_log:
363                 break
364             if not crash_log or not [line for line in crash_log.splitlines() if not line.startswith('ERROR')]:
365                 sleep_fn(0.1)
366                 now = time_fn()
367
368         if not crash_log:
369             return stderr, None
370         return stderr, crash_log
371
372     def testing_device(self, number):
373         # FIXME: rather than calling lookup_or_create_device every time, we should just store a mapping of
374         # number to device_udid.
375         device_type = self.simulator_device_type()
376         _log.debug(' testing_device %s using device_type %s', number, device_type)
377         return Simulator().lookup_or_create_device(device_type.name + ' WebKit Tester' + str(number), device_type, self.simulator_runtime)
378
379     def get_simulator_path(self, suffix=""):
380         return os.path.join(self.SIMULATOR_DIRECTORY, "Simulator" + str(suffix) + ".app")
381
382     def diff_image(self, expected_contents, actual_contents, tolerance=None):
383         if not actual_contents and not expected_contents:
384             return (None, 0, None)
385         if not actual_contents or not expected_contents:
386             return (True, 0, None)
387         if not self._image_differ:
388             self._image_differ = image_diff.IOSSimulatorImageDiffer(self)
389         self.set_option_default('tolerance', 0.1)
390         if tolerance is None:
391             tolerance = self.get_option('tolerance')
392         return self._image_differ.diff_image(expected_contents, actual_contents, tolerance)
393
394     def reset_preferences(self):
395         _log.debug("reset_preferences")
396         self._quit_ios_simulator()
397         # Maybe this should delete all devices that we've created?
398
399     def nm_command(self):
400         return self.xcrun_find('nm')
401
402     def xcrun_find(self, command, fallback=None):
403         fallback = fallback or command
404         try:
405             return self._executive.run_command(['xcrun', '--sdk', 'iphonesimulator', '-find', command]).rstrip()
406         except ScriptError:
407             _log.warn("xcrun failed; falling back to '%s'." % fallback)
408             return fallback
409
410     @property
411     @memoized
412     def developer_dir(self):
413         return self._executive.run_command(['xcode-select', '--print-path']).rstrip()
414
415     def logging_patterns_to_strip(self):
416         return []
417
418     def stderr_patterns_to_strip(self):
419         return []
420
421     def _createSimulatorApps(self):
422         for i in xrange(self.child_processes()):
423             self._createSimulatorApp(i)
424
425     def _createSimulatorApp(self, suffix):
426         destination = self.get_simulator_path(suffix)
427         _log.info("Creating app:" + destination)
428         if os.path.exists(destination):
429             shutil.rmtree(destination, ignore_errors=True)
430         simulator_app_path = self.developer_dir + "/Applications/Simulator.app"
431         shutil.copytree(simulator_app_path, destination)
432
433         # Update app's package-name inside plist and re-code-sign it
434         plist_path = destination + "/Contents/Info.plist"
435         command = "Set CFBundleIdentifier com.apple.iphonesimulator" + str(suffix)
436         subprocess.check_output(["/usr/libexec/PlistBuddy", "-c", command, plist_path])
437         subprocess.check_output(["install_name_tool", "-add_rpath", self.developer_dir + "/Library/PrivateFrameworks/", destination + "/Contents/MacOS/Simulator"])
438         subprocess.check_output(["codesign", "-fs", "-", destination])
439         subprocess.check_output([self.LSREGISTER_PATH, "-f", destination])