Make it possible to use an existing simulator instance for one-off testing
[WebKit-https.git] / Tools / Scripts / webkitpy / port / ios.py
1 # Copyright (C) 2014-2016 Apple Inc. All rights reserved.
2 #
3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions
5 # are met:
6 # 1.  Redistributions of source code must retain the above copyright
7 #     notice, this list of conditions and the following disclaimer.
8 # 2.  Redistributions in binary form must reproduce the above copyright
9 #     notice, this list of conditions and the following disclaimer in the
10 #     documentation and/or other materials provided with the distribution.
11 #
12 # THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND
13 # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
14 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
15 # DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR
16 # ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
17 # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
18 # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
19 # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
20 # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
21 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
22
23 import itertools
24 import logging
25 import os
26 import re
27 import shutil
28 import subprocess
29 import time
30
31 from webkitpy.common.memoized import memoized
32 from webkitpy.common.system.executive import ScriptError
33 from webkitpy.layout_tests.models.test_configuration import TestConfiguration
34 from webkitpy.port import config as port_config
35 from webkitpy.port import driver, image_diff
36 from webkitpy.port.darwin import DarwinPort
37 from webkitpy.xcode.simulator import Simulator, Runtime, DeviceType
38 from webkitpy.common.system.crashlogs import CrashLogs
39
40
41 _log = logging.getLogger(__name__)
42
43
44 class IOSPort(DarwinPort):
45     port_name = "ios"
46
47     ARCHITECTURES = ['armv7', 'armv7s', 'arm64']
48     DEFAULT_ARCHITECTURE = 'arm64'
49     VERSION_FALLBACK_ORDER = ['ios-7', 'ios-8', 'ios-9', 'ios-10']
50
51     @classmethod
52     def determine_full_port_name(cls, host, options, port_name):
53         if port_name == cls.port_name:
54             iphoneos_sdk_version = host.platform.xcode_sdk_version('iphoneos')
55             if not iphoneos_sdk_version:
56                 raise Exception("Please install the iOS SDK.")
57             major_version_number = iphoneos_sdk_version.split('.')[0]
58             port_name = port_name + '-' + major_version_number
59         return port_name
60
61     # Despite their names, these flags do not actually get passed all the way down to webkit-build.
62     def _build_driver_flags(self):
63         return ['--sdk', 'iphoneos'] + (['ARCHS=%s' % self.architecture()] if self.architecture() else [])
64
65     def operating_system(self):
66         return 'ios'
67
68
69 class IOSSimulatorPort(DarwinPort):
70     port_name = "ios-simulator"
71
72     FUTURE_VERSION = 'future'
73     ARCHITECTURES = ['x86_64', 'x86']
74     DEFAULT_ARCHITECTURE = 'x86_64'
75
76     DEFAULT_DEVICE_CLASS = 'iphone'
77     CUSTOM_DEVICE_CLASSES = ['ipad', 'iphone7']
78     SDK = 'iphonesimulator'
79
80     SIMULATOR_BUNDLE_ID = 'com.apple.iphonesimulator'
81     relay_name = 'LayoutTestRelay'
82     SIMULATOR_DIRECTORY = "/tmp/WebKitTestingSimulators/"
83     LSREGISTER_PATH = "/System/Library/Frameworks/CoreServices.framework/Frameworks/LaunchServices.framework/Versions/Current/Support/lsregister"
84     PROCESS_COUNT_ESTIMATE_PER_SIMULATOR_INSTANCE = 100
85
86     DEVICE_CLASS_MAP = {
87         'x86_64': {
88             'iphone': 'iPhone 5s',
89             'iphone7': 'iPhone 7',
90             'ipad': 'iPad Air'
91         },
92         'x86': {
93             'iphone': 'iPhone 5',
94             'ipad': 'iPad Retina'
95         },
96     }
97
98     def __init__(self, host, port_name, **kwargs):
99         DarwinPort.__init__(self, host, port_name, **kwargs)
100
101         optional_device_class = self.get_option('device_class')
102         self._printing_cmd_line = False
103         self._device_class = optional_device_class if optional_device_class else self.DEFAULT_DEVICE_CLASS
104         _log.debug('IOSSimulatorPort _device_class is %s', self._device_class)
105
106         self._current_device = Simulator(host).current_device()
107         if not self._current_device:
108             self.set_option('dedicated_simulators', True)
109         if not self.get_option('dedicated_simulators'):
110             if self.get_option('child_processes') > 1:
111                 _log.warn('Cannot have more than one child process when using a running simulator.  Setting child_processes to 1.')
112             self.set_option('child_processes', 1)
113
114     def driver_name(self):
115         if self.get_option('driver_name'):
116             return self.get_option('driver_name')
117         if self.get_option('webkit_test_runner'):
118             return 'WebKitTestRunnerApp.app'
119         return 'DumpRenderTree.app'
120
121     def driver_cmd_line_for_logging(self):
122         # Avoid spinning up devices just for logging the commandline.
123         self._printing_cmd_line = True
124         result = super(IOSSimulatorPort, self).driver_cmd_line_for_logging()
125         self._printing_cmd_line = False
126         return result
127
128     @property
129     @memoized
130     def simulator_runtime(self):
131         runtime_identifier = self.get_option('runtime')
132         if runtime_identifier:
133             runtime = Runtime.from_identifier(runtime_identifier)
134         else:
135             runtime = Runtime.from_version_string(self.host.platform.xcode_sdk_version('iphonesimulator'))
136         return runtime
137
138     def simulator_device_type(self):
139         device_type_identifier = self.get_option('device_type')
140         if device_type_identifier:
141             _log.debug('simulator_device_type for device identifier %s', device_type_identifier)
142             device_type = DeviceType.from_identifier(device_type_identifier)
143         else:
144             _log.debug('simulator_device_type for device %s', self._device_class)
145             device_name = self.DEVICE_CLASS_MAP[self.architecture()][self._device_class]
146             if not device_name:
147                 raise Exception('Failed to find device for architecture {} and device class {}'.format(self.architecture()), self._device_class)
148             device_type = DeviceType.from_name(device_name)
149         return device_type
150
151     @property
152     @memoized
153     def relay_path(self):
154         if self._root_was_set:
155             path = self._filesystem.abspath(self.get_option('root'))
156         else:
157             mac_config = port_config.Config(self._executive, self._filesystem, 'mac')
158             path = mac_config.build_directory(self.get_option('configuration'))
159         return self._filesystem.join(path, self.relay_name)
160
161     @memoized
162     def child_processes(self):
163         return int(self.get_option('child_processes'))
164
165     @memoized
166     def default_child_processes(self):
167         """Return the number of Simulators instances to use for this port."""
168         best_child_process_count_for_cpu = self._executive.cpu_count() / 2
169         system_process_count_limit = int(subprocess.check_output(["ulimit", "-u"]).strip())
170         current_process_count = len(subprocess.check_output(["ps", "aux"]).strip().split('\n'))
171         _log.debug('Process limit: %d, current #processes: %d' % (system_process_count_limit, current_process_count))
172         maximum_simulator_count_on_this_system = (system_process_count_limit - current_process_count) // self.PROCESS_COUNT_ESTIMATE_PER_SIMULATOR_INSTANCE
173         # FIXME: We should also take into account the available RAM.
174
175         if (maximum_simulator_count_on_this_system < best_child_process_count_for_cpu):
176             _log.warn("This machine could support %s simulators, but is only configured for %s."
177                 % (best_child_process_count_for_cpu, maximum_simulator_count_on_this_system))
178             _log.warn('Please see <https://trac.webkit.org/wiki/IncreasingKernelLimits>.')
179
180         if maximum_simulator_count_on_this_system == 0:
181             maximum_simulator_count_on_this_system = 1
182
183         return min(maximum_simulator_count_on_this_system, best_child_process_count_for_cpu)
184
185     def _check_relay(self):
186         if not self._filesystem.exists(self.relay_path):
187             _log.error("%s was not found at %s" % (self.relay_name, self.relay_path))
188             return False
189         return True
190
191     def _check_port_build(self):
192         if not self._root_was_set and self.get_option('build') and not self._build_relay():
193             return False
194         if not self._check_relay():
195             return False
196         return True
197
198     def _get_crash_log(self, name, pid, stdout, stderr, newer_than, time_fn=time.time, sleep_fn=time.sleep, wait_for_log=True):
199         time_fn = time_fn or time.time
200         sleep_fn = sleep_fn or time.sleep
201
202         # FIXME: We should collect the actual crash log for DumpRenderTree.app because it includes more
203         # information (e.g. exception codes) than is available in the stack trace written to standard error.
204         stderr_lines = []
205         crashed_subprocess_name_and_pid = None  # e.g. ('DumpRenderTree.app', 1234)
206         for line in (stderr or '').splitlines():
207             if not crashed_subprocess_name_and_pid:
208                 match = self.SUBPROCESS_CRASH_REGEX.match(line)
209                 if match:
210                     crashed_subprocess_name_and_pid = (match.group('subprocess_name'), int(match.group('subprocess_pid')))
211                     continue
212             stderr_lines.append(line)
213
214         if crashed_subprocess_name_and_pid:
215             return self._get_crash_log(crashed_subprocess_name_and_pid[0], crashed_subprocess_name_and_pid[1], stdout,
216                 '\n'.join(stderr_lines), newer_than, time_fn, sleep_fn, wait_for_log)
217
218         # LayoutTestRelay crashed
219         _log.debug('looking for crash log for %s:%s' % (name, str(pid)))
220         crash_log = ''
221         crash_logs = CrashLogs(self.host)
222         now = time_fn()
223         deadline = now + 5 * int(self.get_option('child_processes', 1))
224         while not crash_log and now <= deadline:
225             crash_log = crash_logs.find_newest_log(name, pid, include_errors=True, newer_than=newer_than)
226             if not wait_for_log:
227                 break
228             if not crash_log or not [line for line in crash_log.splitlines() if not line.startswith('ERROR')]:
229                 sleep_fn(0.1)
230                 now = time_fn()
231
232         if not crash_log:
233             return stderr, None
234         return stderr, crash_log
235
236     def _build_relay(self):
237         environment = self.host.copy_current_environment()
238         environment.disable_gcc_smartquotes()
239         env = environment.to_dictionary()
240
241         try:
242             # FIXME: We should be passing _arguments_for_configuration(), which respects build configuration and port,
243             # instead of hardcoding --ios-simulator.
244             self._run_script("build-layouttestrelay", args=["--ios-simulator"], env=env)
245         except ScriptError, e:
246             _log.error(e.message_with_output(output_limit=None))
247             return False
248         return True
249
250     def _build_driver(self):
251         built_tool = super(IOSSimulatorPort, self)._build_driver()
252         built_relay = self._build_relay()
253         return built_tool and built_relay
254
255     def _build_driver_flags(self):
256         archs = ['ARCHS=i386'] if self.architecture() == 'x86' else []
257         sdk = ['--sdk', 'iphonesimulator']
258         return archs + sdk
259
260     def _generate_all_test_configurations(self):
261         configurations = []
262         for build_type in self.ALL_BUILD_TYPES:
263             for architecture in self.ARCHITECTURES:
264                 configurations.append(TestConfiguration(version=self._version, architecture=architecture, build_type=build_type))
265         return configurations
266
267     def _driver_class(self):
268         return driver.IOSSimulatorDriver
269
270     def default_baseline_search_path(self):
271         if self.get_option('webkit_test_runner'):
272             fallback_names = [self._wk2_port_name()] + [self.port_name] + ['wk2']
273         else:
274             fallback_names = [self.port_name + '-wk1'] + [self.port_name]
275
276         return map(self._webkit_baseline_path, fallback_names)
277
278     def _set_device_class(self, device_class):
279         self._device_class = device_class if device_class else self.DEFAULT_DEVICE_CLASS
280
281     def _create_simulators(self):
282         if (self.default_child_processes() < self.child_processes()):
283                 _log.warn("You have specified very high value({0}) for --child-processes".format(self.child_processes()))
284                 _log.warn("maximum child-processes which can be supported on this system are: {0}".format(self.default_child_processes()))
285                 _log.warn("This is very likely to fail.")
286
287         if self._using_dedicated_simulators():
288             self._createSimulatorApps()
289
290             for i in xrange(self.child_processes()):
291                 self._create_device(i)
292
293             for i in xrange(self.child_processes()):
294                 device_udid = self._testing_device(i).udid
295                 Simulator.wait_until_device_is_in_state(device_udid, Simulator.DeviceState.SHUTDOWN)
296                 Simulator.reset_device(device_udid)
297         else:
298             assert(self._current_device)
299             if self._current_device.name != self.simulator_device_type().name:
300                 _log.warn("Expected simulator of type '" + self.simulator_device_type().name + "' but found simulator of type '" + self._current_device.name + "'")
301                 _log.warn('The next block of tests may fail due to device mis-match')
302
303     def setup_test_run(self, device_class=None):
304         mac_os_version = self.host.platform.os_version
305
306         self._set_device_class(device_class)
307
308         _log.debug('')
309         _log.debug('setup_test_run for %s', self._device_class)
310
311         self._create_simulators()
312
313         if not self._using_dedicated_simulators():
314             return
315
316         for i in xrange(self.child_processes()):
317             device_udid = self._testing_device(i).udid
318             _log.debug('testing device %s has udid %s', i, device_udid)
319
320             # FIXME: <rdar://problem/20916140> Switch to using CoreSimulator.framework for launching and quitting iOS Simulator
321             self._executive.run_command([
322                 'open', '-g', '-b', self.SIMULATOR_BUNDLE_ID + str(i),
323                 '--args', '-CurrentDeviceUDID', device_udid])
324
325             if mac_os_version in ['elcapitan', 'yosemite', 'mavericks']:
326                 time.sleep(2.5)
327
328         _log.info('Waiting for all iOS Simulators to finish booting.')
329         for i in xrange(self.child_processes()):
330             Simulator.wait_until_device_is_booted(self._testing_device(i).udid)
331
332     def _quit_ios_simulator(self):
333         if not self._using_dedicated_simulators():
334             return
335         _log.debug("_quit_ios_simulator killing all Simulator processes")
336         # FIXME: We should kill only the Simulators we started.
337         subprocess.call(["killall", "-9", "-m", "Simulator"])
338
339     def clean_up_test_run(self):
340         super(IOSSimulatorPort, self).clean_up_test_run()
341         _log.debug("clean_up_test_run")
342         self._quit_ios_simulator()
343         fifos = [path for path in os.listdir('/tmp') if re.search('org.webkit.(DumpRenderTree|WebKitTestRunner).*_(IN|OUT|ERROR)', path)]
344         for fifo in fifos:
345             try:
346                 os.remove(os.path.join('/tmp', fifo))
347             except OSError:
348                 _log.warning('Unable to remove ' + fifo)
349                 pass
350
351         if not self._using_dedicated_simulators():
352             return
353
354         for i in xrange(self.child_processes()):
355             simulator_path = self.get_simulator_path(i)
356             device_udid = self._testing_device(i).udid
357             self._remove_device(i)
358
359             if not os.path.exists(simulator_path):
360                 continue
361             try:
362                 self._executive.run_command([self.LSREGISTER_PATH, "-u", simulator_path])
363
364                 _log.debug('rmtree %s', simulator_path)
365                 self._filesystem.rmtree(simulator_path)
366
367                 logs_path = self._filesystem.join(self._filesystem.expanduser("~"), "Library/Logs/CoreSimulator/", device_udid)
368                 _log.debug('rmtree %s', logs_path)
369                 self._filesystem.rmtree(logs_path)
370
371                 saved_state_path = self._filesystem.join(self._filesystem.expanduser("~"), "Library/Saved Application State/", self.SIMULATOR_BUNDLE_ID + str(i) + ".savedState")
372                 _log.debug('rmtree %s', saved_state_path)
373                 self._filesystem.rmtree(saved_state_path)
374
375             except:
376                 _log.warning('Unable to remove Simulator' + str(i))
377
378     def setup_environ_for_server(self, server_name=None):
379         _log.debug("setup_environ_for_server")
380         env = super(IOSSimulatorPort, self).setup_environ_for_server(server_name)
381         if server_name == self.driver_name():
382             if self.get_option('leaks'):
383                 env['MallocStackLogging'] = '1'
384                 env['__XPC_MallocStackLogging'] = '1'
385                 env['MallocScribble'] = '1'
386                 env['__XPC_MallocScribble'] = '1'
387             if self.get_option('guard_malloc'):
388                 self._append_value_colon_separated(env, 'DYLD_INSERT_LIBRARIES', '/usr/lib/libgmalloc.dylib')
389                 self._append_value_colon_separated(env, '__XPC_DYLD_INSERT_LIBRARIES', '/usr/lib/libgmalloc.dylib')
390         env['XML_CATALOG_FILES'] = ''  # work around missing /etc/catalog <rdar://problem/4292995>
391         return env
392
393     def operating_system(self):
394         return 'ios-simulator'
395
396     def check_sys_deps(self, needs_http):
397         if not self.simulator_runtime.available:
398             _log.error('The iOS Simulator runtime with identifier "{0}" cannot be used because it is unavailable.'.format(self.simulator_runtime.identifier))
399             return False
400         return super(IOSSimulatorPort, self).check_sys_deps(needs_http)
401
402     SUBPROCESS_CRASH_REGEX = re.compile('#CRASHED - (?P<subprocess_name>\S+) \(pid (?P<subprocess_pid>\d+)\)')
403
404     def _using_dedicated_simulators(self):
405         return self.get_option('dedicated_simulators')
406
407     def _create_device(self, number):
408         return Simulator.create_device(number, self.simulator_device_type(), self.simulator_runtime)
409
410     def _remove_device(self, number):
411         Simulator.remove_device(number)
412
413     def _testing_device(self, number):
414         return Simulator.device_number(number)
415
416     # This is only exposed so that IOSSimulatorDriver can use it.
417     def device_id_for_worker_number(self, number):
418         if self._printing_cmd_line:
419             return '<dummy id>'
420         if self._using_dedicated_simulators():
421             return self._testing_device(number).udid
422         return self._current_device.udid
423
424     def get_simulator_path(self, suffix=""):
425         return os.path.join(self.SIMULATOR_DIRECTORY, "Simulator" + str(suffix) + ".app")
426
427     def diff_image(self, expected_contents, actual_contents, tolerance=None):
428         if not actual_contents and not expected_contents:
429             return (None, 0, None)
430         if not actual_contents or not expected_contents:
431             return (True, 0, None)
432         if not self._image_differ:
433             self._image_differ = image_diff.IOSSimulatorImageDiffer(self)
434         self.set_option_default('tolerance', 0.1)
435         if tolerance is None:
436             tolerance = self.get_option('tolerance')
437         return self._image_differ.diff_image(expected_contents, actual_contents, tolerance)
438
439     def reset_preferences(self):
440         _log.debug("reset_preferences")
441         self._quit_ios_simulator()
442         # Maybe this should delete all devices that we've created?
443
444     def nm_command(self):
445         return self.xcrun_find('nm')
446
447     @property
448     @memoized
449     def developer_dir(self):
450         return self._executive.run_command(['xcode-select', '--print-path']).rstrip()
451
452     def logging_patterns_to_strip(self):
453         return []
454
455     def stderr_patterns_to_strip(self):
456         return []
457
458     def _createSimulatorApps(self):
459         for i in xrange(self.child_processes()):
460             self._createSimulatorApp(i)
461
462     def _createSimulatorApp(self, suffix):
463         destination = self.get_simulator_path(suffix)
464         _log.info("Creating app:" + destination)
465         if os.path.exists(destination):
466             shutil.rmtree(destination, ignore_errors=True)
467         simulator_app_path = self.developer_dir + "/Applications/Simulator.app"
468         shutil.copytree(simulator_app_path, destination)
469
470         # Update app's package-name inside plist and re-code-sign it
471         plist_path = destination + "/Contents/Info.plist"
472         command = "Set CFBundleIdentifier com.apple.iphonesimulator" + str(suffix)
473         subprocess.check_output(["/usr/libexec/PlistBuddy", "-c", command, plist_path])
474         subprocess.check_output(["install_name_tool", "-add_rpath", self.developer_dir + "/Library/PrivateFrameworks/", destination + "/Contents/MacOS/Simulator"])
475         subprocess.check_output(["install_name_tool", "-add_rpath", self.developer_dir + "/../Frameworks/", destination + "/Contents/MacOS/Simulator"])
476         subprocess.check_output(["codesign", "-fs", "-", destination])
477         subprocess.check_output([self.LSREGISTER_PATH, "-f", destination])