webkitpy: Teardown iOS Simulators on exit if managed Simulators are still running
[WebKit-https.git] / Tools / Scripts / webkitpy / port / ios_simulator.py
1 # Copyright (C) 2014-2017 Apple Inc. All rights reserved.
2 #
3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions
5 # are met:
6 # 1.  Redistributions of source code must retain the above copyright
7 #     notice, this list of conditions and the following disclaimer.
8 # 2.  Redistributions in binary form must reproduce the above copyright
9 #     notice, this list of conditions and the following disclaimer in the
10 #     documentation and/or other materials provided with the distribution.
11 #
12 # THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND
13 # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
14 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
15 # DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR
16 # ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
17 # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
18 # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
19 # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
20 # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
21 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
22
23 import atexit
24 import logging
25 import os
26 import re
27 import shutil
28 import subprocess
29 import time
30
31 from webkitpy.common.memoized import memoized
32 from webkitpy.port import image_diff
33 from webkitpy.port.device import Device
34 from webkitpy.port.ios import IOSPort
35 from webkitpy.xcode.simulator import Simulator, Runtime, DeviceType
36 from webkitpy.common.system.crashlogs import CrashLogs
37
38
39 _log = logging.getLogger(__name__)
40
41
42 class IOSSimulatorPort(IOSPort):
43     port_name = "ios-simulator"
44
45     FUTURE_VERSION = 'future'
46     ARCHITECTURES = ['x86_64', 'x86']
47     DEFAULT_ARCHITECTURE = 'x86_64'
48
49     DEFAULT_DEVICE_CLASS = 'iphone'
50     CUSTOM_DEVICE_CLASSES = ['ipad', 'iphone7']
51     SDK = 'iphonesimulator'
52
53     SIMULATOR_BUNDLE_ID = 'com.apple.iphonesimulator'
54     SIMULATOR_DIRECTORY = "/tmp/WebKitTestingSimulators/"
55     LSREGISTER_PATH = "/System/Library/Frameworks/CoreServices.framework/Frameworks/LaunchServices.framework/Versions/Current/Support/lsregister"
56     PROCESS_COUNT_ESTIMATE_PER_SIMULATOR_INSTANCE = 125
57
58     DEVICE_CLASS_MAP = {
59         'x86_64': {
60             'iphone': 'iPhone 5s',
61             'iphone7': 'iPhone 7',
62             'ipad': 'iPad Air',
63         },
64         'x86': {
65             'iphone': 'iPhone 5',
66             'ipad': 'iPad Retina',
67         },
68     }
69
70     #FIXME: Ports are recreated in each process. This is a problem for IOSSimulatorPort, it means devices are not
71     # persistent and devices hold a listening socket expected to be persistent across processes.
72     _DEVICE_MAP = {}
73     _CURRENT_DEVICE = None
74
75     def __init__(self, host, port_name, **kwargs):
76         super(IOSSimulatorPort, self).__init__(host, port_name, **kwargs)
77
78         optional_device_class = self.get_option('device_class')
79         self._device_class = optional_device_class if optional_device_class else self.DEFAULT_DEVICE_CLASS
80         _log.debug('IOSSimulatorPort _device_class is %s', self._device_class)
81
82         if not IOSSimulatorPort._CURRENT_DEVICE:
83             IOSSimulatorPort._CURRENT_DEVICE = Device(Simulator(host).current_device())
84         self._current_device = IOSSimulatorPort._CURRENT_DEVICE
85         if not self._current_device:
86             self.set_option('dedicated_simulators', True)
87         if not self.get_option('dedicated_simulators'):
88             if self.get_option('child_processes') > 1:
89                 _log.warn('Cannot have more than one child process when using a running simulator.  Setting child_processes to 1.')
90             self.set_option('child_processes', 1)
91
92     def _device_for_worker_number_map(self):
93         return IOSSimulatorPort._DEVICE_MAP
94
95     @property
96     @memoized
97     def simulator_runtime(self):
98         runtime_identifier = self.get_option('runtime')
99         if runtime_identifier:
100             runtime = Runtime.from_identifier(runtime_identifier)
101         else:
102             runtime = Runtime.from_version_string(self.host.platform.xcode_sdk_version('iphonesimulator'))
103         return runtime
104
105     def simulator_device_type(self):
106         device_type_identifier = self.get_option('device_type')
107         if device_type_identifier:
108             _log.debug('simulator_device_type for device identifier %s', device_type_identifier)
109             device_type = DeviceType.from_identifier(device_type_identifier)
110         else:
111             _log.debug('simulator_device_type for device %s', self._device_class)
112             device_name = self.DEVICE_CLASS_MAP[self.architecture()][self._device_class]
113             if not device_name:
114                 raise Exception('Failed to find device for architecture {} and device class {}'.format(self.architecture()), self._device_class)
115             device_type = DeviceType.from_name(device_name)
116         return device_type
117
118     @memoized
119     def default_child_processes(self):
120         """Return the number of Simulators instances to use for this port."""
121         best_child_process_count_for_cpu = self._executive.cpu_count() / 2
122         system_process_count_limit = int(subprocess.check_output(["ulimit", "-u"]).strip())
123         current_process_count = len(subprocess.check_output(["ps", "aux"]).strip().split('\n'))
124         _log.debug('Process limit: %d, current #processes: %d' % (system_process_count_limit, current_process_count))
125         maximum_simulator_count_on_this_system = (system_process_count_limit - current_process_count) // self.PROCESS_COUNT_ESTIMATE_PER_SIMULATOR_INSTANCE
126         # FIXME: We should also take into account the available RAM.
127
128         if (maximum_simulator_count_on_this_system < best_child_process_count_for_cpu):
129             _log.warn("This machine could support %s simulators, but is only configured for %s."
130                 % (best_child_process_count_for_cpu, maximum_simulator_count_on_this_system))
131             _log.warn('Please see <https://trac.webkit.org/wiki/IncreasingKernelLimits>.')
132
133         if maximum_simulator_count_on_this_system == 0:
134             maximum_simulator_count_on_this_system = 1
135
136         return min(maximum_simulator_count_on_this_system, best_child_process_count_for_cpu)
137
138     def _get_crash_log(self, name, pid, stdout, stderr, newer_than, time_fn=time.time, sleep_fn=time.sleep, wait_for_log=True):
139         time_fn = time_fn or time.time
140         sleep_fn = sleep_fn or time.sleep
141
142         # FIXME: We should collect the actual crash log for DumpRenderTree.app because it includes more
143         # information (e.g. exception codes) than is available in the stack trace written to standard error.
144         stderr_lines = []
145         crashed_subprocess_name_and_pid = None  # e.g. ('DumpRenderTree.app', 1234)
146         for line in (stderr or '').splitlines():
147             if not crashed_subprocess_name_and_pid:
148                 match = self.SUBPROCESS_CRASH_REGEX.match(line)
149                 if match:
150                     crashed_subprocess_name_and_pid = (match.group('subprocess_name'), int(match.group('subprocess_pid')))
151                     continue
152             stderr_lines.append(line)
153
154         if crashed_subprocess_name_and_pid:
155             return self._get_crash_log(crashed_subprocess_name_and_pid[0], crashed_subprocess_name_and_pid[1], stdout,
156                 '\n'.join(stderr_lines), newer_than, time_fn, sleep_fn, wait_for_log)
157
158         # App crashed
159         _log.debug('looking for crash log for %s:%s' % (name, str(pid)))
160         crash_log = ''
161         crash_logs = CrashLogs(self.host)
162         now = time_fn()
163         deadline = now + 5 * int(self.get_option('child_processes', 1))
164         while not crash_log and now <= deadline:
165             crash_log = crash_logs.find_newest_log(name, pid, include_errors=True, newer_than=newer_than)
166             if not wait_for_log:
167                 break
168             if not crash_log or not [line for line in crash_log.splitlines() if not line.startswith('ERROR')]:
169                 sleep_fn(0.1)
170                 now = time_fn()
171
172         if not crash_log:
173             return stderr, None
174         return stderr, crash_log
175
176     def _build_driver_flags(self):
177         archs = ['ARCHS=i386'] if self.architecture() == 'x86' else []
178         sdk = ['--sdk', 'iphonesimulator']
179         return archs + sdk
180
181     def _set_device_class(self, device_class):
182         self._device_class = device_class if device_class else self.DEFAULT_DEVICE_CLASS
183
184     # This function may be called more than once.
185     def _teardown_managed_simulators(self):
186         if not self._using_dedicated_simulators():
187             return
188         self._quit_ios_simulator()
189
190         for i in xrange(len(Simulator.managed_devices)):
191             simulator_path = self.get_simulator_path(i)
192             device_udid = Simulator.managed_devices[i].udid
193             Simulator.remove_device(i)
194
195             if not os.path.exists(simulator_path):
196                 continue
197             try:
198                 self._executive.run_command([IOSSimulatorPort.LSREGISTER_PATH, "-u", simulator_path])
199
200                 _log.debug('rmtree %s', simulator_path)
201                 self._filesystem.rmtree(simulator_path)
202
203                 logs_path = self._filesystem.join(self._filesystem.expanduser("~"), "Library/Logs/CoreSimulator/", device_udid)
204                 _log.debug('rmtree %s', logs_path)
205                 self._filesystem.rmtree(logs_path)
206
207                 saved_state_path = self._filesystem.join(self._filesystem.expanduser("~"), "Library/Saved Application State/", IOSSimulatorPort.SIMULATOR_BUNDLE_ID + str(i) + ".savedState")
208                 _log.debug('rmtree %s', saved_state_path)
209                 self._filesystem.rmtree(saved_state_path)
210             except:
211                 _log.warning('Unable to remove Simulator' + str(i))
212
213     def _create_simulators(self):
214         if (self.default_child_processes() < self.child_processes()):
215             _log.warn('You have specified very high value({0}) for --child-processes'.format(self.child_processes()))
216             _log.warn('maximum child-processes which can be supported on this system are: {0}'.format(self.default_child_processes()))
217             _log.warn('This is very likely to fail.')
218
219         if self._using_dedicated_simulators():
220             atexit.register(lambda: self._teardown_managed_simulators())
221             self._createSimulatorApps()
222
223             for i in xrange(self.child_processes()):
224                 self._create_device(i)
225
226             for i in xrange(self.child_processes()):
227                 device_udid = Simulator.managed_devices[i].udid
228                 Simulator.wait_until_device_is_in_state(device_udid, Simulator.DeviceState.SHUTDOWN)
229                 Simulator.reset_device(device_udid)
230         else:
231             assert(self._current_device)
232             if self._current_device.platform_device.name != self.simulator_device_type().name:
233                 _log.warn("Expected simulator of type '" + self.simulator_device_type().name + "' but found simulator of type '" + self._current_device.platform_device.name + "'")
234                 _log.warn('The next block of tests may fail due to device mis-match')
235
236     def _create_devices(self, device_class):
237         mac_os_version = self.host.platform.os_version
238
239         self._set_device_class(device_class)
240
241         _log.debug('')
242         _log.debug('setup_test_run for %s', self._device_class)
243
244         self._create_simulators()
245
246         if not self._using_dedicated_simulators():
247             return
248
249         for i in xrange(self.child_processes()):
250             device_udid = Simulator.managed_devices[i].udid
251             _log.debug('testing device %s has udid %s', i, device_udid)
252
253             # FIXME: <rdar://problem/20916140> Switch to using CoreSimulator.framework for launching and quitting iOS Simulator
254             self._executive.run_command([
255                 'open', '-g', '-b', self.SIMULATOR_BUNDLE_ID + str(i),
256                 '--args', '-CurrentDeviceUDID', device_udid])
257
258             if mac_os_version in ['elcapitan', 'yosemite', 'mavericks']:
259                 time.sleep(2.5)
260
261         _log.info('Waiting for all iOS Simulators to finish booting.')
262         for i in xrange(self.child_processes()):
263             Simulator.wait_until_device_is_booted(Simulator.managed_devices[i].udid)
264
265         IOSSimulatorPort._DEVICE_MAP = {}
266         for i in xrange(self.child_processes()):
267             IOSSimulatorPort._DEVICE_MAP[i] = Device(Simulator.managed_devices[i])
268
269     def _quit_ios_simulator(self):
270         if not self._using_dedicated_simulators():
271             return
272         _log.debug("_quit_ios_simulator killing all Simulator processes")
273         # FIXME: We should kill only the Simulators we started.
274         subprocess.call(["killall", "-9", "-m", "Simulator"])
275
276     def clean_up_test_run(self):
277         super(IOSSimulatorPort, self).clean_up_test_run()
278         _log.debug("clean_up_test_run")
279         fifos = [path for path in os.listdir('/tmp') if re.search('org.webkit.(DumpRenderTree|WebKitTestRunner).*_(IN|OUT|ERROR)', path)]
280         for fifo in fifos:
281             try:
282                 os.remove(os.path.join('/tmp', fifo))
283             except OSError:
284                 _log.warning('Unable to remove ' + fifo)
285                 pass
286
287         if not self._using_dedicated_simulators():
288             return
289
290         self._teardown_managed_simulators()
291         IOSSimulatorPort._DEVICE_MAP = {}
292
293     def setup_environ_for_server(self, server_name=None):
294         _log.debug("setup_environ_for_server")
295         env = super(IOSSimulatorPort, self).setup_environ_for_server(server_name)
296         if server_name == self.driver_name():
297             if self.get_option('leaks'):
298                 env['MallocStackLogging'] = '1'
299                 env['__XPC_MallocStackLogging'] = '1'
300                 env['MallocScribble'] = '1'
301                 env['__XPC_MallocScribble'] = '1'
302             if self.get_option('guard_malloc'):
303                 self._append_value_colon_separated(env, 'DYLD_INSERT_LIBRARIES', '/usr/lib/libgmalloc.dylib')
304                 self._append_value_colon_separated(env, '__XPC_DYLD_INSERT_LIBRARIES', '/usr/lib/libgmalloc.dylib')
305         env['XML_CATALOG_FILES'] = ''  # work around missing /etc/catalog <rdar://problem/4292995>
306         return env
307
308     def operating_system(self):
309         return 'ios-simulator'
310
311     def check_sys_deps(self, needs_http):
312         if not self.simulator_runtime.available:
313             _log.error('The iOS Simulator runtime with identifier "{0}" cannot be used because it is unavailable.'.format(self.simulator_runtime.identifier))
314             return False
315         return super(IOSSimulatorPort, self).check_sys_deps(needs_http)
316
317     SUBPROCESS_CRASH_REGEX = re.compile('#CRASHED - (?P<subprocess_name>\S+) \(pid (?P<subprocess_pid>\d+)\)')
318
319     def _using_dedicated_simulators(self):
320         return self.get_option('dedicated_simulators')
321
322     def using_multiple_devices(self):
323         return self._using_dedicated_simulators()
324
325     def _create_device(self, number):
326         return Simulator.create_device(number, self.simulator_device_type(), self.simulator_runtime)
327
328     def get_simulator_path(self, suffix=""):
329         return os.path.join(self.SIMULATOR_DIRECTORY, "Simulator" + str(suffix) + ".app")
330
331     def diff_image(self, expected_contents, actual_contents, tolerance=None):
332         if not actual_contents and not expected_contents:
333             return (None, 0, None)
334         if not actual_contents or not expected_contents:
335             return (True, 0, None)
336         if not self._image_differ:
337             self._image_differ = image_diff.IOSSimulatorImageDiffer(self)
338         self.set_option_default('tolerance', 0.1)
339         if tolerance is None:
340             tolerance = self.get_option('tolerance')
341         return self._image_differ.diff_image(expected_contents, actual_contents, tolerance)
342
343     def reset_preferences(self):
344         _log.debug("reset_preferences")
345         self._quit_ios_simulator()
346         # Maybe this should delete all devices that we've created?
347
348     def nm_command(self):
349         return self.xcrun_find('nm')
350
351     @property
352     @memoized
353     def developer_dir(self):
354         return self._executive.run_command(['xcode-select', '--print-path']).rstrip()
355
356     def logging_patterns_to_strip(self):
357         return []
358
359     def stderr_patterns_to_strip(self):
360         return []
361
362     def _createSimulatorApps(self):
363         for i in xrange(self.child_processes()):
364             self._createSimulatorApp(i)
365
366     def _createSimulatorApp(self, suffix):
367         destination = self.get_simulator_path(suffix)
368         _log.info("Creating app:" + destination)
369         if os.path.exists(destination):
370             shutil.rmtree(destination, ignore_errors=True)
371         simulator_app_path = self.developer_dir + "/Applications/Simulator.app"
372         shutil.copytree(simulator_app_path, destination)
373
374         # Update app's package-name inside plist and re-code-sign it
375         plist_path = destination + "/Contents/Info.plist"
376         command = "Set CFBundleIdentifier com.apple.iphonesimulator" + str(suffix)
377         subprocess.check_output(["/usr/libexec/PlistBuddy", "-c", command, plist_path])
378         subprocess.check_output(["install_name_tool", "-add_rpath", self.developer_dir + "/Library/PrivateFrameworks/", destination + "/Contents/MacOS/Simulator"])
379         subprocess.check_output(["install_name_tool", "-add_rpath", self.developer_dir + "/../Frameworks/", destination + "/Contents/MacOS/Simulator"])
380         subprocess.check_output(["codesign", "-fs", "-", destination])
381         subprocess.check_output([self.LSREGISTER_PATH, "-f", destination])