REGRESSION: Reseting _device_map on each child process
[WebKit-https.git] / Tools / Scripts / webkitpy / port / ios_simulator.py
1 # Copyright (C) 2014-2017 Apple Inc. All rights reserved.
2 #
3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions
5 # are met:
6 # 1.  Redistributions of source code must retain the above copyright
7 #     notice, this list of conditions and the following disclaimer.
8 # 2.  Redistributions in binary form must reproduce the above copyright
9 #     notice, this list of conditions and the following disclaimer in the
10 #     documentation and/or other materials provided with the distribution.
11 #
12 # THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND
13 # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
14 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
15 # DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR
16 # ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
17 # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
18 # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
19 # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
20 # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
21 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
22
23 import logging
24 import os
25 import re
26 import shutil
27 import subprocess
28 import time
29
30 from webkitpy.common.memoized import memoized
31 from webkitpy.layout_tests.models.test_configuration import TestConfiguration
32 from webkitpy.port import image_diff
33 from webkitpy.port.device import Device
34 from webkitpy.port.ios import IOSPort
35 from webkitpy.xcode.simulator import Simulator, Runtime, DeviceType
36 from webkitpy.common.system.crashlogs import CrashLogs
37
38
39 _log = logging.getLogger(__name__)
40
41
42 class IOSSimulatorPort(IOSPort):
43     port_name = "ios-simulator"
44
45     FUTURE_VERSION = 'future'
46     ARCHITECTURES = ['x86_64', 'x86']
47     DEFAULT_ARCHITECTURE = 'x86_64'
48
49     DEFAULT_DEVICE_CLASS = 'iphone'
50     CUSTOM_DEVICE_CLASSES = ['ipad', 'iphone7']
51     SDK = 'iphonesimulator'
52
53     SIMULATOR_BUNDLE_ID = 'com.apple.iphonesimulator'
54     SIMULATOR_DIRECTORY = "/tmp/WebKitTestingSimulators/"
55     LSREGISTER_PATH = "/System/Library/Frameworks/CoreServices.framework/Frameworks/LaunchServices.framework/Versions/Current/Support/lsregister"
56     PROCESS_COUNT_ESTIMATE_PER_SIMULATOR_INSTANCE = 100
57
58     DEVICE_CLASS_MAP = {
59         'x86_64': {
60             'iphone': 'iPhone 5s',
61             'iphone7': 'iPhone 7',
62             'ipad': 'iPad Air',
63         },
64         'x86': {
65             'iphone': 'iPhone 5',
66             'ipad': 'iPad Retina',
67         },
68     }
69
70     _DEVICE_MAP = {}
71
72     def __init__(self, host, port_name, **kwargs):
73         super(IOSSimulatorPort, self).__init__(host, port_name, **kwargs)
74
75         optional_device_class = self.get_option('device_class')
76         self._device_class = optional_device_class if optional_device_class else self.DEFAULT_DEVICE_CLASS
77         _log.debug('IOSSimulatorPort _device_class is %s', self._device_class)
78
79         self._current_device = Device(Simulator(host).current_device())
80         if not self._current_device:
81             self.set_option('dedicated_simulators', True)
82         if not self.get_option('dedicated_simulators'):
83             if self.get_option('child_processes') > 1:
84                 _log.warn('Cannot have more than one child process when using a running simulator.  Setting child_processes to 1.')
85             self.set_option('child_processes', 1)
86
87     def _device_for_worker_number_map(self):
88         return IOSSimulatorPort._DEVICE_MAP
89
90     @property
91     @memoized
92     def simulator_runtime(self):
93         runtime_identifier = self.get_option('runtime')
94         if runtime_identifier:
95             runtime = Runtime.from_identifier(runtime_identifier)
96         else:
97             runtime = Runtime.from_version_string(self.host.platform.xcode_sdk_version('iphonesimulator'))
98         return runtime
99
100     def simulator_device_type(self):
101         device_type_identifier = self.get_option('device_type')
102         if device_type_identifier:
103             _log.debug('simulator_device_type for device identifier %s', device_type_identifier)
104             device_type = DeviceType.from_identifier(device_type_identifier)
105         else:
106             _log.debug('simulator_device_type for device %s', self._device_class)
107             device_name = self.DEVICE_CLASS_MAP[self.architecture()][self._device_class]
108             if not device_name:
109                 raise Exception('Failed to find device for architecture {} and device class {}'.format(self.architecture()), self._device_class)
110             device_type = DeviceType.from_name(device_name)
111         return device_type
112
113     @memoized
114     def default_child_processes(self):
115         """Return the number of Simulators instances to use for this port."""
116         best_child_process_count_for_cpu = self._executive.cpu_count() / 2
117         system_process_count_limit = int(subprocess.check_output(["ulimit", "-u"]).strip())
118         current_process_count = len(subprocess.check_output(["ps", "aux"]).strip().split('\n'))
119         _log.debug('Process limit: %d, current #processes: %d' % (system_process_count_limit, current_process_count))
120         maximum_simulator_count_on_this_system = (system_process_count_limit - current_process_count) // self.PROCESS_COUNT_ESTIMATE_PER_SIMULATOR_INSTANCE
121         # FIXME: We should also take into account the available RAM.
122
123         if (maximum_simulator_count_on_this_system < best_child_process_count_for_cpu):
124             _log.warn("This machine could support %s simulators, but is only configured for %s."
125                 % (best_child_process_count_for_cpu, maximum_simulator_count_on_this_system))
126             _log.warn('Please see <https://trac.webkit.org/wiki/IncreasingKernelLimits>.')
127
128         if maximum_simulator_count_on_this_system == 0:
129             maximum_simulator_count_on_this_system = 1
130
131         return min(maximum_simulator_count_on_this_system, best_child_process_count_for_cpu)
132
133     def _get_crash_log(self, name, pid, stdout, stderr, newer_than, time_fn=time.time, sleep_fn=time.sleep, wait_for_log=True):
134         time_fn = time_fn or time.time
135         sleep_fn = sleep_fn or time.sleep
136
137         # FIXME: We should collect the actual crash log for DumpRenderTree.app because it includes more
138         # information (e.g. exception codes) than is available in the stack trace written to standard error.
139         stderr_lines = []
140         crashed_subprocess_name_and_pid = None  # e.g. ('DumpRenderTree.app', 1234)
141         for line in (stderr or '').splitlines():
142             if not crashed_subprocess_name_and_pid:
143                 match = self.SUBPROCESS_CRASH_REGEX.match(line)
144                 if match:
145                     crashed_subprocess_name_and_pid = (match.group('subprocess_name'), int(match.group('subprocess_pid')))
146                     continue
147             stderr_lines.append(line)
148
149         if crashed_subprocess_name_and_pid:
150             return self._get_crash_log(crashed_subprocess_name_and_pid[0], crashed_subprocess_name_and_pid[1], stdout,
151                 '\n'.join(stderr_lines), newer_than, time_fn, sleep_fn, wait_for_log)
152
153         # App crashed
154         _log.debug('looking for crash log for %s:%s' % (name, str(pid)))
155         crash_log = ''
156         crash_logs = CrashLogs(self.host)
157         now = time_fn()
158         deadline = now + 5 * int(self.get_option('child_processes', 1))
159         while not crash_log and now <= deadline:
160             crash_log = crash_logs.find_newest_log(name, pid, include_errors=True, newer_than=newer_than)
161             if not wait_for_log:
162                 break
163             if not crash_log or not [line for line in crash_log.splitlines() if not line.startswith('ERROR')]:
164                 sleep_fn(0.1)
165                 now = time_fn()
166
167         if not crash_log:
168             return stderr, None
169         return stderr, crash_log
170
171     def _build_driver_flags(self):
172         archs = ['ARCHS=i386'] if self.architecture() == 'x86' else []
173         sdk = ['--sdk', 'iphonesimulator']
174         return archs + sdk
175
176     def _generate_all_test_configurations(self):
177         configurations = []
178         for build_type in self.ALL_BUILD_TYPES:
179             for architecture in self.ARCHITECTURES:
180                 configurations.append(TestConfiguration(version=self._version, architecture=architecture, build_type=build_type))
181         return configurations
182
183     def default_baseline_search_path(self):
184         if self.get_option('webkit_test_runner'):
185             fallback_names = [self._wk2_port_name()] + [self.port_name] + ['wk2']
186         else:
187             fallback_names = [self.port_name + '-wk1'] + [self.port_name]
188
189         return map(self._webkit_baseline_path, fallback_names)
190
191     def _set_device_class(self, device_class):
192         self._device_class = device_class if device_class else self.DEFAULT_DEVICE_CLASS
193
194     def _create_simulators(self):
195         if (self.default_child_processes() < self.child_processes()):
196             _log.warn('You have specified very high value({0}) for --child-processes'.format(self.child_processes()))
197             _log.warn('maximum child-processes which can be supported on this system are: {0}'.format(self.default_child_processes()))
198             _log.warn('This is very likely to fail.')
199
200         if self._using_dedicated_simulators():
201             self._createSimulatorApps()
202
203             for i in xrange(self.child_processes()):
204                 self._create_device(i)
205
206             for i in xrange(self.child_processes()):
207                 device_udid = Simulator.managed_devices[i].udid
208                 Simulator.wait_until_device_is_in_state(device_udid, Simulator.DeviceState.SHUTDOWN)
209                 Simulator.reset_device(device_udid)
210         else:
211             assert(self._current_device)
212             if self._current_device.platform_device.name != self.simulator_device_type().name:
213                 _log.warn("Expected simulator of type '" + self.simulator_device_type().name + "' but found simulator of type '" + self._current_device.platform_device.name + "'")
214                 _log.warn('The next block of tests may fail due to device mis-match')
215
216     def _create_devices(self, device_class):
217         mac_os_version = self.host.platform.os_version
218
219         self._set_device_class(device_class)
220
221         _log.debug('')
222         _log.debug('setup_test_run for %s', self._device_class)
223
224         self._create_simulators()
225
226         if not self._using_dedicated_simulators():
227             return
228
229         for i in xrange(self.child_processes()):
230             device_udid = Simulator.managed_devices[i].udid
231             _log.debug('testing device %s has udid %s', i, device_udid)
232
233             # FIXME: <rdar://problem/20916140> Switch to using CoreSimulator.framework for launching and quitting iOS Simulator
234             self._executive.run_command([
235                 'open', '-g', '-b', self.SIMULATOR_BUNDLE_ID + str(i),
236                 '--args', '-CurrentDeviceUDID', device_udid])
237
238             if mac_os_version in ['elcapitan', 'yosemite', 'mavericks']:
239                 time.sleep(2.5)
240
241         _log.info('Waiting for all iOS Simulators to finish booting.')
242         for i in xrange(self.child_processes()):
243             Simulator.wait_until_device_is_booted(Simulator.managed_devices[i].udid)
244
245         IOSSimulatorPort._DEVICE_MAP = {}
246         for i in xrange(self.child_processes()):
247             IOSSimulatorPort._DEVICE_MAP[i] = Device(Simulator.managed_devices[i])
248
249     def _quit_ios_simulator(self):
250         if not self._using_dedicated_simulators():
251             return
252         _log.debug("_quit_ios_simulator killing all Simulator processes")
253         # FIXME: We should kill only the Simulators we started.
254         subprocess.call(["killall", "-9", "-m", "Simulator"])
255
256     def clean_up_test_run(self):
257         super(IOSSimulatorPort, self).clean_up_test_run()
258         _log.debug("clean_up_test_run")
259         self._quit_ios_simulator()
260         fifos = [path for path in os.listdir('/tmp') if re.search('org.webkit.(DumpRenderTree|WebKitTestRunner).*_(IN|OUT|ERROR)', path)]
261         for fifo in fifos:
262             try:
263                 os.remove(os.path.join('/tmp', fifo))
264             except OSError:
265                 _log.warning('Unable to remove ' + fifo)
266                 pass
267
268         if not self._using_dedicated_simulators():
269             return
270
271         for i in xrange(self.child_processes()):
272             simulator_path = self.get_simulator_path(i)
273             device_udid = self._testing_device(i).udid
274             self._remove_device(i)
275
276             if not os.path.exists(simulator_path):
277                 continue
278             try:
279                 self._executive.run_command([self.LSREGISTER_PATH, "-u", simulator_path])
280
281                 _log.debug('rmtree %s', simulator_path)
282                 self._filesystem.rmtree(simulator_path)
283
284                 logs_path = self._filesystem.join(self._filesystem.expanduser("~"), "Library/Logs/CoreSimulator/", device_udid)
285                 _log.debug('rmtree %s', logs_path)
286                 self._filesystem.rmtree(logs_path)
287
288                 saved_state_path = self._filesystem.join(self._filesystem.expanduser("~"), "Library/Saved Application State/", self.SIMULATOR_BUNDLE_ID + str(i) + ".savedState")
289                 _log.debug('rmtree %s', saved_state_path)
290                 self._filesystem.rmtree(saved_state_path)
291
292             except:
293                 _log.warning('Unable to remove Simulator' + str(i))
294         IOSSimulatorPort._DEVICE_MAP = {}
295
296     def setup_environ_for_server(self, server_name=None):
297         _log.debug("setup_environ_for_server")
298         env = super(IOSSimulatorPort, self).setup_environ_for_server(server_name)
299         if server_name == self.driver_name():
300             if self.get_option('leaks'):
301                 env['MallocStackLogging'] = '1'
302                 env['__XPC_MallocStackLogging'] = '1'
303                 env['MallocScribble'] = '1'
304                 env['__XPC_MallocScribble'] = '1'
305             if self.get_option('guard_malloc'):
306                 self._append_value_colon_separated(env, 'DYLD_INSERT_LIBRARIES', '/usr/lib/libgmalloc.dylib')
307                 self._append_value_colon_separated(env, '__XPC_DYLD_INSERT_LIBRARIES', '/usr/lib/libgmalloc.dylib')
308         env['XML_CATALOG_FILES'] = ''  # work around missing /etc/catalog <rdar://problem/4292995>
309         return env
310
311     def operating_system(self):
312         return 'ios-simulator'
313
314     def check_sys_deps(self, needs_http):
315         if not self.simulator_runtime.available:
316             _log.error('The iOS Simulator runtime with identifier "{0}" cannot be used because it is unavailable.'.format(self.simulator_runtime.identifier))
317             return False
318         return super(IOSSimulatorPort, self).check_sys_deps(needs_http)
319
320     SUBPROCESS_CRASH_REGEX = re.compile('#CRASHED - (?P<subprocess_name>\S+) \(pid (?P<subprocess_pid>\d+)\)')
321
322     def _using_dedicated_simulators(self):
323         return self.get_option('dedicated_simulators')
324
325     def using_multiple_devices(self):
326         return self._using_dedicated_simulators()
327
328     def _create_device(self, number):
329         return Simulator.create_device(number, self.simulator_device_type(), self.simulator_runtime)
330
331     def _remove_device(self, number):
332         Simulator.remove_device(number)
333
334     def get_simulator_path(self, suffix=""):
335         return os.path.join(self.SIMULATOR_DIRECTORY, "Simulator" + str(suffix) + ".app")
336
337     def diff_image(self, expected_contents, actual_contents, tolerance=None):
338         if not actual_contents and not expected_contents:
339             return (None, 0, None)
340         if not actual_contents or not expected_contents:
341             return (True, 0, None)
342         if not self._image_differ:
343             self._image_differ = image_diff.IOSSimulatorImageDiffer(self)
344         self.set_option_default('tolerance', 0.1)
345         if tolerance is None:
346             tolerance = self.get_option('tolerance')
347         return self._image_differ.diff_image(expected_contents, actual_contents, tolerance)
348
349     def reset_preferences(self):
350         _log.debug("reset_preferences")
351         self._quit_ios_simulator()
352         # Maybe this should delete all devices that we've created?
353
354     def nm_command(self):
355         return self.xcrun_find('nm')
356
357     @property
358     @memoized
359     def developer_dir(self):
360         return self._executive.run_command(['xcode-select', '--print-path']).rstrip()
361
362     def logging_patterns_to_strip(self):
363         return []
364
365     def stderr_patterns_to_strip(self):
366         return []
367
368     def _createSimulatorApps(self):
369         for i in xrange(self.child_processes()):
370             self._createSimulatorApp(i)
371
372     def _createSimulatorApp(self, suffix):
373         destination = self.get_simulator_path(suffix)
374         _log.info("Creating app:" + destination)
375         if os.path.exists(destination):
376             shutil.rmtree(destination, ignore_errors=True)
377         simulator_app_path = self.developer_dir + "/Applications/Simulator.app"
378         shutil.copytree(simulator_app_path, destination)
379
380         # Update app's package-name inside plist and re-code-sign it
381         plist_path = destination + "/Contents/Info.plist"
382         command = "Set CFBundleIdentifier com.apple.iphonesimulator" + str(suffix)
383         subprocess.check_output(["/usr/libexec/PlistBuddy", "-c", command, plist_path])
384         subprocess.check_output(["install_name_tool", "-add_rpath", self.developer_dir + "/Library/PrivateFrameworks/", destination + "/Contents/MacOS/Simulator"])
385         subprocess.check_output(["install_name_tool", "-add_rpath", self.developer_dir + "/../Frameworks/", destination + "/Contents/MacOS/Simulator"])
386         subprocess.check_output(["codesign", "-fs", "-", destination])
387         subprocess.check_output([self.LSREGISTER_PATH, "-f", destination])