webkitpy: Refactor setup_test_run for IOSPort and IOSSimulator
[WebKit-https.git] / Tools / Scripts / webkitpy / port / ios_simulator.py
1 # Copyright (C) 2014-2017 Apple Inc. All rights reserved.
2 #
3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions
5 # are met:
6 # 1.  Redistributions of source code must retain the above copyright
7 #     notice, this list of conditions and the following disclaimer.
8 # 2.  Redistributions in binary form must reproduce the above copyright
9 #     notice, this list of conditions and the following disclaimer in the
10 #     documentation and/or other materials provided with the distribution.
11 #
12 # THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND
13 # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
14 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
15 # DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR
16 # ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
17 # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
18 # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
19 # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
20 # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
21 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
22
23 import logging
24 import os
25 import re
26 import shutil
27 import subprocess
28 import time
29
30 from webkitpy.common.memoized import memoized
31 from webkitpy.layout_tests.models.test_configuration import TestConfiguration
32 from webkitpy.port import image_diff
33 from webkitpy.port.ios import IOSPort
34 from webkitpy.xcode.simulator import Simulator, Runtime, DeviceType
35 from webkitpy.common.system.crashlogs import CrashLogs
36
37
38 _log = logging.getLogger(__name__)
39
40
41 class IOSSimulatorPort(IOSPort):
42     port_name = "ios-simulator"
43
44     FUTURE_VERSION = 'future'
45     ARCHITECTURES = ['x86_64', 'x86']
46     DEFAULT_ARCHITECTURE = 'x86_64'
47
48     DEFAULT_DEVICE_CLASS = 'iphone'
49     CUSTOM_DEVICE_CLASSES = ['ipad', 'iphone7']
50     SDK = 'iphonesimulator'
51
52     SIMULATOR_BUNDLE_ID = 'com.apple.iphonesimulator'
53     SIMULATOR_DIRECTORY = "/tmp/WebKitTestingSimulators/"
54     LSREGISTER_PATH = "/System/Library/Frameworks/CoreServices.framework/Frameworks/LaunchServices.framework/Versions/Current/Support/lsregister"
55     PROCESS_COUNT_ESTIMATE_PER_SIMULATOR_INSTANCE = 100
56
57     DEVICE_CLASS_MAP = {
58         'x86_64': {
59             'iphone': 'iPhone 5s',
60             'iphone7': 'iPhone 7',
61             'ipad': 'iPad Air',
62         },
63         'x86': {
64             'iphone': 'iPhone 5',
65             'ipad': 'iPad Retina',
66         },
67     }
68
69     def __init__(self, host, port_name, **kwargs):
70         super(IOSSimulatorPort, self).__init__(host, port_name, **kwargs)
71
72         optional_device_class = self.get_option('device_class')
73         self._device_class = optional_device_class if optional_device_class else self.DEFAULT_DEVICE_CLASS
74         _log.debug('IOSSimulatorPort _device_class is %s', self._device_class)
75
76         self._current_device = Simulator(host).current_device()
77         if not self._current_device:
78             self.set_option('dedicated_simulators', True)
79         if not self.get_option('dedicated_simulators'):
80             if self.get_option('child_processes') > 1:
81                 _log.warn('Cannot have more than one child process when using a running simulator.  Setting child_processes to 1.')
82             self.set_option('child_processes', 1)
83
84     def _device_for_worker_number_map(self):
85         return Simulator.managed_devices
86
87     @property
88     @memoized
89     def simulator_runtime(self):
90         runtime_identifier = self.get_option('runtime')
91         if runtime_identifier:
92             runtime = Runtime.from_identifier(runtime_identifier)
93         else:
94             runtime = Runtime.from_version_string(self.host.platform.xcode_sdk_version('iphonesimulator'))
95         return runtime
96
97     def simulator_device_type(self):
98         device_type_identifier = self.get_option('device_type')
99         if device_type_identifier:
100             _log.debug('simulator_device_type for device identifier %s', device_type_identifier)
101             device_type = DeviceType.from_identifier(device_type_identifier)
102         else:
103             _log.debug('simulator_device_type for device %s', self._device_class)
104             device_name = self.DEVICE_CLASS_MAP[self.architecture()][self._device_class]
105             if not device_name:
106                 raise Exception('Failed to find device for architecture {} and device class {}'.format(self.architecture()), self._device_class)
107             device_type = DeviceType.from_name(device_name)
108         return device_type
109
110     @memoized
111     def default_child_processes(self):
112         """Return the number of Simulators instances to use for this port."""
113         best_child_process_count_for_cpu = self._executive.cpu_count() / 2
114         system_process_count_limit = int(subprocess.check_output(["ulimit", "-u"]).strip())
115         current_process_count = len(subprocess.check_output(["ps", "aux"]).strip().split('\n'))
116         _log.debug('Process limit: %d, current #processes: %d' % (system_process_count_limit, current_process_count))
117         maximum_simulator_count_on_this_system = (system_process_count_limit - current_process_count) // self.PROCESS_COUNT_ESTIMATE_PER_SIMULATOR_INSTANCE
118         # FIXME: We should also take into account the available RAM.
119
120         if (maximum_simulator_count_on_this_system < best_child_process_count_for_cpu):
121             _log.warn("This machine could support %s simulators, but is only configured for %s."
122                 % (best_child_process_count_for_cpu, maximum_simulator_count_on_this_system))
123             _log.warn('Please see <https://trac.webkit.org/wiki/IncreasingKernelLimits>.')
124
125         if maximum_simulator_count_on_this_system == 0:
126             maximum_simulator_count_on_this_system = 1
127
128         return min(maximum_simulator_count_on_this_system, best_child_process_count_for_cpu)
129
130     def _get_crash_log(self, name, pid, stdout, stderr, newer_than, time_fn=time.time, sleep_fn=time.sleep, wait_for_log=True):
131         time_fn = time_fn or time.time
132         sleep_fn = sleep_fn or time.sleep
133
134         # FIXME: We should collect the actual crash log for DumpRenderTree.app because it includes more
135         # information (e.g. exception codes) than is available in the stack trace written to standard error.
136         stderr_lines = []
137         crashed_subprocess_name_and_pid = None  # e.g. ('DumpRenderTree.app', 1234)
138         for line in (stderr or '').splitlines():
139             if not crashed_subprocess_name_and_pid:
140                 match = self.SUBPROCESS_CRASH_REGEX.match(line)
141                 if match:
142                     crashed_subprocess_name_and_pid = (match.group('subprocess_name'), int(match.group('subprocess_pid')))
143                     continue
144             stderr_lines.append(line)
145
146         if crashed_subprocess_name_and_pid:
147             return self._get_crash_log(crashed_subprocess_name_and_pid[0], crashed_subprocess_name_and_pid[1], stdout,
148                 '\n'.join(stderr_lines), newer_than, time_fn, sleep_fn, wait_for_log)
149
150         # App crashed
151         _log.debug('looking for crash log for %s:%s' % (name, str(pid)))
152         crash_log = ''
153         crash_logs = CrashLogs(self.host)
154         now = time_fn()
155         deadline = now + 5 * int(self.get_option('child_processes', 1))
156         while not crash_log and now <= deadline:
157             crash_log = crash_logs.find_newest_log(name, pid, include_errors=True, newer_than=newer_than)
158             if not wait_for_log:
159                 break
160             if not crash_log or not [line for line in crash_log.splitlines() if not line.startswith('ERROR')]:
161                 sleep_fn(0.1)
162                 now = time_fn()
163
164         if not crash_log:
165             return stderr, None
166         return stderr, crash_log
167
168     def _build_driver_flags(self):
169         archs = ['ARCHS=i386'] if self.architecture() == 'x86' else []
170         sdk = ['--sdk', 'iphonesimulator']
171         return archs + sdk
172
173     def _generate_all_test_configurations(self):
174         configurations = []
175         for build_type in self.ALL_BUILD_TYPES:
176             for architecture in self.ARCHITECTURES:
177                 configurations.append(TestConfiguration(version=self._version, architecture=architecture, build_type=build_type))
178         return configurations
179
180     def default_baseline_search_path(self):
181         if self.get_option('webkit_test_runner'):
182             fallback_names = [self._wk2_port_name()] + [self.port_name] + ['wk2']
183         else:
184             fallback_names = [self.port_name + '-wk1'] + [self.port_name]
185
186         return map(self._webkit_baseline_path, fallback_names)
187
188     def _set_device_class(self, device_class):
189         self._device_class = device_class if device_class else self.DEFAULT_DEVICE_CLASS
190
191     def _create_simulators(self):
192         if (self.default_child_processes() < self.child_processes()):
193             _log.warn('You have specified very high value({0}) for --child-processes'.format(self.child_processes()))
194             _log.warn('maximum child-processes which can be supported on this system are: {0}'.format(self.default_child_processes()))
195             _log.warn('This is very likely to fail.')
196
197         if self._using_dedicated_simulators():
198             self._createSimulatorApps()
199
200             for i in xrange(self.child_processes()):
201                 self._create_device(i)
202
203             for i in xrange(self.child_processes()):
204                 device_udid = self._testing_device(i).udid
205                 Simulator.wait_until_device_is_in_state(device_udid, Simulator.DeviceState.SHUTDOWN)
206                 Simulator.reset_device(device_udid)
207         else:
208             assert(self._current_device)
209             if self._current_device.name != self.simulator_device_type().name:
210                 _log.warn("Expected simulator of type '" + self.simulator_device_type().name + "' but found simulator of type '" + self._current_device.name + "'")
211                 _log.warn('The next block of tests may fail due to device mis-match')
212
213     def _create_devices(self, device_class):
214         mac_os_version = self.host.platform.os_version
215
216         self._set_device_class(device_class)
217
218         _log.debug('')
219         _log.debug('setup_test_run for %s', self._device_class)
220
221         self._create_simulators()
222
223         if not self._using_dedicated_simulators():
224             return
225
226         for i in xrange(self.child_processes()):
227             device_udid = self._testing_device(i).udid
228             _log.debug('testing device %s has udid %s', i, device_udid)
229
230             # FIXME: <rdar://problem/20916140> Switch to using CoreSimulator.framework for launching and quitting iOS Simulator
231             self._executive.run_command([
232                 'open', '-g', '-b', self.SIMULATOR_BUNDLE_ID + str(i),
233                 '--args', '-CurrentDeviceUDID', device_udid])
234
235             if mac_os_version in ['elcapitan', 'yosemite', 'mavericks']:
236                 time.sleep(2.5)
237
238         _log.info('Waiting for all iOS Simulators to finish booting.')
239         for i in xrange(self.child_processes()):
240             Simulator.wait_until_device_is_booted(self._testing_device(i).udid)
241
242     def _quit_ios_simulator(self):
243         if not self._using_dedicated_simulators():
244             return
245         _log.debug("_quit_ios_simulator killing all Simulator processes")
246         # FIXME: We should kill only the Simulators we started.
247         subprocess.call(["killall", "-9", "-m", "Simulator"])
248
249     def clean_up_test_run(self):
250         super(IOSSimulatorPort, self).clean_up_test_run()
251         _log.debug("clean_up_test_run")
252         self._quit_ios_simulator()
253         fifos = [path for path in os.listdir('/tmp') if re.search('org.webkit.(DumpRenderTree|WebKitTestRunner).*_(IN|OUT|ERROR)', path)]
254         for fifo in fifos:
255             try:
256                 os.remove(os.path.join('/tmp', fifo))
257             except OSError:
258                 _log.warning('Unable to remove ' + fifo)
259                 pass
260
261         if not self._using_dedicated_simulators():
262             return
263
264         for i in xrange(self.child_processes()):
265             simulator_path = self.get_simulator_path(i)
266             device_udid = self._testing_device(i).udid
267             self._remove_device(i)
268
269             if not os.path.exists(simulator_path):
270                 continue
271             try:
272                 self._executive.run_command([self.LSREGISTER_PATH, "-u", simulator_path])
273
274                 _log.debug('rmtree %s', simulator_path)
275                 self._filesystem.rmtree(simulator_path)
276
277                 logs_path = self._filesystem.join(self._filesystem.expanduser("~"), "Library/Logs/CoreSimulator/", device_udid)
278                 _log.debug('rmtree %s', logs_path)
279                 self._filesystem.rmtree(logs_path)
280
281                 saved_state_path = self._filesystem.join(self._filesystem.expanduser("~"), "Library/Saved Application State/", self.SIMULATOR_BUNDLE_ID + str(i) + ".savedState")
282                 _log.debug('rmtree %s', saved_state_path)
283                 self._filesystem.rmtree(saved_state_path)
284
285             except:
286                 _log.warning('Unable to remove Simulator' + str(i))
287
288     def setup_environ_for_server(self, server_name=None):
289         _log.debug("setup_environ_for_server")
290         env = super(IOSSimulatorPort, self).setup_environ_for_server(server_name)
291         if server_name == self.driver_name():
292             if self.get_option('leaks'):
293                 env['MallocStackLogging'] = '1'
294                 env['__XPC_MallocStackLogging'] = '1'
295                 env['MallocScribble'] = '1'
296                 env['__XPC_MallocScribble'] = '1'
297             if self.get_option('guard_malloc'):
298                 self._append_value_colon_separated(env, 'DYLD_INSERT_LIBRARIES', '/usr/lib/libgmalloc.dylib')
299                 self._append_value_colon_separated(env, '__XPC_DYLD_INSERT_LIBRARIES', '/usr/lib/libgmalloc.dylib')
300         env['XML_CATALOG_FILES'] = ''  # work around missing /etc/catalog <rdar://problem/4292995>
301         return env
302
303     def operating_system(self):
304         return 'ios-simulator'
305
306     def check_sys_deps(self, needs_http):
307         if not self.simulator_runtime.available:
308             _log.error('The iOS Simulator runtime with identifier "{0}" cannot be used because it is unavailable.'.format(self.simulator_runtime.identifier))
309             return False
310         return super(IOSSimulatorPort, self).check_sys_deps(needs_http)
311
312     SUBPROCESS_CRASH_REGEX = re.compile('#CRASHED - (?P<subprocess_name>\S+) \(pid (?P<subprocess_pid>\d+)\)')
313
314     def _using_dedicated_simulators(self):
315         return self.get_option('dedicated_simulators')
316
317     def using_multiple_devices(self):
318         return self._using_dedicated_simulators()
319
320     def _create_device(self, number):
321         return Simulator.create_device(number, self.simulator_device_type(), self.simulator_runtime)
322
323     def _remove_device(self, number):
324         Simulator.remove_device(number)
325
326     def get_simulator_path(self, suffix=""):
327         return os.path.join(self.SIMULATOR_DIRECTORY, "Simulator" + str(suffix) + ".app")
328
329     def diff_image(self, expected_contents, actual_contents, tolerance=None):
330         if not actual_contents and not expected_contents:
331             return (None, 0, None)
332         if not actual_contents or not expected_contents:
333             return (True, 0, None)
334         if not self._image_differ:
335             self._image_differ = image_diff.IOSSimulatorImageDiffer(self)
336         self.set_option_default('tolerance', 0.1)
337         if tolerance is None:
338             tolerance = self.get_option('tolerance')
339         return self._image_differ.diff_image(expected_contents, actual_contents, tolerance)
340
341     def reset_preferences(self):
342         _log.debug("reset_preferences")
343         self._quit_ios_simulator()
344         # Maybe this should delete all devices that we've created?
345
346     def nm_command(self):
347         return self.xcrun_find('nm')
348
349     @property
350     @memoized
351     def developer_dir(self):
352         return self._executive.run_command(['xcode-select', '--print-path']).rstrip()
353
354     def logging_patterns_to_strip(self):
355         return []
356
357     def stderr_patterns_to_strip(self):
358         return []
359
360     def _createSimulatorApps(self):
361         for i in xrange(self.child_processes()):
362             self._createSimulatorApp(i)
363
364     def _createSimulatorApp(self, suffix):
365         destination = self.get_simulator_path(suffix)
366         _log.info("Creating app:" + destination)
367         if os.path.exists(destination):
368             shutil.rmtree(destination, ignore_errors=True)
369         simulator_app_path = self.developer_dir + "/Applications/Simulator.app"
370         shutil.copytree(simulator_app_path, destination)
371
372         # Update app's package-name inside plist and re-code-sign it
373         plist_path = destination + "/Contents/Info.plist"
374         command = "Set CFBundleIdentifier com.apple.iphonesimulator" + str(suffix)
375         subprocess.check_output(["/usr/libexec/PlistBuddy", "-c", command, plist_path])
376         subprocess.check_output(["install_name_tool", "-add_rpath", self.developer_dir + "/Library/PrivateFrameworks/", destination + "/Contents/MacOS/Simulator"])
377         subprocess.check_output(["install_name_tool", "-add_rpath", self.developer_dir + "/../Frameworks/", destination + "/Contents/MacOS/Simulator"])
378         subprocess.check_output(["codesign", "-fs", "-", destination])
379         subprocess.check_output([self.LSREGISTER_PATH, "-f", destination])