b6c4d819dd72bcf52e00be5959e993fbfd506957
[WebKit-https.git] / Tools / Scripts / webkitpy / port / ios.py
1 # Copyright (C) 2014-2016 Apple Inc. All rights reserved.
2 #
3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions
5 # are met:
6 # 1.  Redistributions of source code must retain the above copyright
7 #     notice, this list of conditions and the following disclaimer.
8 # 2.  Redistributions in binary form must reproduce the above copyright
9 #     notice, this list of conditions and the following disclaimer in the
10 #     documentation and/or other materials provided with the distribution.
11 #
12 # THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND
13 # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
14 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
15 # DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR
16 # ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
17 # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
18 # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
19 # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
20 # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
21 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
22
23 import itertools
24 import logging
25 import os
26 import re
27 import shutil
28 import subprocess
29 import time
30
31 from webkitpy.common.memoized import memoized
32 from webkitpy.common.system.crashlogs import CrashLogs
33 from webkitpy.common.system.executive import ScriptError
34 from webkitpy.layout_tests.models.test_configuration import TestConfiguration
35 from webkitpy.port import config as port_config
36 from webkitpy.port import driver, image_diff
37 from webkitpy.port.apple import ApplePort
38 from webkitpy.port.base import Port
39 from webkitpy.xcode.simulator import Simulator, Runtime, DeviceType
40
41
42 _log = logging.getLogger(__name__)
43
44
45 class IOSPort(ApplePort):
46     port_name = "ios"
47
48     ARCHITECTURES = ['armv7', 'armv7s', 'arm64']
49     DEFAULT_ARCHITECTURE = 'arm64'
50     VERSION_FALLBACK_ORDER = ['ios-7', 'ios-8', 'ios-9']
51
52     @classmethod
53     def determine_full_port_name(cls, host, options, port_name):
54         if port_name == cls.port_name:
55             iphoneos_sdk_version = host.platform.xcode_sdk_version('iphoneos')
56             if not iphoneos_sdk_version:
57                 raise Exception("Please install the iOS SDK.")
58             major_version_number = iphoneos_sdk_version.split('.')[0]
59             port_name = port_name + '-' + major_version_number
60         return port_name
61
62     # Despite their names, these flags do not actually get passed all the way down to webkit-build.
63     def _build_driver_flags(self):
64         return ['--sdk', 'iphoneos'] + (['ARCHS=%s' % self.architecture()] if self.architecture() else [])
65
66     def operating_system(self):
67         return 'ios'
68
69
70 class IOSSimulatorPort(ApplePort):
71     port_name = "ios-simulator"
72
73     FUTURE_VERSION = 'future'
74     ARCHITECTURES = ['x86_64', 'x86']
75     DEFAULT_ARCHITECTURE = 'x86_64'
76
77     SIMULATOR_BUNDLE_ID = 'com.apple.iphonesimulator'
78     relay_name = 'LayoutTestRelay'
79     SIMULATOR_DIRECTORY = "/tmp/WebKitTestingSimulators/"
80     LSREGISTER_PATH = "/System/Library/Frameworks/CoreServices.framework/Frameworks/LaunchServices.framework/Versions/Current/Support/lsregister"
81     PROCESS_COUNT_ESTIMATE_PER_SIMULATOR_INSTANCE = 100
82
83     def __init__(self, host, port_name, **kwargs):
84         super(IOSSimulatorPort, self).__init__(host, port_name, **kwargs)
85
86     def driver_name(self):
87         if self.get_option('driver_name'):
88             return self.get_option('driver_name')
89         if self.get_option('webkit_test_runner'):
90             return 'WebKitTestRunnerApp.app'
91         return 'DumpRenderTree.app'
92
93     @property
94     @memoized
95     def simulator_runtime(self):
96         runtime_identifier = self.get_option('runtime')
97         if runtime_identifier:
98             runtime = Runtime.from_identifier(runtime_identifier)
99         else:
100             runtime = Runtime.from_version_string(self.host.platform.xcode_sdk_version('iphonesimulator'))
101         return runtime
102
103     @property
104     @memoized
105     def simulator_device_type(self):
106         device_type_identifier = self.get_option('device_type')
107         if device_type_identifier:
108             device_type = DeviceType.from_identifier(device_type_identifier)
109         else:
110             if self.architecture() == 'x86_64':
111                 device_type = DeviceType.from_name('iPhone 5s')
112             else:
113                 device_type = DeviceType.from_name('iPhone 5')
114         return device_type
115
116     @property
117     @memoized
118     def relay_path(self):
119         if self._root_was_set:
120             path = self._filesystem.abspath(self.get_option('root'))
121         else:
122             mac_config = port_config.Config(self._executive, self._filesystem, 'mac')
123             path = mac_config.build_directory(self.get_option('configuration'))
124         return self._filesystem.join(path, self.relay_name)
125
126     @memoized
127     def child_processes(self):
128         return int(self.get_option('child_processes'))
129
130     @memoized
131     def default_child_processes(self):
132         """Return the number of Simulators instances to use for this port."""
133         best_child_process_count_for_cpu = self._executive.cpu_count() / 2
134         system_process_count_limit = int(subprocess.check_output(["ulimit", "-u"]).strip())
135         current_process_count = len(subprocess.check_output(["ps", "aux"]).strip().split('\n'))
136         _log.info('Process limit: %d, current #processes: %d' % (system_process_count_limit, current_process_count))
137         maximum_simulator_count_on_this_system = (system_process_count_limit - current_process_count) // self.PROCESS_COUNT_ESTIMATE_PER_SIMULATOR_INSTANCE
138         # FIXME: We should also take into account the available RAM.
139
140         if (maximum_simulator_count_on_this_system < best_child_process_count_for_cpu):
141             _log.warn("This machine could support %s simulators, but is only configured for %s."
142                 % (best_child_process_count_for_cpu, maximum_simulator_count_on_this_system))
143             _log.warn('Please see <https://trac.webkit.org/wiki/IncreasingKernelLimits>.')
144
145         if maximum_simulator_count_on_this_system == 0:
146             maximum_simulator_count_on_this_system = 1
147
148         return min(maximum_simulator_count_on_this_system, best_child_process_count_for_cpu)
149
150     def _check_relay(self):
151         if not self._filesystem.exists(self.relay_path):
152             _log.error("%s was not found at %s" % (self.relay_name, self.relay_path))
153             return False
154         return True
155
156     def _check_port_build(self):
157         if not self._root_was_set and self.get_option('build') and not self._build_relay():
158             return False
159         if not self._check_relay():
160             return False
161         return True
162
163     def _build_relay(self):
164         environment = self.host.copy_current_environment()
165         environment.disable_gcc_smartquotes()
166         env = environment.to_dictionary()
167
168         try:
169             # FIXME: We should be passing _arguments_for_configuration(), which respects build configuration and port,
170             # instead of hardcoding --ios-simulator.
171             self._run_script("build-layouttestrelay", args=["--ios-simulator"], env=env)
172         except ScriptError, e:
173             _log.error(e.message_with_output(output_limit=None))
174             return False
175         return True
176
177     def _build_driver(self):
178         built_tool = super(IOSSimulatorPort, self)._build_driver()
179         built_relay = self._build_relay()
180         return built_tool and built_relay
181
182     def _build_driver_flags(self):
183         archs = ['ARCHS=i386'] if self.architecture() == 'x86' else []
184         sdk = ['--sdk', 'iphonesimulator']
185         return archs + sdk
186
187     def _generate_all_test_configurations(self):
188         configurations = []
189         for build_type in self.ALL_BUILD_TYPES:
190             for architecture in self.ARCHITECTURES:
191                 configurations.append(TestConfiguration(version=self._version, architecture=architecture, build_type=build_type))
192         return configurations
193
194     def _driver_class(self):
195         return driver.IOSSimulatorDriver
196
197     def default_baseline_search_path(self):
198         if self.get_option('webkit_test_runner'):
199             fallback_names = [self._wk2_port_name()] + [self.port_name] + ['wk2']
200         else:
201             fallback_names = [self.port_name + '-wk1'] + [self.port_name]
202
203         return map(self._webkit_baseline_path, fallback_names)
204
205     def _port_specific_expectations_files(self):
206         return list(reversed([self._filesystem.join(self._webkit_baseline_path(p), 'TestExpectations') for p in self.baseline_search_path()]))
207
208     def setup_test_run(self):
209         mac_os_version = self.host.platform.os_version
210         for i in xrange(self.child_processes()):
211             device_udid = self.testing_device(i).udid
212             # FIXME: <rdar://problem/20916140> Switch to using CoreSimulator.framework for launching and quitting iOS Simulator
213
214             self._executive.run_command([
215                 'open', '-g', '-b', self.SIMULATOR_BUNDLE_ID + str(i),
216                 '--args', '-CurrentDeviceUDID', device_udid])
217
218             if mac_os_version in ['elcapitan', 'yosemite', 'mavericks']:
219                 time.sleep(2.5)
220
221         _log.info('Waiting for all iOS Simulators to finish booting.')
222         for i in xrange(self.child_processes()):
223             Simulator.wait_until_device_is_booted(self.testing_device(i).udid)
224
225     def _quit_ios_simulator(self):
226         _log.debug("_quit_ios_simulator")
227         # FIXME: We should kill only the Simulators we started.
228         subprocess.call(["killall", "-9", "-m", "Simulator"])
229
230     def clean_up_test_run(self):
231         super(IOSSimulatorPort, self).clean_up_test_run()
232         _log.debug("clean_up_test_run")
233         self._quit_ios_simulator()
234         fifos = [path for path in os.listdir('/tmp') if re.search('org.webkit.(DumpRenderTree|WebKitTestRunner).*_(IN|OUT|ERROR)', path)]
235         for fifo in fifos:
236             try:
237                 os.remove(os.path.join('/tmp', fifo))
238             except OSError:
239                 _log.warning('Unable to remove ' + fifo)
240                 pass
241
242         for i in xrange(self.child_processes()):
243             simulator_path = self.get_simulator_path(i)
244             device_udid = self.testing_device(i).udid
245             if not os.path.exists(simulator_path):
246                 continue
247             try:
248                 self._executive.run_command([self.LSREGISTER_PATH, "-u", simulator_path])
249
250                 _log.debug('rmtree %s', simulator_path)
251                 self._filesystem.rmtree(simulator_path)
252
253                 logs_path = self._filesystem.join(self._filesystem.expanduser("~"), "Library/Logs/CoreSimulator/", device_udid)
254                 _log.debug('rmtree %s', logs_path)
255                 self._filesystem.rmtree(logs_path)
256
257                 saved_state_path = self._filesystem.join(self._filesystem.expanduser("~"), "Library/Saved Application State/", self.SIMULATOR_BUNDLE_ID + str(i) + ".savedState")
258                 _log.debug('rmtree %s', saved_state_path)
259                 self._filesystem.rmtree(saved_state_path)
260
261                 Simulator().delete_device(device_udid)
262             except:
263                 _log.warning('Unable to remove Simulator' + str(i))
264
265     def setup_environ_for_server(self, server_name=None):
266         _log.debug("setup_environ_for_server")
267         env = super(IOSSimulatorPort, self).setup_environ_for_server(server_name)
268         if server_name == self.driver_name():
269             if self.get_option('leaks'):
270                 env['MallocStackLogging'] = '1'
271                 env['__XPC_MallocStackLogging'] = '1'
272                 env['MallocScribble'] = '1'
273                 env['__XPC_MallocScribble'] = '1'
274             if self.get_option('guard_malloc'):
275                 self._append_value_colon_separated(env, 'DYLD_INSERT_LIBRARIES', '/usr/lib/libgmalloc.dylib')
276                 self._append_value_colon_separated(env, '__XPC_DYLD_INSERT_LIBRARIES', '/usr/lib/libgmalloc.dylib')
277         env['XML_CATALOG_FILES'] = ''  # work around missing /etc/catalog <rdar://problem/4292995>
278         return env
279
280     def operating_system(self):
281         return 'ios-simulator'
282
283     def check_sys_deps(self, needs_http):
284         if not self.simulator_runtime.available:
285             _log.error('The iOS Simulator runtime with identifier "{0}" cannot be used because it is unavailable.'.format(self.simulator_runtime.identifier))
286             return False
287         for i in xrange(self.child_processes()):
288             # FIXME: This creates the devices sequentially, doing this in parallel can improve performance.
289             testing_device = self.testing_device(i)
290         return super(IOSSimulatorPort, self).check_sys_deps(needs_http)
291
292     SUBPROCESS_CRASH_REGEX = re.compile('#CRASHED - (?P<subprocess_name>\S+) \(pid (?P<subprocess_pid>\d+)\)')
293
294     def _get_crash_log(self, name, pid, stdout, stderr, newer_than, time_fn=time.time, sleep_fn=time.sleep, wait_for_log=True):
295         time_fn = time_fn or time.time
296         sleep_fn = sleep_fn or time.sleep
297
298         # FIXME: We should collect the actual crash log for DumpRenderTree.app because it includes more
299         # information (e.g. exception codes) than is available in the stack trace written to standard error.
300         stderr_lines = []
301         crashed_subprocess_name_and_pid = None  # e.g. ('DumpRenderTree.app', 1234)
302         for line in (stderr or '').splitlines():
303             if not crashed_subprocess_name_and_pid:
304                 match = self.SUBPROCESS_CRASH_REGEX.match(line)
305                 if match:
306                     crashed_subprocess_name_and_pid = (match.group('subprocess_name'), int(match.group('subprocess_pid')))
307                     continue
308             stderr_lines.append(line)
309
310         if crashed_subprocess_name_and_pid:
311             return self._get_crash_log(crashed_subprocess_name_and_pid[0], crashed_subprocess_name_and_pid[1], stdout,
312                 '\n'.join(stderr_lines), newer_than, time_fn, sleep_fn, wait_for_log)
313
314         # LayoutTestRelay crashed
315         _log.debug('looking for crash log for %s:%s' % (name, str(pid)))
316         crash_log = ''
317         crash_logs = CrashLogs(self.host)
318         now = time_fn()
319         deadline = now + 5 * int(self.get_option('child_processes', 1))
320         while not crash_log and now <= deadline:
321             crash_log = crash_logs.find_newest_log(name, pid, include_errors=True, newer_than=newer_than)
322             if not wait_for_log:
323                 break
324             if not crash_log or not [line for line in crash_log.splitlines() if not line.startswith('ERROR')]:
325                 sleep_fn(0.1)
326                 now = time_fn()
327
328         if not crash_log:
329             return stderr, None
330         return stderr, crash_log
331
332     @memoized
333     def testing_device(self, number):
334         return Simulator().lookup_or_create_device(self.simulator_device_type.name + ' WebKit Tester' + str(number), self.simulator_device_type, self.simulator_runtime)
335
336     def get_simulator_path(self, suffix=""):
337         return os.path.join(self.SIMULATOR_DIRECTORY, "Simulator" + str(suffix) + ".app")
338
339     def diff_image(self, expected_contents, actual_contents, tolerance=None):
340         if not actual_contents and not expected_contents:
341             return (None, 0, None)
342         if not actual_contents or not expected_contents:
343             return (True, 0, None)
344         if not self._image_differ:
345             self._image_differ = image_diff.IOSSimulatorImageDiffer(self)
346         self.set_option_default('tolerance', 0.1)
347         if tolerance is None:
348             tolerance = self.get_option('tolerance')
349         return self._image_differ.diff_image(expected_contents, actual_contents, tolerance)
350
351     def reset_preferences(self):
352         _log.debug("reset_preferences")
353         if (self.default_child_processes() < self.child_processes()):
354                 _log.warn("You have specified very high value({0}) for --child-processes".format(self.child_processes()))
355                 _log.warn("maximum child-processes which can be supported on this system are: {0}".format(self.default_child_processes()))
356                 _log.warn("This is very likely to fail.")
357
358         self._quit_ios_simulator()
359         self._createSimulatorApps()
360
361         for i in xrange(self.child_processes()):
362             Simulator.wait_until_device_is_in_state(self.testing_device(i).udid, Simulator.DeviceState.SHUTDOWN)
363             Simulator.reset_device(self.testing_device(i).udid)
364
365     def nm_command(self):
366         return self.xcrun_find('nm')
367
368     def xcrun_find(self, command, fallback=None):
369         fallback = fallback or command
370         try:
371             return self._executive.run_command(['xcrun', '--sdk', 'iphonesimulator', '-find', command]).rstrip()
372         except ScriptError:
373             _log.warn("xcrun failed; falling back to '%s'." % fallback)
374             return fallback
375
376     @property
377     @memoized
378     def developer_dir(self):
379         return self._executive.run_command(['xcode-select', '--print-path']).rstrip()
380
381     def logging_patterns_to_strip(self):
382         return []
383
384     def stderr_patterns_to_strip(self):
385         return []
386
387     def _createSimulatorApps(self):
388         for i in xrange(self.child_processes()):
389             self._createSimulatorApp(i)
390
391     def _createSimulatorApp(self, suffix):
392         destination = self.get_simulator_path(suffix)
393         _log.info("Creating app:" + destination)
394         if os.path.exists(destination):
395             shutil.rmtree(destination, ignore_errors=True)
396         simulator_app_path = self.developer_dir + "/Applications/Simulator.app"
397         shutil.copytree(simulator_app_path, destination)
398
399         # Update app's package-name inside plist and re-code-sign it
400         plist_path = destination + "/Contents/Info.plist"
401         command = "Set CFBundleIdentifier com.apple.iphonesimulator" + str(suffix)
402         subprocess.check_output(["/usr/libexec/PlistBuddy", "-c", command, plist_path])
403         subprocess.check_output(["install_name_tool", "-add_rpath", self.developer_dir + "/Library/PrivateFrameworks/", destination + "/Contents/MacOS/Simulator"])
404         subprocess.check_output(["codesign", "-fs", "-", destination])
405         subprocess.check_output([self.LSREGISTER_PATH, "-f", destination])