bec6c2ff08e3618d19cc7af2f17c2f89775fcc05
[WebKit-https.git] / Tools / Scripts / webkitpy / port / ios.py
1 # Copyright (C) 2014-2016 Apple Inc. All rights reserved.
2 #
3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions
5 # are met:
6 # 1.  Redistributions of source code must retain the above copyright
7 #     notice, this list of conditions and the following disclaimer.
8 # 2.  Redistributions in binary form must reproduce the above copyright
9 #     notice, this list of conditions and the following disclaimer in the
10 #     documentation and/or other materials provided with the distribution.
11 #
12 # THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND
13 # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
14 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
15 # DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR
16 # ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
17 # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
18 # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
19 # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
20 # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
21 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
22
23 import itertools
24 import logging
25 import os
26 import re
27 import shutil
28 import subprocess
29 import time
30
31 from webkitpy.common.memoized import memoized
32 from webkitpy.common.system.crashlogs import CrashLogs
33 from webkitpy.common.system.executive import ScriptError
34 from webkitpy.layout_tests.models.test_configuration import TestConfiguration
35 from webkitpy.port import config as port_config
36 from webkitpy.port import driver, image_diff
37 from webkitpy.port.apple import ApplePort
38 from webkitpy.port.base import Port
39 from webkitpy.port.leakdetector import LeakDetector
40 from webkitpy.xcode.simulator import Simulator, Runtime, DeviceType
41
42
43 _log = logging.getLogger(__name__)
44
45
46 class IOSPort(ApplePort):
47     port_name = "ios"
48
49     ARCHITECTURES = ['armv7', 'armv7s', 'arm64']
50     DEFAULT_ARCHITECTURE = 'arm64'
51     VERSION_FALLBACK_ORDER = ['ios-7', 'ios-8', 'ios-9']
52
53     @classmethod
54     def determine_full_port_name(cls, host, options, port_name):
55         if port_name == cls.port_name:
56             iphoneos_sdk_version = host.platform.xcode_sdk_version('iphoneos')
57             if not iphoneos_sdk_version:
58                 raise Exception("Please install the iOS SDK.")
59             major_version_number = iphoneos_sdk_version.split('.')[0]
60             port_name = port_name + '-' + major_version_number
61         return port_name
62
63     # Despite their names, these flags do not actually get passed all the way down to webkit-build.
64     def _build_driver_flags(self):
65         return ['--sdk', 'iphoneos'] + (['ARCHS=%s' % self.architecture()] if self.architecture() else [])
66
67     def operating_system(self):
68         return 'ios'
69
70
71 class IOSSimulatorPort(Port):
72     port_name = "ios-simulator"
73     FUTURE_VERSION = 'future'
74     ARCHITECTURES = ['x86_64', 'x86']
75     DEFAULT_ARCHITECTURE = 'x86_64'
76     SIMULATOR_BUNDLE_ID = 'com.apple.iphonesimulator'
77     relay_name = 'LayoutTestRelay'
78     SIMULATOR_DIRECTORY = "/tmp/WebKitTestingSimulators/"
79     LSREGISTER_PATH = "/System/Library/Frameworks/CoreServices.framework/Frameworks/LaunchServices.framework/Versions/Current/Support/lsregister"
80     PROCESS_COUNT_ESTIMATE_PER_SIMULATOR_INSTANCE = 100
81
82     def __init__(self, *args, **kwargs):
83         super(IOSSimulatorPort, self).__init__(*args, **kwargs)
84
85         self._leak_detector = LeakDetector(self)
86         if self.get_option("leaks"):
87             # DumpRenderTree slows down noticably if we run more than about 1000 tests in a batch
88             # with MallocStackLogging enabled.
89             self.set_option_default("batch_size", 1000)
90
91     def driver_name(self):
92         if self.get_option('driver_name'):
93             return self.get_option('driver_name')
94         if self.get_option('webkit_test_runner'):
95             return 'WebKitTestRunnerApp.app'
96         return 'DumpRenderTree.app'
97
98     @property
99     @memoized
100     def simulator_runtime(self):
101         runtime_identifier = self.get_option('runtime')
102         if runtime_identifier:
103             runtime = Runtime.from_identifier(runtime_identifier)
104         else:
105             runtime = Runtime.from_version_string(self.host.platform.xcode_sdk_version('iphonesimulator'))
106         return runtime
107
108     @property
109     @memoized
110     def simulator_device_type(self):
111         device_type_identifier = self.get_option('device_type')
112         if device_type_identifier:
113             device_type = DeviceType.from_identifier(device_type_identifier)
114         else:
115             if self.architecture() == 'x86_64':
116                 device_type = DeviceType.from_name('iPhone 5s')
117             else:
118                 device_type = DeviceType.from_name('iPhone 5')
119         return device_type
120
121     @property
122     @memoized
123     def relay_path(self):
124         if self._root_was_set:
125             path = self._filesystem.abspath(self.get_option('root'))
126         else:
127             mac_config = port_config.Config(self._executive, self._filesystem, 'mac')
128             path = mac_config.build_directory(self.get_option('configuration'))
129         return self._filesystem.join(path, self.relay_name)
130
131     @memoized
132     def child_processes(self):
133         return int(self.get_option('child_processes'))
134
135     @memoized
136     def default_child_processes(self):
137         """Return the number of Simulators instances to use for this port."""
138         best_child_process_count_for_cpu = self._executive.cpu_count() / 2
139         system_process_count_limit = int(subprocess.check_output(["ulimit", "-u"]).strip())
140         current_process_count = len(subprocess.check_output(["ps", "aux"]).strip().split('\n'))
141         _log.info('Process limit: %d, current #processes: %d' % (system_process_count_limit, current_process_count))
142         maximum_simulator_count_on_this_system = (system_process_count_limit - current_process_count) // self.PROCESS_COUNT_ESTIMATE_PER_SIMULATOR_INSTANCE
143         # FIXME: We should also take into account the available RAM.
144
145         if (maximum_simulator_count_on_this_system < best_child_process_count_for_cpu):
146             _log.warn("This machine could support %s child processes, but only has enough process limit for %s."
147                 % (best_child_process_count_for_cpu, maximum_simulator_count_on_this_system))
148             _log.warn('Run "launchctl limit" to check these limits')
149             # FIXME: Add url for webpage explaining how to increase these limits.
150
151         if maximum_simulator_count_on_this_system == 0:
152             maximum_simulator_count_on_this_system = 1
153
154         return min(maximum_simulator_count_on_this_system, best_child_process_count_for_cpu)
155
156     def default_timeout_ms(self):
157         if self.get_option('guard_malloc'):
158             return 350 * 1000
159         return super(IOSSimulatorPort, self).default_timeout_ms()
160
161     def supports_per_test_timeout(self):
162         return True
163
164     def _check_relay(self):
165         if not self._filesystem.exists(self.relay_path):
166             _log.error("%s was not found at %s" % (self.relay_name, self.relay_path))
167             return False
168         return True
169
170     def _check_port_build(self):
171         if not self._root_was_set and self.get_option('build') and not self._build_relay():
172             return False
173         if not self._check_relay():
174             return False
175         return True
176
177     def _build_relay(self):
178         environment = self.host.copy_current_environment()
179         environment.disable_gcc_smartquotes()
180         env = environment.to_dictionary()
181
182         try:
183             # FIXME: We should be passing _arguments_for_configuration(), which respects build configuration and port,
184             # instead of hardcoding --ios-simulator.
185             self._run_script("build-layouttestrelay", args=["--ios-simulator"], env=env)
186         except ScriptError, e:
187             _log.error(e.message_with_output(output_limit=None))
188             return False
189         return True
190
191     def _build_driver(self):
192         built_tool = super(IOSSimulatorPort, self)._build_driver()
193         built_relay = self._build_relay()
194         return built_tool and built_relay
195
196     def _build_driver_flags(self):
197         archs = ['ARCHS=i386'] if self.architecture() == 'x86' else []
198         sdk = ['--sdk', 'iphonesimulator']
199         return archs + sdk
200
201     def should_retry_crashes(self):
202         return True
203
204     def _generate_all_test_configurations(self):
205         configurations = []
206         for build_type in self.ALL_BUILD_TYPES:
207             for architecture in self.ARCHITECTURES:
208                 configurations.append(TestConfiguration(version=self._version, architecture=architecture, build_type=build_type))
209         return configurations
210
211     def _driver_class(self):
212         return driver.IOSSimulatorDriver
213
214     def default_baseline_search_path(self):
215         if self.get_option('webkit_test_runner'):
216             fallback_names = [self._wk2_port_name()] + [self.port_name] + ['wk2']
217         else:
218             fallback_names = [self.port_name + '-wk1'] + [self.port_name]
219
220         return map(self._webkit_baseline_path, fallback_names)
221
222     def _port_specific_expectations_files(self):
223         return list(reversed([self._filesystem.join(self._webkit_baseline_path(p), 'TestExpectations') for p in self.baseline_search_path()]))
224
225     def setup_test_run(self):
226         mac_os_version = self.host.platform.os_version
227         for i in xrange(self.child_processes()):
228             device_udid = self.testing_device(i).udid
229             # FIXME: <rdar://problem/20916140> Switch to using CoreSimulator.framework for launching and quitting iOS Simulator
230             self._executive.run_command([
231                 'open', '-b', self.SIMULATOR_BUNDLE_ID + str(i),
232                 '--args', '-CurrentDeviceUDID', device_udid])
233
234             if mac_os_version in ['elcapitan', 'yosemite', 'mavericks']:
235                 time.sleep(2.5)
236
237         _log.info('Waiting for all iOS Simulators to finish booting.')
238         for i in xrange(self.child_processes()):
239             Simulator.wait_until_device_is_booted(self.testing_device(i).udid)
240
241     def _quit_ios_simulator(self):
242         # FIXME: We should kill only the Simulators we started.
243         subprocess.call(["killall", "-9", "-m", "Simulator"])
244
245     def clean_up_test_run(self):
246         super(IOSSimulatorPort, self).clean_up_test_run()
247         self._quit_ios_simulator()
248         fifos = [path for path in os.listdir('/tmp') if re.search('org.webkit.(DumpRenderTree|WebKitTestRunner).*_(IN|OUT|ERROR)', path)]
249         for fifo in fifos:
250             try:
251                 os.remove(os.path.join('/tmp', fifo))
252             except OSError:
253                 _log.warning('Unable to remove ' + fifo)
254                 pass
255
256         for i in xrange(self.child_processes()):
257             if not os.path.exists(self.get_simulator_path(i)):
258                 continue
259             try:
260                 subprocess.call([self.LSREGISTER_PATH, "-u", self.get_simulator_path(i)])
261                 shutil.rmtree(self.get_simulator_path(i), ignore_errors=True)
262                 shutil.rmtree(os.path.join(os.path.expanduser("~"), "Library/Logs/CoreSimulator/",
263                     self.testing_device(i).udid), ignore_errors=True)
264                 shutil.rmtree(os.path.join(os.path.expanduser("~"), "Library/Saved Application State/",
265                     self.SIMULATOR_BUNDLE_ID + str(i) + ".savedState"), ignore_errors=True)
266                 Simulator().delete_device(self.testing_device(i).udid)
267             except:
268                 _log.warning('Unable to remove Simulator' + str(i))
269
270     def setup_environ_for_server(self, server_name=None):
271         env = super(IOSSimulatorPort, self).setup_environ_for_server(server_name)
272         if server_name == self.driver_name():
273             if self.get_option('leaks'):
274                 env['MallocStackLogging'] = '1'
275                 env['__XPC_MallocStackLogging'] = '1'
276                 env['MallocScribble'] = '1'
277                 env['__XPC_MallocScribble'] = '1'
278             if self.get_option('guard_malloc'):
279                 self._append_value_colon_separated(env, 'DYLD_INSERT_LIBRARIES', '/usr/lib/libgmalloc.dylib')
280                 self._append_value_colon_separated(env, '__XPC_DYLD_INSERT_LIBRARIES', '/usr/lib/libgmalloc.dylib')
281             self._append_value_colon_separated(env, 'DYLD_INSERT_LIBRARIES', self._build_path("libWebCoreTestShim.dylib"))
282         env['XML_CATALOG_FILES'] = ''  # work around missing /etc/catalog <rdar://problem/4292995>
283         return env
284
285     def operating_system(self):
286         return 'ios-simulator'
287
288     def check_sys_deps(self, needs_http):
289         if not self.simulator_runtime.available:
290             _log.error('The iOS Simulator runtime with identifier "{0}" cannot be used because it is unavailable.'.format(self.simulator_runtime.identifier))
291             return False
292         for i in xrange(self.child_processes()):
293             # FIXME: This creates the devices sequentially, doing this in parallel can improve performance.
294             testing_device = self.testing_device(i)
295         return super(IOSSimulatorPort, self).check_sys_deps(needs_http)
296
297     def check_for_leaks(self, process_name, process_pid):
298         if not self.get_option('leaks'):
299             return
300         # We could use http://code.google.com/p/psutil/ to get the process_name from the pid.
301         self._leak_detector.check_for_leaks(process_name, process_pid)
302
303     def print_leaks_summary(self):
304         if not self.get_option('leaks'):
305             return
306         # We're in the manager process, so the leak detector will not have a valid list of leak files.
307         leaks_files = self._leak_detector.leaks_files_in_directory(self.results_directory())
308         if not leaks_files:
309             return
310         total_bytes_string, unique_leaks = self._leak_detector.count_total_bytes_and_unique_leaks(leaks_files)
311         total_leaks = self._leak_detector.count_total_leaks(leaks_files)
312         _log.info("%s total leaks found for a total of %s." % (total_leaks, total_bytes_string))
313         _log.info("%s unique leaks found." % unique_leaks)
314
315     def _path_to_webcore_library(self):
316         return self._build_path('WebCore.framework/Versions/A/WebCore')
317
318     def show_results_html_file(self, results_filename):
319         # We don't use self._run_script() because we don't want to wait for the script
320         # to exit and we want the output to show up on stdout in case there are errors
321         # launching the browser.
322         self._executive.popen([self.path_to_script('run-safari')] + self._arguments_for_configuration() + ['--no-saved-state', '-NSOpen', results_filename],
323             cwd=self.webkit_base(), stdout=file(os.devnull), stderr=file(os.devnull))
324
325     def sample_file_path(self, name, pid):
326         return self._filesystem.join(self.results_directory(), "{0}-{1}-sample.txt".format(name, pid))
327
328     SUBPROCESS_CRASH_REGEX = re.compile('#CRASHED - (?P<subprocess_name>\S+) \(pid (?P<subprocess_pid>\d+)\)')
329
330     def _get_crash_log(self, name, pid, stdout, stderr, newer_than, time_fn=time.time, sleep_fn=time.sleep, wait_for_log=True):
331         time_fn = time_fn or time.time
332         sleep_fn = sleep_fn or time.sleep
333
334         # FIXME: We should collect the actual crash log for DumpRenderTree.app because it includes more
335         # information (e.g. exception codes) than is available in the stack trace written to standard error.
336         stderr_lines = []
337         crashed_subprocess_name_and_pid = None  # e.g. ('DumpRenderTree.app', 1234)
338         for line in (stderr or '').splitlines():
339             if not crashed_subprocess_name_and_pid:
340                 match = self.SUBPROCESS_CRASH_REGEX.match(line)
341                 if match:
342                     crashed_subprocess_name_and_pid = (match.group('subprocess_name'), int(match.group('subprocess_pid')))
343                     continue
344             stderr_lines.append(line)
345
346         if crashed_subprocess_name_and_pid:
347             return self._get_crash_log(crashed_subprocess_name_and_pid[0], crashed_subprocess_name_and_pid[1], stdout,
348                 '\n'.join(stderr_lines), newer_than, time_fn, sleep_fn, wait_for_log)
349
350         # LayoutTestRelay crashed
351         _log.debug('looking for crash log for %s:%s' % (name, str(pid)))
352         crash_log = ''
353         crash_logs = CrashLogs(self.host)
354         now = time_fn()
355         deadline = now + 5 * int(self.get_option('child_processes', 1))
356         while not crash_log and now <= deadline:
357             crash_log = crash_logs.find_newest_log(name, pid, include_errors=True, newer_than=newer_than)
358             if not wait_for_log:
359                 break
360             if not crash_log or not [line for line in crash_log.splitlines() if not line.startswith('ERROR')]:
361                 sleep_fn(0.1)
362                 now = time_fn()
363
364         if not crash_log:
365             return stderr, None
366         return stderr, crash_log
367
368     @memoized
369     def testing_device(self, number):
370         return Simulator().lookup_or_create_device(self.simulator_device_type.name + ' WebKit Tester' + str(number), self.simulator_device_type, self.simulator_runtime)
371
372     def get_simulator_path(self, suffix=""):
373         return os.path.join(self.SIMULATOR_DIRECTORY, "Simulator" + str(suffix) + ".app")
374
375     def _merge_crash_logs(self, logs, new_logs, crashed_processes):
376         for test, crash_log in new_logs.iteritems():
377             try:
378                 process_name = test.split("-")[0]
379                 pid = int(test.split("-")[1])
380             except IndexError:
381                 continue
382             if not any(entry[1] == process_name and entry[2] == pid for entry in crashed_processes):
383                 # if this is a new crash, then append the logs
384                 logs[test] = crash_log
385         return logs
386
387     def _look_for_all_crash_logs_in_log_dir(self, newer_than):
388         crash_log = CrashLogs(self.host)
389         return crash_log.find_all_logs(include_errors=True, newer_than=newer_than)
390
391     def look_for_new_crash_logs(self, crashed_processes, start_time):
392         crash_logs = {}
393         for (test_name, process_name, pid) in crashed_processes:
394             # Passing None for output.  This is a second pass after the test finished so
395             # if the output had any logging we would have already collected it.
396             crash_log = self._get_crash_log(process_name, pid, None, None, start_time, wait_for_log=False)[1]
397             if not crash_log:
398                 continue
399             crash_logs[test_name] = crash_log
400         all_crash_log = self._look_for_all_crash_logs_in_log_dir(start_time)
401         return self._merge_crash_logs(crash_logs, all_crash_log, crashed_processes)
402
403     def look_for_new_samples(self, unresponsive_processes, start_time):
404         sample_files = {}
405         for (test_name, process_name, pid) in unresponsive_processes:
406             sample_file = self.sample_file_path(process_name, pid)
407             if not self._filesystem.isfile(sample_file):
408                 continue
409             sample_files[test_name] = sample_file
410         return sample_files
411
412     def sample_process(self, name, pid):
413         try:
414             hang_report = self.sample_file_path(name, pid)
415             self._executive.run_command([
416                 "/usr/bin/sample",
417                 pid,
418                 10,
419                 10,
420                 "-file",
421                 hang_report,
422             ])
423         except ScriptError as e:
424             _log.warning('Unable to sample process:' + str(e))
425
426     def _path_to_helper(self):
427         binary_name = 'LayoutTestHelper'
428         return self._build_path(binary_name)
429
430     def diff_image(self, expected_contents, actual_contents, tolerance=None):
431         if not actual_contents and not expected_contents:
432             return (None, 0, None)
433         if not actual_contents or not expected_contents:
434             return (True, 0, None)
435         if not self._image_differ:
436             self._image_differ = image_diff.IOSSimulatorImageDiffer(self)
437         self.set_option_default('tolerance', 0.1)
438         if tolerance is None:
439             tolerance = self.get_option('tolerance')
440         return self._image_differ.diff_image(expected_contents, actual_contents, tolerance)
441
442     def reset_preferences(self):
443         if (self.default_child_processes() < self.child_processes()):
444                 _log.warn("You have specified very high value({0}) for --child-processes".format(self.child_processes()))
445                 _log.warn("maximum child-processes which can be supported on this system are: {0}".format(self.default_child_processes()))
446                 _log.warn("This is very likely to fail.")
447
448         self._quit_ios_simulator()
449         self._createSimulatorApps()
450
451         for i in xrange(self.child_processes()):
452             Simulator.wait_until_device_is_in_state(self.testing_device(i).udid, Simulator.DeviceState.SHUTDOWN)
453
454             data_path = os.path.join(self.testing_device(i).path, 'data')
455             if os.path.isdir(data_path):
456                 shutil.rmtree(data_path)
457
458     def make_command(self):
459         return self.xcrun_find('make', '/usr/bin/make')
460
461     def nm_command(self):
462         return self.xcrun_find('nm')
463
464     def xcrun_find(self, command, fallback=None):
465         fallback = fallback or command
466         try:
467             return self._executive.run_command(['xcrun', '--sdk', 'iphonesimulator', '-find', command]).rstrip()
468         except ScriptError:
469             _log.warn("xcrun failed; falling back to '%s'." % fallback)
470             return fallback
471
472     @property
473     def developer_dir(self):
474         return self._executive.run_command(['xcode-select', '--print-path']).rstrip()
475
476     def logging_patterns_to_strip(self):
477         return []
478
479     def stderr_patterns_to_strip(self):
480         return []
481
482     def _createSimulatorApps(self):
483         for i in xrange(self.child_processes()):
484             self._createSimulatorApp(i)
485
486     def _createSimulatorApp(self, suffix):
487         destination = self.get_simulator_path(suffix)
488         _log.info("Creating app:" + destination)
489         if os.path.exists(destination):
490             shutil.rmtree(destination, ignore_errors=True)
491         simulator_app_path = self.developer_dir + "/Applications/Simulator.app"
492         shutil.copytree(simulator_app_path, destination)
493
494         # Update app's package-name inside plist and re-code-sign it
495         plist_path = destination + "/Contents/Info.plist"
496         command = "Set CFBundleIdentifier com.apple.iphonesimulator" + str(suffix)
497         subprocess.check_output(["/usr/libexec/PlistBuddy", "-c", command, plist_path])
498         subprocess.check_output(["install_name_tool", "-add_rpath", self.developer_dir + "/Library/PrivateFrameworks/", destination + "/Contents/MacOS/Simulator"])
499         subprocess.check_output(["codesign", "-fs", "-", destination])
500         subprocess.check_output([self.LSREGISTER_PATH, "-f", destination])