[iOS] run-webkit-tests should check that simctl can boot and shutdown simulator devic...
[WebKit-https.git] / Tools / Scripts / webkitpy / port / ios.py
1 # Copyright (C) 2014, 2015 Apple Inc. All rights reserved.
2 #
3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions
5 # are met:
6 # 1.  Redistributions of source code must retain the above copyright
7 #     notice, this list of conditions and the following disclaimer.
8 # 2.  Redistributions in binary form must reproduce the above copyright
9 #     notice, this list of conditions and the following disclaimer in the
10 #     documentation and/or other materials provided with the distribution.
11 #
12 # THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND
13 # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
14 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
15 # DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR
16 # ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
17 # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
18 # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
19 # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
20 # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
21 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
22
23 import itertools
24 import logging
25 import os
26 import re
27 import shutil
28 import subprocess
29 import time
30
31 from webkitpy.common.memoized import memoized
32 from webkitpy.common.system.crashlogs import CrashLogs
33 from webkitpy.common.system.executive import ScriptError
34 from webkitpy.layout_tests.models.test_configuration import TestConfiguration
35 from webkitpy.port import config as port_config
36 from webkitpy.port import driver, image_diff
37 from webkitpy.port.apple import ApplePort
38 from webkitpy.port.base import Port
39 from webkitpy.port.leakdetector import LeakDetector
40 from webkitpy.xcode.simulator import Simulator, Runtime, DeviceType
41
42
43 _log = logging.getLogger(__name__)
44
45
46 class IOSPort(ApplePort):
47     port_name = "ios"
48
49     ARCHITECTURES = ['armv7', 'armv7s', 'arm64']
50     DEFAULT_ARCHITECTURE = 'armv7'
51     VERSION_FALLBACK_ORDER = ['ios-7', 'ios-8', 'ios-9']
52
53     @classmethod
54     def determine_full_port_name(cls, host, options, port_name):
55         if port_name == cls.port_name:
56             iphoneos_sdk_version = host.platform.xcode_sdk_version('iphoneos')
57             if not iphoneos_sdk_version:
58                 raise Exception("Please install the iOS SDK.")
59             major_version_number = iphoneos_sdk_version.split('.')[0]
60             port_name = port_name + '-' + major_version_number
61         return port_name
62
63     def __init__(self, *args, **kwargs):
64         super(IOSPort, self).__init__(*args, **kwargs)
65
66         self._testing_device = None
67
68     # Despite their names, these flags do not actually get passed all the way down to webkit-build.
69     def _build_driver_flags(self):
70         return ['--sdk', 'iphoneos'] + (['ARCHS=%s' % self.architecture()] if self.architecture() else [])
71
72     def operating_system(self):
73         return 'ios'
74
75
76 class IOSSimulatorPort(Port):
77     port_name = "ios-simulator"
78
79     FUTURE_VERSION = 'future'
80
81     ARCHITECTURES = ['x86_64', 'x86']
82
83     DEFAULT_ARCHITECTURE = 'x86_64'
84
85     relay_name = 'LayoutTestRelay'
86
87     def __init__(self, *args, **kwargs):
88         super(IOSSimulatorPort, self).__init__(*args, **kwargs)
89
90         self._leak_detector = LeakDetector(self)
91         if self.get_option("leaks"):
92             # DumpRenderTree slows down noticably if we run more than about 1000 tests in a batch
93             # with MallocStackLogging enabled.
94             self.set_option_default("batch_size", 1000)
95
96         self._testing_device = None
97
98     def driver_name(self):
99         if self.get_option('driver_name'):
100             return self.get_option('driver_name')
101         if self.get_option('webkit_test_runner'):
102             return 'WebKitTestRunnerApp.app'
103         return 'DumpRenderTree.app'
104
105     @property
106     @memoized
107     def simulator_runtime(self):
108         runtime_identifier = self.get_option('runtime')
109         if runtime_identifier:
110             runtime = Runtime.from_identifier(runtime_identifier)
111         else:
112             runtime = Runtime.from_version_string(self.host.platform.xcode_sdk_version('iphonesimulator'))
113         return runtime
114
115     @property
116     @memoized
117     def simulator_device_type(self):
118         device_type_identifier = self.get_option('device_type')
119         if device_type_identifier:
120             device_type = DeviceType.from_identifier(device_type_identifier)
121         else:
122             if self.architecture() == 'x86_64':
123                 device_type = DeviceType.from_name('iPhone 5s')
124             else:
125                 device_type = DeviceType.from_name('iPhone 5')
126         return device_type
127
128     @property
129     def relay_path(self):
130         mac_config = port_config.Config(self._executive, self._filesystem, 'mac')
131         return self._filesystem.join(mac_config.build_directory(self.get_option('configuration')), self.relay_name)
132
133     def default_timeout_ms(self):
134         if self.get_option('guard_malloc'):
135             return 350 * 1000
136         return super(IOSSimulatorPort, self).default_timeout_ms()
137
138     def supports_per_test_timeout(self):
139         return True
140
141     def _check_relay(self):
142         if not self._filesystem.exists(self.relay_path):
143             _log.error("%s was not found at %s" % (self.relay_name, self.relay_path))
144             return False
145         return True
146
147     def _check_build_relay(self):
148         if self.get_option('build') and not self._build_relay():
149             return False
150         if not self._check_relay():
151             return False
152         return True
153
154     def check_build(self, needs_http):
155         needs_driver = super(IOSSimulatorPort, self).check_build(needs_http)
156         return needs_driver and self._check_build_relay()
157
158     def _build_relay(self):
159         environment = self.host.copy_current_environment()
160         environment.disable_gcc_smartquotes()
161         env = environment.to_dictionary()
162
163         try:
164             self._run_script("build-layouttestrelay", env=env)
165         except ScriptError, e:
166             _log.error(e.message_with_output(output_limit=None))
167             return False
168         return True
169
170     def _build_driver(self):
171         built_tool = super(IOSSimulatorPort, self)._build_driver()
172         built_relay = self._build_relay()
173         return built_tool and built_relay
174
175     def _build_driver_flags(self):
176         archs = ['ARCHS=i386'] if self.architecture() == 'x86' else []
177         sdk = ['--sdk', 'iphonesimulator']
178         return archs + sdk
179
180     def should_retry_crashes(self):
181         return True
182
183     def _generate_all_test_configurations(self):
184         configurations = []
185         for build_type in self.ALL_BUILD_TYPES:
186             for architecture in self.ARCHITECTURES:
187                 configurations.append(TestConfiguration(version=self._version, architecture=architecture, build_type=build_type))
188         return configurations
189
190     def _driver_class(self):
191         return driver.IOSSimulatorDriver
192
193     def default_baseline_search_path(self):
194         if self.get_option('webkit_test_runner'):
195             fallback_names = [self._wk2_port_name(), 'wk2'] + [self.port_name]
196         else:
197             fallback_names = [self.port_name + '-wk1'] + [self.port_name]
198
199         return map(self._webkit_baseline_path, fallback_names)
200
201     def _port_specific_expectations_files(self):
202         return list(reversed([self._filesystem.join(self._webkit_baseline_path(p), 'TestExpectations') for p in self.baseline_search_path()]))
203
204     def setup_test_run(self):
205         self._executive.run_command(['osascript', '-e', 'tell application "iOS Simulator" to quit'])
206         device_udid = self.testing_device.udid
207         Simulator.wait_until_device_is_in_state(device_udid, Simulator.DeviceState.SHUTDOWN)
208         self._executive.run_command([
209             'open', '-a', os.path.join(self.developer_dir, 'Applications', 'iOS Simulator.app'),
210             '--args', '-CurrentDeviceUDID', device_udid])
211         Simulator.wait_until_device_is_in_state(device_udid, Simulator.DeviceState.BOOTED)
212
213     def clean_up_test_run(self):
214         super(IOSSimulatorPort, self).clean_up_test_run()
215         fifos = [path for path in os.listdir('/tmp') if re.search('org.webkit.(DumpRenderTree|WebKitTestRunner).*_(IN|OUT|ERROR)', path)]
216         for fifo in fifos:
217             try:
218                 os.remove(os.path.join('/tmp', fifo))
219             except OSError:
220                 _log.warning('Unable to remove ' + fifo)
221                 pass
222
223     def setup_environ_for_server(self, server_name=None):
224         env = super(IOSSimulatorPort, self).setup_environ_for_server(server_name)
225         if server_name == self.driver_name():
226             if self.get_option('leaks'):
227                 env['MallocStackLogging'] = '1'
228             if self.get_option('guard_malloc'):
229                 env['DYLD_INSERT_LIBRARIES'] = '/usr/lib/libgmalloc.dylib:' + self._build_path("libWebCoreTestShim.dylib")
230             else:
231                 env['DYLD_INSERT_LIBRARIES'] = self._build_path("libWebCoreTestShim.dylib")
232         env['XML_CATALOG_FILES'] = ''  # work around missing /etc/catalog <rdar://problem/4292995>
233         return env
234
235     def operating_system(self):
236         return 'ios-simulator'
237
238     def check_sys_deps(self, needs_http):
239         if not self.simulator_runtime.available:
240             _log.error('The iOS Simulator runtime with identifier "{0}" cannot be used because it is unavailable.'.format(self.simulator_runtime.identifier))
241             return False
242         testing_device = self.testing_device  # May create a new simulator device
243         if not Simulator.check_simulator_device_and_erase_if_needed(self.host, testing_device.udid):
244             _log.error('Unable to boot the simulator device with UDID {0}.'.format(testing_device.udid))
245             return False
246         return super(IOSSimulatorPort, self).check_sys_deps(needs_http)
247
248     def check_for_leaks(self, process_name, process_pid):
249         if not self.get_option('leaks'):
250             return
251         # We could use http://code.google.com/p/psutil/ to get the process_name from the pid.
252         self._leak_detector.check_for_leaks(process_name, process_pid)
253
254     def print_leaks_summary(self):
255         if not self.get_option('leaks'):
256             return
257         # We're in the manager process, so the leak detector will not have a valid list of leak files.
258         leaks_files = self._leak_detector.leaks_files_in_directory(self.results_directory())
259         if not leaks_files:
260             return
261         total_bytes_string, unique_leaks = self._leak_detector.count_total_bytes_and_unique_leaks(leaks_files)
262         total_leaks = self._leak_detector.count_total_leaks(leaks_files)
263         _log.info("%s total leaks found for a total of %s." % (total_leaks, total_bytes_string))
264         _log.info("%s unique leaks found." % unique_leaks)
265
266     def _path_to_webcore_library(self):
267         return self._build_path('WebCore.framework/Versions/A/WebCore')
268
269     def show_results_html_file(self, results_filename):
270         # We don't use self._run_script() because we don't want to wait for the script
271         # to exit and we want the output to show up on stdout in case there are errors
272         # launching the browser.
273         self._executive.popen([self.path_to_script('run-safari')] + self._arguments_for_configuration() + ['--no-saved-state', '-NSOpen', results_filename],
274             cwd=self.webkit_base(), stdout=file(os.devnull), stderr=file(os.devnull))
275
276     def sample_file_path(self, name, pid):
277         return self._filesystem.join(self.results_directory(), "{0}-{1}-sample.txt".format(name, pid))
278
279     SUBPROCESS_CRASH_REGEX = re.compile('#CRASHED - (?P<subprocess_name>\S+) \(pid (?P<subprocess_pid>\d+)\)')
280
281     def _get_crash_log(self, name, pid, stdout, stderr, newer_than, time_fn=time.time, sleep_fn=time.sleep, wait_for_log=True):
282         time_fn = time_fn or time.time
283         sleep_fn = sleep_fn or time.sleep
284
285         # FIXME: We should collect the actual crash log for DumpRenderTree.app because it includes more
286         # information (e.g. exception codes) than is available in the stack trace written to standard error.
287         stderr_lines = []
288         crashed_subprocess_name_and_pid = None  # e.g. ('DumpRenderTree.app', 1234)
289         for line in (stderr or '').splitlines():
290             if not crashed_subprocess_name_and_pid:
291                 match = self.SUBPROCESS_CRASH_REGEX.match(line)
292                 if match:
293                     crashed_subprocess_name_and_pid = (match.group('subprocess_name'), int(match.group('subprocess_pid')))
294                     continue
295             stderr_lines.append(line)
296
297         if crashed_subprocess_name_and_pid:
298             return self._get_crash_log(crashed_subprocess_name_and_pid[0], crashed_subprocess_name_and_pid[1], stdout,
299                 '\n'.join(stderr_lines), newer_than, time_fn, sleep_fn, wait_for_log)
300
301         # LayoutTestRelay crashed
302         _log.debug('looking for crash log for %s:%s' % (name, str(pid)))
303         crash_log = ''
304         crash_logs = CrashLogs(self.host)
305         now = time_fn()
306         deadline = now + 5 * int(self.get_option('child_processes', 1))
307         while not crash_log and now <= deadline:
308             crash_log = crash_logs.find_newest_log(name, pid, include_errors=True, newer_than=newer_than)
309             if not wait_for_log:
310                 break
311             if not crash_log or not [line for line in crash_log.splitlines() if not line.startswith('ERROR')]:
312                 sleep_fn(0.1)
313                 now = time_fn()
314
315         if not crash_log:
316             return stderr, None
317         return stderr, crash_log
318
319     @property
320     def testing_device(self):
321         if self._testing_device is not None:
322             return self._testing_device
323         self._testing_device = Simulator().lookup_or_create_device(self.simulator_device_type.name + ' WebKit Tester', self.simulator_device_type, self.simulator_runtime)
324         return self.testing_device
325
326     def look_for_new_crash_logs(self, crashed_processes, start_time):
327         crash_logs = {}
328         for (test_name, process_name, pid) in crashed_processes:
329             # Passing None for output.  This is a second pass after the test finished so
330             # if the output had any logging we would have already collected it.
331             crash_log = self._get_crash_log(process_name, pid, None, None, start_time, wait_for_log=False)[1]
332             if not crash_log:
333                 continue
334             crash_logs[test_name] = crash_log
335         return crash_logs
336
337     def look_for_new_samples(self, unresponsive_processes, start_time):
338         sample_files = {}
339         for (test_name, process_name, pid) in unresponsive_processes:
340             sample_file = self.sample_file_path(process_name, pid)
341             if not self._filesystem.isfile(sample_file):
342                 continue
343             sample_files[test_name] = sample_file
344         return sample_files
345
346     def sample_process(self, name, pid):
347         try:
348             hang_report = self.sample_file_path(name, pid)
349             self._executive.run_command([
350                 "/usr/bin/sample",
351                 pid,
352                 10,
353                 10,
354                 "-file",
355                 hang_report,
356             ])
357         except ScriptError as e:
358             _log.warning('Unable to sample process:' + str(e))
359
360     def _path_to_helper(self):
361         binary_name = 'LayoutTestHelper'
362         return self._build_path(binary_name)
363
364     def diff_image(self, expected_contents, actual_contents, tolerance=None):
365         if not actual_contents and not expected_contents:
366             return (None, 0, None)
367         if not actual_contents or not expected_contents:
368             return (True, 0, None)
369         if not self._image_differ:
370             self._image_differ = image_diff.IOSSimulatorImageDiffer(self)
371         self.set_option_default('tolerance', 0.1)
372         if tolerance is None:
373             tolerance = self.get_option('tolerance')
374         return self._image_differ.diff_image(expected_contents, actual_contents, tolerance)
375
376     def reset_preferences(self):
377         simulator_path = self.testing_device.path
378         data_path = os.path.join(simulator_path, 'data')
379         if os.path.isdir(data_path):
380             shutil.rmtree(data_path)
381
382     def make_command(self):
383         return self.xcrun_find('make', '/usr/bin/make')
384
385     def nm_command(self):
386         return self.xcrun_find('nm')
387
388     def xcrun_find(self, command, fallback=None):
389         fallback = fallback or command
390         try:
391             return self._executive.run_command(['xcrun', '--sdk', 'iphonesimulator', '-find', command]).rstrip()
392         except ScriptError:
393             _log.warn("xcrun failed; falling back to '%s'." % fallback)
394             return fallback
395
396     @property
397     def developer_dir(self):
398         return self._executive.run_command(['xcode-select', '--print-path']).rstrip()
399
400     def logging_patterns_to_strip(self):
401         return []
402
403     def stderr_patterns_to_strip(self):
404         return []