REGRESSION (r180239): run-webkit-test fails to boot simulator device that was booted...
[WebKit-https.git] / Tools / Scripts / webkitpy / port / ios.py
1 # Copyright (C) 2014, 2015 Apple Inc. All rights reserved.
2 #
3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions
5 # are met:
6 # 1.  Redistributions of source code must retain the above copyright
7 #     notice, this list of conditions and the following disclaimer.
8 # 2.  Redistributions in binary form must reproduce the above copyright
9 #     notice, this list of conditions and the following disclaimer in the
10 #     documentation and/or other materials provided with the distribution.
11 #
12 # THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND
13 # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
14 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
15 # DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR
16 # ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
17 # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
18 # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
19 # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
20 # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
21 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
22
23 import itertools
24 import logging
25 import os
26 import re
27 import shutil
28 import subprocess
29 import time
30
31 from webkitpy.common.memoized import memoized
32 from webkitpy.common.system.crashlogs import CrashLogs
33 from webkitpy.common.system.executive import ScriptError
34 from webkitpy.layout_tests.models.test_configuration import TestConfiguration
35 from webkitpy.port import config as port_config
36 from webkitpy.port import driver, image_diff
37 from webkitpy.port.apple import ApplePort
38 from webkitpy.port.base import Port
39 from webkitpy.port.leakdetector import LeakDetector
40 from webkitpy.xcode.simulator import Simulator, Runtime, DeviceType
41
42
43 _log = logging.getLogger(__name__)
44
45
46 class IOSPort(ApplePort):
47     port_name = "ios"
48
49     ARCHITECTURES = ['armv7', 'armv7s', 'arm64']
50     DEFAULT_ARCHITECTURE = 'armv7'
51     VERSION_FALLBACK_ORDER = ['ios-7', 'ios-8', 'ios-9']
52
53     @classmethod
54     def determine_full_port_name(cls, host, options, port_name):
55         if port_name == cls.port_name:
56             iphoneos_sdk_version = host.platform.xcode_sdk_version('iphoneos')
57             if not iphoneos_sdk_version:
58                 raise Exception("Please install the iOS SDK.")
59             major_version_number = iphoneos_sdk_version.split('.')[0]
60             port_name = port_name + '-' + major_version_number
61         return port_name
62
63     def __init__(self, *args, **kwargs):
64         super(IOSPort, self).__init__(*args, **kwargs)
65
66         self._testing_device = None
67
68     # Despite their names, these flags do not actually get passed all the way down to webkit-build.
69     def _build_driver_flags(self):
70         return ['--sdk', 'iphoneos'] + (['ARCHS=%s' % self.architecture()] if self.architecture() else [])
71
72     def operating_system(self):
73         return 'ios'
74
75
76 class IOSSimulatorPort(Port):
77     port_name = "ios-simulator"
78
79     FUTURE_VERSION = 'future'
80
81     ARCHITECTURES = ['x86_64', 'x86']
82
83     DEFAULT_ARCHITECTURE = 'x86_64'
84
85     relay_name = 'LayoutTestRelay'
86
87     def __init__(self, *args, **kwargs):
88         super(IOSSimulatorPort, self).__init__(*args, **kwargs)
89
90         self._leak_detector = LeakDetector(self)
91         if self.get_option("leaks"):
92             # DumpRenderTree slows down noticably if we run more than about 1000 tests in a batch
93             # with MallocStackLogging enabled.
94             self.set_option_default("batch_size", 1000)
95
96         self._testing_device = None
97
98     def driver_name(self):
99         if self.get_option('driver_name'):
100             return self.get_option('driver_name')
101         if self.get_option('webkit_test_runner'):
102             return 'WebKitTestRunnerApp.app'
103         return 'DumpRenderTree.app'
104
105     @property
106     @memoized
107     def simulator_runtime(self):
108         runtime_identifier = self.get_option('runtime')
109         if runtime_identifier:
110             runtime = Runtime.from_identifier(runtime_identifier)
111         else:
112             runtime = Runtime.from_version_string(self.host.platform.xcode_sdk_version('iphonesimulator'))
113         return runtime
114
115     @property
116     @memoized
117     def simulator_device_type(self):
118         device_type_identifier = self.get_option('device_type')
119         if device_type_identifier:
120             device_type = DeviceType.from_identifier(device_type_identifier)
121         else:
122             if self.architecture() == 'x86_64':
123                 device_type = DeviceType.from_name('iPhone 5s')
124             else:
125                 device_type = DeviceType.from_name('iPhone 5')
126         return device_type
127
128     @property
129     def relay_path(self):
130         mac_config = port_config.Config(self._executive, self._filesystem, 'mac')
131         return self._filesystem.join(mac_config.build_directory(self.get_option('configuration')), self.relay_name)
132
133     def default_timeout_ms(self):
134         if self.get_option('guard_malloc'):
135             return 350 * 1000
136         return super(IOSSimulatorPort, self).default_timeout_ms()
137
138     def supports_per_test_timeout(self):
139         return True
140
141     def _check_relay(self):
142         if not self._filesystem.exists(self.relay_path):
143             _log.error("%s was not found at %s" % (self.relay_name, self.relay_path))
144             return False
145         return True
146
147     def _check_build_relay(self):
148         if self.get_option('build') and not self._build_relay():
149             return False
150         if not self._check_relay():
151             return False
152         return True
153
154     def check_build(self, needs_http):
155         needs_driver = super(IOSSimulatorPort, self).check_build(needs_http)
156         return needs_driver and self._check_build_relay()
157
158     def _build_relay(self):
159         environment = self.host.copy_current_environment()
160         environment.disable_gcc_smartquotes()
161         env = environment.to_dictionary()
162
163         try:
164             self._run_script("build-layouttestrelay", env=env)
165         except ScriptError, e:
166             _log.error(e.message_with_output(output_limit=None))
167             return False
168         return True
169
170     def _build_driver(self):
171         built_tool = super(IOSSimulatorPort, self)._build_driver()
172         built_relay = self._build_relay()
173         return built_tool and built_relay
174
175     def _build_driver_flags(self):
176         archs = ['ARCHS=i386'] if self.architecture() == 'x86' else []
177         sdk = ['--sdk', 'iphonesimulator']
178         return archs + sdk
179
180     def should_retry_crashes(self):
181         return True
182
183     def _generate_all_test_configurations(self):
184         configurations = []
185         for build_type in self.ALL_BUILD_TYPES:
186             for architecture in self.ARCHITECTURES:
187                 configurations.append(TestConfiguration(version=self._version, architecture=architecture, build_type=build_type))
188         return configurations
189
190     def _driver_class(self):
191         return driver.IOSSimulatorDriver
192
193     def default_baseline_search_path(self):
194         if self.get_option('webkit_test_runner'):
195             fallback_names = [self._wk2_port_name(), 'wk2'] + [self.port_name]
196         else:
197             fallback_names = [self.port_name + '-wk1'] + [self.port_name]
198
199         return map(self._webkit_baseline_path, fallback_names)
200
201     def _port_specific_expectations_files(self):
202         return list(reversed([self._filesystem.join(self._webkit_baseline_path(p), 'TestExpectations') for p in self.baseline_search_path()]))
203
204     def setup_test_run(self):
205         device_udid = self.testing_device.udid
206         self._executive.run_command([
207             'open', '-a', os.path.join(self.developer_dir, 'Applications', 'iOS Simulator.app'),
208             '--args', '-CurrentDeviceUDID', device_udid])
209         Simulator.wait_until_device_is_in_state(device_udid, Simulator.DeviceState.BOOTED)
210
211     def clean_up_test_run(self):
212         super(IOSSimulatorPort, self).clean_up_test_run()
213         fifos = [path for path in os.listdir('/tmp') if re.search('org.webkit.(DumpRenderTree|WebKitTestRunner).*_(IN|OUT|ERROR)', path)]
214         for fifo in fifos:
215             try:
216                 os.remove(os.path.join('/tmp', fifo))
217             except OSError:
218                 _log.warning('Unable to remove ' + fifo)
219                 pass
220
221     def setup_environ_for_server(self, server_name=None):
222         env = super(IOSSimulatorPort, self).setup_environ_for_server(server_name)
223         if server_name == self.driver_name():
224             if self.get_option('leaks'):
225                 env['MallocStackLogging'] = '1'
226             if self.get_option('guard_malloc'):
227                 env['DYLD_INSERT_LIBRARIES'] = '/usr/lib/libgmalloc.dylib:' + self._build_path("libWebCoreTestShim.dylib")
228             else:
229                 env['DYLD_INSERT_LIBRARIES'] = self._build_path("libWebCoreTestShim.dylib")
230         env['XML_CATALOG_FILES'] = ''  # work around missing /etc/catalog <rdar://problem/4292995>
231         return env
232
233     def operating_system(self):
234         return 'ios-simulator'
235
236     def check_sys_deps(self, needs_http):
237         if not self.simulator_runtime.available:
238             _log.error('The iOS Simulator runtime with identifier "{0}" cannot be used because it is unavailable.'.format(self.simulator_runtime.identifier))
239             return False
240         testing_device = self.testing_device  # May create a new simulator device
241
242         # testing_device will fail to boot if it is already booted. We assume that if testing_device
243         # is booted that it was booted by the iOS Simulator app (as opposed to simctl). So, quit the
244         # iOS Simulator app to shutdown testing_device.
245         self._executive.run_command(['osascript', '-e', 'tell application "iOS Simulator" to quit'])
246         Simulator.wait_until_device_is_in_state(testing_device.udid, Simulator.DeviceState.SHUTDOWN)
247
248         if not Simulator.check_simulator_device_and_erase_if_needed(self.host, testing_device.udid):
249             _log.error('Unable to boot the simulator device with UDID {0}.'.format(testing_device.udid))
250             return False
251         return super(IOSSimulatorPort, self).check_sys_deps(needs_http)
252
253     def check_for_leaks(self, process_name, process_pid):
254         if not self.get_option('leaks'):
255             return
256         # We could use http://code.google.com/p/psutil/ to get the process_name from the pid.
257         self._leak_detector.check_for_leaks(process_name, process_pid)
258
259     def print_leaks_summary(self):
260         if not self.get_option('leaks'):
261             return
262         # We're in the manager process, so the leak detector will not have a valid list of leak files.
263         leaks_files = self._leak_detector.leaks_files_in_directory(self.results_directory())
264         if not leaks_files:
265             return
266         total_bytes_string, unique_leaks = self._leak_detector.count_total_bytes_and_unique_leaks(leaks_files)
267         total_leaks = self._leak_detector.count_total_leaks(leaks_files)
268         _log.info("%s total leaks found for a total of %s." % (total_leaks, total_bytes_string))
269         _log.info("%s unique leaks found." % unique_leaks)
270
271     def _path_to_webcore_library(self):
272         return self._build_path('WebCore.framework/Versions/A/WebCore')
273
274     def show_results_html_file(self, results_filename):
275         # We don't use self._run_script() because we don't want to wait for the script
276         # to exit and we want the output to show up on stdout in case there are errors
277         # launching the browser.
278         self._executive.popen([self.path_to_script('run-safari')] + self._arguments_for_configuration() + ['--no-saved-state', '-NSOpen', results_filename],
279             cwd=self.webkit_base(), stdout=file(os.devnull), stderr=file(os.devnull))
280
281     def sample_file_path(self, name, pid):
282         return self._filesystem.join(self.results_directory(), "{0}-{1}-sample.txt".format(name, pid))
283
284     SUBPROCESS_CRASH_REGEX = re.compile('#CRASHED - (?P<subprocess_name>\S+) \(pid (?P<subprocess_pid>\d+)\)')
285
286     def _get_crash_log(self, name, pid, stdout, stderr, newer_than, time_fn=time.time, sleep_fn=time.sleep, wait_for_log=True):
287         time_fn = time_fn or time.time
288         sleep_fn = sleep_fn or time.sleep
289
290         # FIXME: We should collect the actual crash log for DumpRenderTree.app because it includes more
291         # information (e.g. exception codes) than is available in the stack trace written to standard error.
292         stderr_lines = []
293         crashed_subprocess_name_and_pid = None  # e.g. ('DumpRenderTree.app', 1234)
294         for line in (stderr or '').splitlines():
295             if not crashed_subprocess_name_and_pid:
296                 match = self.SUBPROCESS_CRASH_REGEX.match(line)
297                 if match:
298                     crashed_subprocess_name_and_pid = (match.group('subprocess_name'), int(match.group('subprocess_pid')))
299                     continue
300             stderr_lines.append(line)
301
302         if crashed_subprocess_name_and_pid:
303             return self._get_crash_log(crashed_subprocess_name_and_pid[0], crashed_subprocess_name_and_pid[1], stdout,
304                 '\n'.join(stderr_lines), newer_than, time_fn, sleep_fn, wait_for_log)
305
306         # LayoutTestRelay crashed
307         _log.debug('looking for crash log for %s:%s' % (name, str(pid)))
308         crash_log = ''
309         crash_logs = CrashLogs(self.host)
310         now = time_fn()
311         deadline = now + 5 * int(self.get_option('child_processes', 1))
312         while not crash_log and now <= deadline:
313             crash_log = crash_logs.find_newest_log(name, pid, include_errors=True, newer_than=newer_than)
314             if not wait_for_log:
315                 break
316             if not crash_log or not [line for line in crash_log.splitlines() if not line.startswith('ERROR')]:
317                 sleep_fn(0.1)
318                 now = time_fn()
319
320         if not crash_log:
321             return stderr, None
322         return stderr, crash_log
323
324     @property
325     def testing_device(self):
326         if self._testing_device is not None:
327             return self._testing_device
328         self._testing_device = Simulator().lookup_or_create_device(self.simulator_device_type.name + ' WebKit Tester', self.simulator_device_type, self.simulator_runtime)
329         return self.testing_device
330
331     def look_for_new_crash_logs(self, crashed_processes, start_time):
332         crash_logs = {}
333         for (test_name, process_name, pid) in crashed_processes:
334             # Passing None for output.  This is a second pass after the test finished so
335             # if the output had any logging we would have already collected it.
336             crash_log = self._get_crash_log(process_name, pid, None, None, start_time, wait_for_log=False)[1]
337             if not crash_log:
338                 continue
339             crash_logs[test_name] = crash_log
340         return crash_logs
341
342     def look_for_new_samples(self, unresponsive_processes, start_time):
343         sample_files = {}
344         for (test_name, process_name, pid) in unresponsive_processes:
345             sample_file = self.sample_file_path(process_name, pid)
346             if not self._filesystem.isfile(sample_file):
347                 continue
348             sample_files[test_name] = sample_file
349         return sample_files
350
351     def sample_process(self, name, pid):
352         try:
353             hang_report = self.sample_file_path(name, pid)
354             self._executive.run_command([
355                 "/usr/bin/sample",
356                 pid,
357                 10,
358                 10,
359                 "-file",
360                 hang_report,
361             ])
362         except ScriptError as e:
363             _log.warning('Unable to sample process:' + str(e))
364
365     def _path_to_helper(self):
366         binary_name = 'LayoutTestHelper'
367         return self._build_path(binary_name)
368
369     def diff_image(self, expected_contents, actual_contents, tolerance=None):
370         if not actual_contents and not expected_contents:
371             return (None, 0, None)
372         if not actual_contents or not expected_contents:
373             return (True, 0, None)
374         if not self._image_differ:
375             self._image_differ = image_diff.IOSSimulatorImageDiffer(self)
376         self.set_option_default('tolerance', 0.1)
377         if tolerance is None:
378             tolerance = self.get_option('tolerance')
379         return self._image_differ.diff_image(expected_contents, actual_contents, tolerance)
380
381     def reset_preferences(self):
382         simulator_path = self.testing_device.path
383         data_path = os.path.join(simulator_path, 'data')
384         if os.path.isdir(data_path):
385             shutil.rmtree(data_path)
386
387     def make_command(self):
388         return self.xcrun_find('make', '/usr/bin/make')
389
390     def nm_command(self):
391         return self.xcrun_find('nm')
392
393     def xcrun_find(self, command, fallback=None):
394         fallback = fallback or command
395         try:
396             return self._executive.run_command(['xcrun', '--sdk', 'iphonesimulator', '-find', command]).rstrip()
397         except ScriptError:
398             _log.warn("xcrun failed; falling back to '%s'." % fallback)
399             return fallback
400
401     @property
402     def developer_dir(self):
403         return self._executive.run_command(['xcode-select', '--print-path']).rstrip()
404
405     def logging_patterns_to_strip(self):
406         return []
407
408     def stderr_patterns_to_strip(self):
409         return []