test-webkitpy fails on Mac without iphoneos SDK
[WebKit-https.git] / Tools / Scripts / webkitpy / port / ios.py
1 # Copyright (C) 2014, 2015 Apple Inc. All rights reserved.
2 #
3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions
5 # are met:
6 # 1.  Redistributions of source code must retain the above copyright
7 #     notice, this list of conditions and the following disclaimer.
8 # 2.  Redistributions in binary form must reproduce the above copyright
9 #     notice, this list of conditions and the following disclaimer in the
10 #     documentation and/or other materials provided with the distribution.
11 #
12 # THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND
13 # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
14 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
15 # DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR
16 # ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
17 # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
18 # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
19 # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
20 # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
21 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
22
23 import logging
24 import os
25 import shutil
26 import time
27 import re
28 import itertools
29 import subprocess
30
31 from webkitpy.layout_tests.models.test_configuration import TestConfiguration
32 from webkitpy.common.system.crashlogs import CrashLogs
33 from webkitpy.common.system.executive import ScriptError
34 from webkitpy.port.apple import ApplePort
35 from webkitpy.port import driver, image_diff
36 from webkitpy.port.base import Port
37 from webkitpy.port.leakdetector import LeakDetector
38 from webkitpy.port import config as port_config
39 from webkitpy.xcode import simulator
40
41
42 _log = logging.getLogger(__name__)
43
44
45 class IOSPort(ApplePort):
46     port_name = "ios"
47
48     ARCHITECTURES = ['armv7', 'armv7s', 'arm64']
49     DEFAULT_ARCHITECTURE = 'armv7'
50     VERSION_FALLBACK_ORDER = ['ios-7', 'ios-8', 'ios-9']
51
52     @classmethod
53     def determine_full_port_name(cls, host, options, port_name):
54         if port_name == cls.port_name:
55             iphoneos_sdk_version = host.platform.xcode_sdk_version('iphoneos')
56             if not iphoneos_sdk_version:
57                 raise Exception("Please install the iOS SDK.")
58             major_version_number = iphoneos_sdk_version.split('.')[0]
59             port_name = port_name + '-' + major_version_number
60         return port_name
61
62     def __init__(self, *args, **kwargs):
63         super(IOSPort, self).__init__(*args, **kwargs)
64
65         self._testing_device = None
66
67     # Despite their names, these flags do not actually get passed all the way down to webkit-build.
68     def _build_driver_flags(self):
69         return ['--sdk', 'iphoneos'] + (['ARCHS=%s' % self.architecture()] if self.architecture() else [])
70
71     def operating_system(self):
72         return 'ios'
73
74
75 class IOSSimulatorPort(Port):
76     port_name = "ios-simulator"
77
78     FUTURE_VERSION = 'future'
79
80     ARCHITECTURES = ['x86_64', 'x86']
81
82     DEFAULT_ARCHITECTURE = 'x86_64'
83
84     relay_name = 'LayoutTestRelay'
85
86     def __init__(self, *args, **kwargs):
87         super(IOSSimulatorPort, self).__init__(*args, **kwargs)
88
89         self._leak_detector = LeakDetector(self)
90         if self.get_option("leaks"):
91             # DumpRenderTree slows down noticably if we run more than about 1000 tests in a batch
92             # with MallocStackLogging enabled.
93             self.set_option_default("batch_size", 1000)
94
95         self._testing_device = None
96
97     def driver_name(self):
98         if self.get_option('driver_name'):
99             return self.get_option('driver_name')
100         if self.get_option('webkit_test_runner'):
101             return 'WebKitTestRunnerApp.app'
102         return 'DumpRenderTree.app'
103
104     @property
105     def relay_path(self):
106         mac_config = port_config.Config(self._executive, self._filesystem, 'mac')
107         return self._filesystem.join(mac_config.build_directory(self.get_option('configuration')), self.relay_name)
108
109     def default_timeout_ms(self):
110         if self.get_option('guard_malloc'):
111             return 350 * 1000
112         return super(IOSSimulatorPort, self).default_timeout_ms()
113
114     def supports_per_test_timeout(self):
115         return True
116
117     def _check_relay(self):
118         if not self._filesystem.exists(self.relay_path):
119             _log.error("%s was not found at %s" % (self.relay_name, self.relay_path))
120             return False
121         return True
122
123     def _check_build_relay(self):
124         if self.get_option('build') and not self._build_relay():
125             return False
126         if not self._check_relay():
127             return False
128         return True
129
130     def check_build(self, needs_http):
131         needs_driver = super(IOSSimulatorPort, self).check_build(needs_http)
132         return needs_driver and self._check_build_relay()
133
134     def _build_relay(self):
135         environment = self.host.copy_current_environment()
136         environment.disable_gcc_smartquotes()
137         env = environment.to_dictionary()
138
139         try:
140             self._run_script("build-layouttestrelay", env=env)
141         except ScriptError, e:
142             _log.error(e.message_with_output(output_limit=None))
143             return False
144         return True
145
146     def _build_driver(self):
147         built_tool = super(IOSSimulatorPort, self)._build_driver()
148         built_relay = self._build_relay()
149         return built_tool and built_relay
150
151     def _build_driver_flags(self):
152         archs = ['ARCHS=i386'] if self.architecture() == 'x86' else []
153         sdk = ['--sdk', 'iphonesimulator']
154         return archs + sdk
155
156     def should_retry_crashes(self):
157         return True
158
159     def _generate_all_test_configurations(self):
160         configurations = []
161         for build_type in self.ALL_BUILD_TYPES:
162             for architecture in self.ARCHITECTURES:
163                 configurations.append(TestConfiguration(version=self._version, architecture=architecture, build_type=build_type))
164         return configurations
165
166     def _driver_class(self):
167         return driver.IOSSimulatorDriver
168
169     def default_baseline_search_path(self):
170         if self.get_option('webkit_test_runner'):
171             fallback_names = [self._wk2_port_name(), 'wk2'] + [self.port_name]
172         else:
173             fallback_names = [self.port_name + '-wk1'] + [self.port_name]
174
175         return map(self._webkit_baseline_path, fallback_names)
176
177     def _port_specific_expectations_files(self):
178         return list(reversed([self._filesystem.join(self._webkit_baseline_path(p), 'TestExpectations') for p in self.baseline_search_path()]))
179
180     def setup_test_run(self):
181         self._executive.run_command(['osascript', '-e', 'tell application "iOS Simulator" to quit'])
182         time.sleep(2)
183         self._executive.run_command([
184             'open', '-a', os.path.join(self.developer_dir, 'Applications', 'iOS Simulator.app'),
185             '--args', '-CurrentDeviceUDID', self.testing_device.udid])
186
187     def clean_up_test_run(self):
188         super(IOSSimulatorPort, self).clean_up_test_run()
189         fifos = [path for path in os.listdir('/tmp') if re.search('org.webkit.(DumpRenderTree|WebKitTestRunner).*_(IN|OUT|ERROR)', path)]
190         for fifo in fifos:
191             try:
192                 os.remove(os.path.join('/tmp', fifo))
193             except OSError:
194                 _log.warning('Unable to remove ' + fifo)
195                 pass
196
197     def setup_environ_for_server(self, server_name=None):
198         env = super(IOSSimulatorPort, self).setup_environ_for_server(server_name)
199         if server_name == self.driver_name():
200             if self.get_option('leaks'):
201                 env['MallocStackLogging'] = '1'
202             if self.get_option('guard_malloc'):
203                 env['DYLD_INSERT_LIBRARIES'] = '/usr/lib/libgmalloc.dylib:' + self._build_path("libWebCoreTestShim.dylib")
204             else:
205                 env['DYLD_INSERT_LIBRARIES'] = self._build_path("libWebCoreTestShim.dylib")
206         env['XML_CATALOG_FILES'] = ''  # work around missing /etc/catalog <rdar://problem/4292995>
207         return env
208
209     def operating_system(self):
210         return 'ios-simulator'
211
212     def check_for_leaks(self, process_name, process_pid):
213         if not self.get_option('leaks'):
214             return
215         # We could use http://code.google.com/p/psutil/ to get the process_name from the pid.
216         self._leak_detector.check_for_leaks(process_name, process_pid)
217
218     def print_leaks_summary(self):
219         if not self.get_option('leaks'):
220             return
221         # We're in the manager process, so the leak detector will not have a valid list of leak files.
222         leaks_files = self._leak_detector.leaks_files_in_directory(self.results_directory())
223         if not leaks_files:
224             return
225         total_bytes_string, unique_leaks = self._leak_detector.count_total_bytes_and_unique_leaks(leaks_files)
226         total_leaks = self._leak_detector.count_total_leaks(leaks_files)
227         _log.info("%s total leaks found for a total of %s." % (total_leaks, total_bytes_string))
228         _log.info("%s unique leaks found." % unique_leaks)
229
230     def _path_to_webcore_library(self):
231         return self._build_path('WebCore.framework/Versions/A/WebCore')
232
233     def show_results_html_file(self, results_filename):
234         # We don't use self._run_script() because we don't want to wait for the script
235         # to exit and we want the output to show up on stdout in case there are errors
236         # launching the browser.
237         self._executive.popen([self.path_to_script('run-safari')] + self._arguments_for_configuration() + ['--no-saved-state', '-NSOpen', results_filename],
238             cwd=self.webkit_base(), stdout=file(os.devnull), stderr=file(os.devnull))
239
240     def sample_file_path(self, name, pid):
241         return self._filesystem.join(self.results_directory(), "{0}-{1}-sample.txt".format(name, pid))
242
243     SUBPROCESS_CRASH_REGEX = re.compile('#CRASHED - (?P<subprocess_name>\S+) \(pid (?P<subprocess_pid>\d+)\)')
244
245     def _get_crash_log(self, name, pid, stdout, stderr, newer_than, time_fn=time.time, sleep_fn=time.sleep, wait_for_log=True):
246         time_fn = time_fn or time.time
247         sleep_fn = sleep_fn or time.sleep
248
249         # FIXME: We should collect the actual crash log for DumpRenderTree.app because it includes more
250         # information (e.g. exception codes) than is available in the stack trace written to standard error.
251         stderr_lines = []
252         crashed_subprocess_name_and_pid = None  # e.g. ('DumpRenderTree.app', 1234)
253         for line in (stderr or '').splitlines():
254             if not crashed_subprocess_name_and_pid:
255                 match = self.SUBPROCESS_CRASH_REGEX.match(line)
256                 if match:
257                     crashed_subprocess_name_and_pid = (match.group('subprocess_name'), int(match.group('subprocess_pid')))
258                     continue
259             stderr_lines.append(line)
260
261         if crashed_subprocess_name_and_pid:
262             return self._get_crash_log(crashed_subprocess_name_and_pid[0], crashed_subprocess_name_and_pid[1], stdout,
263                 '\n'.join(stderr_lines), newer_than, time_fn, sleep_fn, wait_for_log)
264
265         # LayoutTestRelay crashed
266         _log.debug('looking for crash log for %s:%s' % (name, str(pid)))
267         crash_log = ''
268         crash_logs = CrashLogs(self.host)
269         now = time_fn()
270         deadline = now + 5 * int(self.get_option('child_processes', 1))
271         while not crash_log and now <= deadline:
272             crash_log = crash_logs.find_newest_log(name, pid, include_errors=True, newer_than=newer_than)
273             if not wait_for_log:
274                 break
275             if not crash_log or not [line for line in crash_log.splitlines() if not line.startswith('ERROR')]:
276                 sleep_fn(0.1)
277                 now = time_fn()
278
279         if not crash_log:
280             return stderr, None
281         return stderr, crash_log
282
283     @property
284     def testing_device(self):
285         if self._testing_device is not None:
286             return self._testing_device
287
288         device_type = self.get_option('device_type')
289         runtime = self.get_option('runtime')
290         self._testing_device = simulator.Simulator().lookup_or_create_device(device_type.name + ' WebKit Tester', device_type, runtime)
291         return self.testing_device
292
293     def simulator_path(self, udid):
294         if udid:
295             return os.path.realpath(os.path.expanduser(os.path.join('~/Library/Developer/CoreSimulator/Devices', udid)))
296
297     def look_for_new_crash_logs(self, crashed_processes, start_time):
298         crash_logs = {}
299         for (test_name, process_name, pid) in crashed_processes:
300             # Passing None for output.  This is a second pass after the test finished so
301             # if the output had any logging we would have already collected it.
302             crash_log = self._get_crash_log(process_name, pid, None, None, start_time, wait_for_log=False)[1]
303             if not crash_log:
304                 continue
305             crash_logs[test_name] = crash_log
306         return crash_logs
307
308     def look_for_new_samples(self, unresponsive_processes, start_time):
309         sample_files = {}
310         for (test_name, process_name, pid) in unresponsive_processes:
311             sample_file = self.sample_file_path(process_name, pid)
312             if not self._filesystem.isfile(sample_file):
313                 continue
314             sample_files[test_name] = sample_file
315         return sample_files
316
317     def sample_process(self, name, pid):
318         try:
319             hang_report = self.sample_file_path(name, pid)
320             self._executive.run_command([
321                 "/usr/bin/sample",
322                 pid,
323                 10,
324                 10,
325                 "-file",
326                 hang_report,
327             ])
328         except ScriptError as e:
329             _log.warning('Unable to sample process:' + str(e))
330
331     def _path_to_helper(self):
332         binary_name = 'LayoutTestHelper'
333         return self._build_path(binary_name)
334
335     def diff_image(self, expected_contents, actual_contents, tolerance=None):
336         if not actual_contents and not expected_contents:
337             return (None, 0, None)
338         if not actual_contents or not expected_contents:
339             return (True, 0, None)
340         if not self._image_differ:
341             self._image_differ = image_diff.IOSSimulatorImageDiffer(self)
342         self.set_option_default('tolerance', 0.1)
343         if tolerance is None:
344             tolerance = self.get_option('tolerance')
345         return self._image_differ.diff_image(expected_contents, actual_contents, tolerance)
346
347     def reset_preferences(self):
348         simulator_path = self.testing_device.path
349         data_path = os.path.join(simulator_path, 'data')
350         if os.path.isdir(data_path):
351             shutil.rmtree(data_path)
352
353     def make_command(self):
354         return self.xcrun_find('make', '/usr/bin/make')
355
356     def nm_command(self):
357         return self.xcrun_find('nm')
358
359     def xcrun_find(self, command, fallback=None):
360         fallback = fallback or command
361         try:
362             return self._executive.run_command(['xcrun', '--sdk', 'iphonesimulator', '-find', command]).rstrip()
363         except ScriptError:
364             _log.warn("xcrun failed; falling back to '%s'." % fallback)
365             return fallback
366
367     @property
368     def developer_dir(self):
369         return self._executive.run_command(['xcode-select', '--print-path']).rstrip()
370
371     def logging_patterns_to_strip(self):
372         return []
373
374     def stderr_patterns_to_strip(self):
375         return []