a1c79be8e7663c3f56a2d5578f8d9081f41ac875
[WebKit-https.git] / Tools / Scripts / webkitpy / port / ios.py
1 # Copyright (C) 2014, 2015 Apple Inc. All rights reserved.
2 #
3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions
5 # are met:
6 # 1.  Redistributions of source code must retain the above copyright
7 #     notice, this list of conditions and the following disclaimer.
8 # 2.  Redistributions in binary form must reproduce the above copyright
9 #     notice, this list of conditions and the following disclaimer in the
10 #     documentation and/or other materials provided with the distribution.
11 #
12 # THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND
13 # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
14 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
15 # DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR
16 # ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
17 # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
18 # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
19 # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
20 # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
21 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
22
23 import logging
24 import os
25 import shutil
26 import time
27 import re
28 import itertools
29 import subprocess
30
31 from webkitpy.layout_tests.models.test_configuration import TestConfiguration
32 from webkitpy.common.system.crashlogs import CrashLogs
33 from webkitpy.common.system.executive import ScriptError
34 from webkitpy.port.apple import ApplePort
35 from webkitpy.port import driver, image_diff
36 from webkitpy.port.base import Port
37 from webkitpy.port.leakdetector import LeakDetector
38 from webkitpy.port import config as port_config
39 from webkitpy.xcode import simulator
40
41
42 _log = logging.getLogger(__name__)
43
44
45 class IOSPort(ApplePort):
46     port_name = "ios-device"
47
48     ARCHITECTURES = ['armv7', 'armv7s', 'arm64']
49     VERSION_FALLBACK_ORDER = ['ios-device-7', 'ios-device-8']
50
51     @classmethod
52     def determine_full_port_name(cls, host, options, port_name):
53         if port_name == cls.port_name:
54             sdk_version = '8.0'
55             if host.platform.is_mac():
56                 sdk_command_process = subprocess.Popen('xcrun --sdk iphoneos --show-sdk-version', stdout=subprocess.PIPE, stderr=None, shell=True)
57                 sdk_command_output = sdk_command_process.communicate()[0].strip()
58                 if sdk_command_output:
59                     sdk_version = sdk_command_output
60
61             port_name = port_name + '-' + re.match('^([0-9]+).*', sdk_version).group(1)
62
63         return port_name
64
65     def __init__(self, *args, **kwargs):
66         super(IOSPort, self).__init__(*args, **kwargs)
67
68         self._architecture = self.get_option('architecture')
69
70         if not self._architecture:
71             self._architecture = 'armv7'
72
73         self._testing_device = None
74
75     # Despite their names, these flags do not actually get passed all the way down to webkit-build.
76     def _build_driver_flags(self):
77         return ['--sdk', 'iphoneos'] + (['ARCHS=%s' % self._architecture] if self._architecture else [])
78
79     def operating_system(self):
80         return 'ios'
81
82
83 class IOSSimulatorPort(Port):
84     port_name = "ios-simulator"
85
86     FUTURE_VERSION = 'future'
87
88     ARCHITECTURES = ['x86_64', 'x86']
89
90     relay_name = 'LayoutTestRelay'
91
92     def __init__(self, *args, **kwargs):
93         super(IOSSimulatorPort, self).__init__(*args, **kwargs)
94
95         self._architecture = self.get_option('architecture')
96
97         if not self._architecture:
98             self._architecture = 'x86_64'
99
100         self._leak_detector = LeakDetector(self)
101         if self.get_option("leaks"):
102             # DumpRenderTree slows down noticably if we run more than about 1000 tests in a batch
103             # with MallocStackLogging enabled.
104             self.set_option_default("batch_size", 1000)
105         mac_config = port_config.Config(self._executive, self._filesystem, 'mac')
106         self._mac_build_directory = mac_config.build_directory(self.get_option('configuration'))
107
108         self._testing_device = None
109
110     def driver_name(self):
111         if self.get_option('driver_name'):
112             return self.get_option('driver_name')
113         if self.get_option('webkit_test_runner'):
114             return 'WebKitTestRunnerApp.app'
115         return 'DumpRenderTree.app'
116
117     @property
118     def relay_path(self):
119         return self._filesystem.join(self._mac_build_directory, self.relay_name)
120
121     def default_timeout_ms(self):
122         if self.get_option('guard_malloc'):
123             return 350 * 1000
124         return super(IOSSimulatorPort, self).default_timeout_ms()
125
126     def supports_per_test_timeout(self):
127         return True
128
129     def _check_relay(self):
130         if not self._filesystem.exists(self.relay_path):
131             _log.error("%s was not found at %s" % (self.relay_name, self.relay_path))
132             return False
133         return True
134
135     def _check_build_relay(self):
136         if self.get_option('build') and not self._build_relay():
137             return False
138         if not self._check_relay():
139             return False
140         return True
141
142     def check_build(self, needs_http):
143         needs_driver = super(IOSSimulatorPort, self).check_build(needs_http)
144         return needs_driver and self._check_build_relay()
145
146     def _build_relay(self):
147         environment = self.host.copy_current_environment()
148         environment.disable_gcc_smartquotes()
149         env = environment.to_dictionary()
150
151         try:
152             self._run_script("build-layouttestrelay", env=env)
153         except ScriptError, e:
154             _log.error(e.message_with_output(output_limit=None))
155             return False
156         return True
157
158     def _build_driver(self):
159         built_tool = super(IOSSimulatorPort, self)._build_driver()
160         built_relay = self._build_relay()
161         return built_tool and built_relay
162
163     def _build_driver_flags(self):
164         archs = ['ARCHS=i386'] if self.architecture() == 'x86' else []
165         sdk = ['--sdk', 'iphonesimulator']
166         return archs + sdk
167
168     def should_retry_crashes(self):
169         return True
170
171     def _generate_all_test_configurations(self):
172         configurations = []
173         for build_type in self.ALL_BUILD_TYPES:
174             for architecture in self.ARCHITECTURES:
175                 configurations.append(TestConfiguration(version=self._version, architecture=architecture, build_type=build_type))
176         return configurations
177
178     def _driver_class(self):
179         return driver.IOSSimulatorDriver
180
181     def default_baseline_search_path(self):
182         if self.get_option('webkit_test_runner'):
183             fallback_names = [self.port_name + '-wk2'] + [self.port_name]
184         else:
185             fallback_names = [self.port_name]
186
187         return map(self._webkit_baseline_path, fallback_names)
188
189     def _port_specific_expectations_files(self):
190         return list(reversed([self._filesystem.join(self._webkit_baseline_path(p), 'TestExpectations') for p in self.baseline_search_path()]))
191
192     def setup_test_run(self):
193         self._executive.run_command(['osascript', '-e', 'tell application "iOS Simulator" to quit'])
194         time.sleep(2)
195         self._executive.run_command([
196             'open', '-a', os.path.join(self.developer_dir, 'Applications', 'iOS Simulator.app'),
197             '--args', '-CurrentDeviceUDID', self.testing_device.udid])
198
199     def clean_up_test_run(self):
200         super(IOSSimulatorPort, self).clean_up_test_run()
201         fifos = [path for path in os.listdir('/tmp') if re.search('org.webkit.(DumpRenderTree|WebKitTestRunner).*_(IN|OUT|ERROR)', path)]
202         for fifo in fifos:
203             try:
204                 os.remove(os.path.join('/tmp', fifo))
205             except OSError:
206                 _log.warning('Unable to remove ' + fifo)
207                 pass
208
209     def setup_environ_for_server(self, server_name=None):
210         env = super(IOSSimulatorPort, self).setup_environ_for_server(server_name)
211         if server_name == self.driver_name():
212             if self.get_option('leaks'):
213                 env['MallocStackLogging'] = '1'
214             if self.get_option('guard_malloc'):
215                 env['DYLD_INSERT_LIBRARIES'] = '/usr/lib/libgmalloc.dylib:' + self._build_path("libWebCoreTestShim.dylib")
216             else:
217                 env['DYLD_INSERT_LIBRARIES'] = self._build_path("libWebCoreTestShim.dylib")
218         env['XML_CATALOG_FILES'] = ''  # work around missing /etc/catalog <rdar://problem/4292995>
219         return env
220
221     def operating_system(self):
222         return 'ios-simulator'
223
224     def check_for_leaks(self, process_name, process_pid):
225         if not self.get_option('leaks'):
226             return
227         # We could use http://code.google.com/p/psutil/ to get the process_name from the pid.
228         self._leak_detector.check_for_leaks(process_name, process_pid)
229
230     def print_leaks_summary(self):
231         if not self.get_option('leaks'):
232             return
233         # We're in the manager process, so the leak detector will not have a valid list of leak files.
234         leaks_files = self._leak_detector.leaks_files_in_directory(self.results_directory())
235         if not leaks_files:
236             return
237         total_bytes_string, unique_leaks = self._leak_detector.count_total_bytes_and_unique_leaks(leaks_files)
238         total_leaks = self._leak_detector.count_total_leaks(leaks_files)
239         _log.info("%s total leaks found for a total of %s!" % (total_leaks, total_bytes_string))
240         _log.info("%s unique leaks found!" % unique_leaks)
241
242     def _path_to_webcore_library(self):
243         return self._build_path('WebCore.framework/Versions/A/WebCore')
244
245     def show_results_html_file(self, results_filename):
246         # We don't use self._run_script() because we don't want to wait for the script
247         # to exit and we want the output to show up on stdout in case there are errors
248         # launching the browser.
249         self._executive.popen([self.path_to_script('run-safari')] + self._arguments_for_configuration() + ['--no-saved-state', '-NSOpen', results_filename],
250             cwd=self.webkit_base(), stdout=file(os.devnull), stderr=file(os.devnull))
251
252     def sample_file_path(self, name, pid):
253         return self._filesystem.join(self.results_directory(), "{0}-{1}-sample.txt".format(name, pid))
254
255     SUBPROCESS_CRASH_REGEX = re.compile('#CRASHED - (?P<subprocess_name>\S+) \(pid (?P<subprocess_pid>\d+)\)')
256
257     def _get_crash_log(self, name, pid, stdout, stderr, newer_than, time_fn=time.time, sleep_fn=time.sleep, wait_for_log=True):
258         time_fn = time_fn or time.time
259         sleep_fn = sleep_fn or time.sleep
260
261         # FIXME: We should collect the actual crash log for DumpRenderTree.app because it includes more
262         # information (e.g. exception codes) than is available in the stack trace written to standard error.
263         stderr_lines = []
264         crashed_subprocess_name_and_pid = None  # e.g. ('DumpRenderTree.app', 1234)
265         for line in (stderr or '').splitlines():
266             if not crashed_subprocess_name_and_pid:
267                 match = self.SUBPROCESS_CRASH_REGEX.match(line)
268                 if match:
269                     crashed_subprocess_name_and_pid = (match.group('subprocess_name'), int(match.group('subprocess_pid')))
270                     continue
271             stderr_lines.append(line)
272
273         if crashed_subprocess_name_and_pid:
274             return self._get_crash_log(crashed_subprocess_name_and_pid[0], crashed_subprocess_name_and_pid[1], stdout,
275                 '\n'.join(stderr_lines), newer_than, time_fn, sleep_fn, wait_for_log)
276
277         # LayoutTestRelay crashed
278         _log.debug('looking for crash log for %s:%s' % (name, str(pid)))
279         crash_log = ''
280         crash_logs = CrashLogs(self.host)
281         now = time_fn()
282         deadline = now + 5 * int(self.get_option('child_processes', 1))
283         while not crash_log and now <= deadline:
284             crash_log = crash_logs.find_newest_log(name, pid, include_errors=True, newer_than=newer_than)
285             if not wait_for_log:
286                 break
287             if not crash_log or not [line for line in crash_log.splitlines() if not line.startswith('ERROR')]:
288                 sleep_fn(0.1)
289                 now = time_fn()
290
291         if not crash_log:
292             return stderr, None
293         return stderr, crash_log
294
295     @property
296     def testing_device(self):
297         if self._testing_device is not None:
298             return self._testing_device
299
300         device_type = self.get_option('device_type')
301         runtime = self.get_option('runtime')
302         self._testing_device = simulator.Simulator().lookup_or_create_device(device_type.name + ' WebKit Tester', device_type, runtime)
303         return self.testing_device
304
305     def simulator_path(self, udid):
306         if udid:
307             return os.path.realpath(os.path.expanduser(os.path.join('~/Library/Developer/CoreSimulator/Devices', udid)))
308
309     def look_for_new_crash_logs(self, crashed_processes, start_time):
310         crash_logs = {}
311         for (test_name, process_name, pid) in crashed_processes:
312             # Passing None for output.  This is a second pass after the test finished so
313             # if the output had any logging we would have already collected it.
314             crash_log = self._get_crash_log(process_name, pid, None, None, start_time, wait_for_log=False)[1]
315             if not crash_log:
316                 continue
317             crash_logs[test_name] = crash_log
318         return crash_logs
319
320     def look_for_new_samples(self, unresponsive_processes, start_time):
321         sample_files = {}
322         for (test_name, process_name, pid) in unresponsive_processes:
323             sample_file = self.sample_file_path(process_name, pid)
324             if not self._filesystem.isfile(sample_file):
325                 continue
326             sample_files[test_name] = sample_file
327         return sample_files
328
329     def sample_process(self, name, pid):
330         try:
331             hang_report = self.sample_file_path(name, pid)
332             self._executive.run_command([
333                 "/usr/bin/sample",
334                 pid,
335                 10,
336                 10,
337                 "-file",
338                 hang_report,
339             ])
340         except ScriptError as e:
341             _log.warning('Unable to sample process:' + str(e))
342
343     def _path_to_helper(self):
344         binary_name = 'LayoutTestHelper'
345         return self._build_path(binary_name)
346
347     def diff_image(self, expected_contents, actual_contents, tolerance=None):
348         if not actual_contents and not expected_contents:
349             return (None, 0, None)
350         if not actual_contents or not expected_contents:
351             return (True, 0, None)
352         if not self._image_differ:
353             self._image_differ = image_diff.IOSSimulatorImageDiffer(self)
354         self.set_option_default('tolerance', 0.1)
355         if tolerance is None:
356             tolerance = self.get_option('tolerance')
357         return self._image_differ.diff_image(expected_contents, actual_contents, tolerance)
358
359     def reset_preferences(self):
360         simulator_path = self.testing_device.path
361         data_path = os.path.join(simulator_path, 'data')
362         if os.path.isdir(data_path):
363             shutil.rmtree(data_path)
364
365     def make_command(self):
366         return self.xcrun_find('make', '/usr/bin/make')
367
368     def nm_command(self):
369         return self.xcrun_find('nm')
370
371     def xcrun_find(self, command, fallback=None):
372         fallback = fallback or command
373         try:
374             return self._executive.run_command(['xcrun', '--sdk', 'iphonesimulator', '-find', command]).rstrip()
375         except ScriptError:
376             _log.warn("xcrun failed; falling back to '%s'." % fallback)
377             return fallback
378
379     @property
380     def developer_dir(self):
381         return self._executive.run_command(['xcode-select', '--print-path']).rstrip()
382
383     def logging_patterns_to_strip(self):
384         return []
385
386     def stderr_patterns_to_strip(self):
387         return []