b23289869585bc0a391db7dc247cd6a431f41920
[WebKit-https.git] / Tools / Scripts / webkitpy / port / ios.py
1 # Copyright (C) 2014 Apple Inc. All rights reserved.
2 #
3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions
5 # are met:
6 # 1.  Redistributions of source code must retain the above copyright
7 #     notice, this list of conditions and the following disclaimer.
8 # 2.  Redistributions in binary form must reproduce the above copyright
9 #     notice, this list of conditions and the following disclaimer in the
10 #     documentation and/or other materials provided with the distribution.
11 #
12 # THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND
13 # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
14 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
15 # DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR
16 # ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
17 # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
18 # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
19 # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
20 # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
21 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
22
23 import logging
24 import os
25 import shutil
26 import time
27 import re
28 import itertools
29 import subprocess
30
31 from webkitpy.layout_tests.models.test_configuration import TestConfiguration
32 from webkitpy.common.system.crashlogs import CrashLogs
33 from webkitpy.common.system.executive import ScriptError
34 from webkitpy.port import driver
35 from webkitpy.port.base import Port
36 from webkitpy.port.leakdetector import LeakDetector
37 from webkitpy.port import config as port_config
38
39
40 _log = logging.getLogger(__name__)
41
42
43 class IOSSimulatorPort(Port):
44     port_name = "ios-simulator"
45
46     FUTURE_VERSION = 'future'
47
48     VERSION_FALLBACK_ORDER = ['ios-simulator', 'mac']
49
50     ARCHITECTURES = ['x86_64', 'x86']
51
52     relay_name = 'LayoutTestRelay'
53
54     def __init__(self, *args, **kwargs):
55         super(IOSSimulatorPort, self).__init__(*args, **kwargs)
56
57         self._architecture = self.get_option('architecture')
58
59         if not self._architecture:
60             self._architecture = 'x86_64'
61
62         self._leak_detector = LeakDetector(self)
63         if self.get_option("leaks"):
64             # DumpRenderTree slows down noticably if we run more than about 1000 tests in a batch
65             # with MallocStackLogging enabled.
66             self.set_option_default("batch_size", 1000)
67
68     def driver_name(self):
69         if self.get_option('driver_name'):
70             return self.get_option('driver_name')
71         if self.get_option('webkit_test_runner'):
72             return 'WebKitTestRunnerApp.app'
73         return 'DumpRenderTree.app'
74
75     @property
76     def relay_path(self):
77         mac_config = port_config.Config(self._executive, self._filesystem, 'mac')
78         mac_build_directory = mac_config.build_directory(self.get_option('configuration'))
79         return self._filesystem.join(mac_build_directory, self.relay_name)
80
81     def default_timeout_ms(self):
82         if self.get_option('guard_malloc'):
83             return 350 * 1000
84         return super(IOSSimulatorPort, self).default_timeout_ms()
85
86     def supports_per_test_timeout(self):
87         return True
88
89     def _check_relay(self):
90         if not self._filesystem.exists(self.relay_path):
91             _log.error("%s was not found at %s" % (self.relay_name, self.relay_path))
92             return False
93         return True
94
95     def _check_build_relay(self):
96         if self.get_option('build') and not self._build_relay():
97             return False
98         if not self._check_relay():
99             return False
100         return True
101
102     def check_build(self, needs_http):
103         needs_driver = super(IOSSimulatorPort, self).check_build(needs_http)
104         return needs_driver and self._check_build_relay() and self._check_build_image_diff()
105
106     def _path_to_image_diff(self):
107         mac_config = port_config.Config(self._executive, self._filesystem, 'mac')
108         mac_build_directory = mac_config.build_directory(self.get_option('configuration'))
109         return self._filesystem.join(mac_build_directory, 'ImageDiff')
110
111     def _build_relay(self):
112         environment = self.host.copy_current_environment()
113         environment.disable_gcc_smartquotes()
114         env = environment.to_dictionary()
115
116         try:
117             self._run_script("build-layouttestrelay", env=env)
118         except ScriptError, e:
119             _log.error(e.message_with_output(output_limit=None))
120             return False
121         return True
122
123     def _check_image_diff(self):
124         image_diff_path = self._path_to_image_diff()
125         if not self._filesystem.exists(image_diff_path):
126             _log.error("%s was not found at %s" % ('ImageDiff', image_diff_path))
127             return False
128         return True
129
130     def _check_build_image_diff(self):
131         if not self._root_was_set and self.get_option('build') and not self._build_driver():
132             return False
133         if not self._check_image_diff():
134             return False
135         return True
136
137     def _build_image_diff(self):
138         environment = self.host.copy_current_environment()
139         environment.disable_gcc_smartquotes()
140         env = environment.to_dictionary()
141
142         try:
143             self._run_script("build-imagediff", env=env)
144         except ScriptError, e:
145             _log.error(e.message_with_output(output_limit=None))
146             return False
147         return True
148
149     def _build_driver(self):
150         built_tool = super(IOSSimulatorPort, self)._build_driver()
151         built_relay = self._build_relay()
152         built_image_diff = self._build_image_diff()
153         return built_tool and built_relay and built_image_diff
154
155     def _build_driver_flags(self):
156         archs = ['ARCHS=i386'] if self.architecture() == 'x86' else []
157         sdk = ['--sdk', 'iphonesimulator']
158         return archs + sdk
159
160     def should_retry_crashes(self):
161         return True
162
163     def _generate_all_test_configurations(self):
164         configurations = []
165         for build_type in self.ALL_BUILD_TYPES:
166             configurations.append(TestConfiguration(version=self._version, architecture='x86', build_type=build_type))
167         return configurations
168
169     def _driver_class(self):
170         return driver.IOSSimulatorDriver
171
172     def default_baseline_search_path(self):
173         name = self._name.replace('-wk2', '')
174         wk_version = [] if self.get_option('webkit_test_runner') else ['mac-wk1']
175         if name.endswith(self.FUTURE_VERSION):
176             fallback_names = wk_version + [self.port_name]
177         else:
178             fallback_names = self.VERSION_FALLBACK_ORDER[self.VERSION_FALLBACK_ORDER.index(name):-1] + wk_version + [self.port_name]
179         # FIXME: mac-wk2 should appear at the same place as mac-wk1.
180         if self.get_option('webkit_test_runner'):
181             fallback_names = [self._wk2_port_name(), 'wk2'] + fallback_names
182         return map(self._webkit_baseline_path, fallback_names)
183
184     def _port_specific_expectations_files(self):
185         return list(reversed([self._filesystem.join(self._webkit_baseline_path(p), 'TestExpectations') for p in self.baseline_search_path()]))
186
187     def setup_test_run(self):
188         self._executive.run_command(['osascript', '-e', 'tell application "iOS Simulator" to quit'])
189         self._executive.run_command([self.xcrun_find('simctl'), 'shutdown', self.simulator_udid()], error_handler=lambda e: None)
190
191     def clean_up_test_run(self):
192         super(IOSSimulatorPort, self).clean_up_test_run()
193         fifos = [path for path in os.listdir('/tmp') if re.search('org.webkit.(DumpRenderTree|WebKitTestRunner).*_(IN|OUT|ERROR)', path)]
194         for fifo in fifos:
195             try:
196                 os.remove(os.path.join('/tmp', fifo))
197             except OSError:
198                 _log.warning('Unable to remove ' + fifo)
199                 pass
200
201     def setup_environ_for_server(self, server_name=None):
202         env = super(IOSSimulatorPort, self).setup_environ_for_server(server_name)
203         if server_name == self.driver_name():
204             if self.get_option('leaks'):
205                 env['MallocStackLogging'] = '1'
206             if self.get_option('guard_malloc'):
207                 env['DYLD_INSERT_LIBRARIES'] = '/usr/lib/libgmalloc.dylib:' + self._build_path("libWebCoreTestShim.dylib")
208             else:
209                 env['DYLD_INSERT_LIBRARIES'] = self._build_path("libWebCoreTestShim.dylib")
210         env['XML_CATALOG_FILES'] = ''  # work around missing /etc/catalog <rdar://problem/4292995>
211         return env
212
213     def operating_system(self):
214         return 'ios-simulator'
215
216     def check_for_leaks(self, process_name, process_pid):
217         if not self.get_option('leaks'):
218             return
219         # We could use http://code.google.com/p/psutil/ to get the process_name from the pid.
220         self._leak_detector.check_for_leaks(process_name, process_pid)
221
222     def print_leaks_summary(self):
223         if not self.get_option('leaks'):
224             return
225         # We're in the manager process, so the leak detector will not have a valid list of leak files.
226         leaks_files = self._leak_detector.leaks_files_in_directory(self.results_directory())
227         if not leaks_files:
228             return
229         total_bytes_string, unique_leaks = self._leak_detector.count_total_bytes_and_unique_leaks(leaks_files)
230         total_leaks = self._leak_detector.count_total_leaks(leaks_files)
231         _log.info("%s total leaks found for a total of %s!" % (total_leaks, total_bytes_string))
232         _log.info("%s unique leaks found!" % unique_leaks)
233
234     def _path_to_webcore_library(self):
235         return self._build_path('WebCore.framework/Versions/A/WebCore')
236
237     def show_results_html_file(self, results_filename):
238         # We don't use self._run_script() because we don't want to wait for the script
239         # to exit and we want the output to show up on stdout in case there are errors
240         # launching the browser.
241         self._executive.popen([self.path_to_script('run-safari')] + self._arguments_for_configuration() + ['--no-saved-state', '-NSOpen', results_filename],
242             cwd=self.webkit_base(), stdout=file(os.devnull), stderr=file(os.devnull))
243
244     def acquire_http_lock(self):
245         pass
246
247     def release_http_lock(self):
248         pass
249
250     def sample_file_path(self, name, pid):
251         return self._filesystem.join(self.results_directory(), "{0}-{1}-sample.txt".format(name, pid))
252
253     def _get_crash_log(self, name, pid, stdout, stderr, newer_than, time_fn=time.time, sleep_fn=time.sleep, wait_for_log=True):
254         time_fn = time_fn or time.time
255         sleep_fn = sleep_fn or time.sleep
256         crash_log = ''
257         crash_logs = CrashLogs(self.host)
258         now = time_fn()
259
260         crash_prefix = 'CRASH: '
261         stderr_lines = []
262         crash_lines = []
263         for line in stderr.splitlines():
264             crash_lines.append(line) if line.startswith(crash_prefix) else stderr_lines.append(line)
265
266         for crash_line in crash_lines:
267             identifier, pid = crash_line[len(crash_prefix):].split(' ')
268             return self._get_crash_log(identifier, int(pid), stdout, '\n'.join(stderr_lines), newer_than, time_fn, sleep_fn, wait_for_log)
269
270         _log.debug('looking for crash log for %s:%s' % (name, str(pid)))
271         deadline = now + 5 * int(self.get_option('child_processes', 1))
272         while not crash_log and now <= deadline:
273             crash_log = crash_logs.find_newest_log(name, pid, include_errors=True, newer_than=newer_than)
274             if not wait_for_log:
275                 break
276             if not crash_log or not [line for line in crash_log.splitlines() if not line.startswith('ERROR')]:
277                 sleep_fn(0.1)
278                 now = time_fn()
279
280         if not crash_log:
281             return stderr, None
282         return stderr, crash_log
283
284     def simulator_udid(self):
285         device_name = self.get_option('device_type').split('.')[-1].replace('-', ' ') + ' WebKit Tester'
286         stdout = subprocess.check_output(['xcrun', '--sdk', 'iphonesimulator', 'simctl', 'list'])
287         lines = stdout.splitlines()
288         try:
289             devices_index = lines.index('== Devices ==')
290             device_regex = re.compile('(?P<device_name>[^(]+) \((?P<udid>[^)]+)\) \((?P<state>[^)]+)\)')
291             for device_line in itertools.takewhile(lambda line: not line.startswith('=='), lines[devices_index + 1:]):
292                 device = device_regex.match(device_line.lstrip().rstrip())
293                 if not device:
294                     continue
295                 if device.group('device_name') == device_name:
296                     return device.group('udid')
297         except ValueError:
298             pass
299
300     def simulator_path(self, udid):
301         if udid:
302             return os.path.realpath(os.path.expanduser(os.path.join('~/Library/Developer/CoreSimulator/Devices', udid)))
303
304     def look_for_new_crash_logs(self, crashed_processes, start_time):
305         crash_logs = {}
306         for (test_name, process_name, pid) in crashed_processes:
307             # Passing None for output.  This is a second pass after the test finished so
308             # if the output had any logging we would have already collected it.
309             crash_log = self._get_crash_log(process_name, pid, None, None, start_time, wait_for_log=False)[1]
310             if not crash_log:
311                 continue
312             crash_logs[test_name] = crash_log
313         return crash_logs
314
315     def look_for_new_samples(self, unresponsive_processes, start_time):
316         sample_files = {}
317         for (test_name, process_name, pid) in unresponsive_processes:
318             sample_file = self.sample_file_path(process_name, pid)
319             if not self._filesystem.isfile(sample_file):
320                 continue
321             sample_files[test_name] = sample_file
322         return sample_files
323
324     def sample_process(self, name, pid):
325         try:
326             hang_report = self.sample_file_path(name, pid)
327             self._executive.run_command([
328                 "/usr/bin/sample",
329                 pid,
330                 10,
331                 10,
332                 "-file",
333                 hang_report,
334             ])
335         except ScriptError as e:
336             _log.warning('Unable to sample process:' + str(e))
337
338     def _path_to_helper(self):
339         binary_name = 'LayoutTestHelper'
340         return self._build_path(binary_name)
341
342     def reset_preferences(self):
343         simulator_path = self.simulator_path(self.simulator_udid())
344         if not simulator_path:
345             return
346         data_path = os.path.join(simulator_path, 'data')
347         if os.path.isdir(data_path):
348             shutil.rmtree(data_path)
349
350     def make_command(self):
351         return self.xcrun_find('make', '/usr/bin/make')
352
353     def nm_command(self):
354         return self.xcrun_find('nm')
355
356     def xcrun_find(self, command, fallback=None):
357         fallback = fallback or command
358         try:
359             return self._executive.run_command(['xcrun', '--sdk', 'iphonesimulator', '-find', command]).rstrip()
360         except ScriptError:
361             _log.warn("xcrun failed; falling back to '%s'." % fallback)
362             return fallback
363
364     def logging_patterns_to_strip(self):
365         return []