c704892d942eabb5ff207aed4e358b11942f5273
[WebKit-https.git] / Tools / Scripts / webkitpy / port / ios.py
1 # Copyright (C) 2014, 2015 Apple Inc. All rights reserved.
2 #
3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions
5 # are met:
6 # 1.  Redistributions of source code must retain the above copyright
7 #     notice, this list of conditions and the following disclaimer.
8 # 2.  Redistributions in binary form must reproduce the above copyright
9 #     notice, this list of conditions and the following disclaimer in the
10 #     documentation and/or other materials provided with the distribution.
11 #
12 # THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND
13 # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
14 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
15 # DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR
16 # ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
17 # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
18 # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
19 # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
20 # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
21 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
22
23 import itertools
24 import logging
25 import os
26 import re
27 import shutil
28 import subprocess
29 import time
30
31 from webkitpy.common.memoized import memoized
32 from webkitpy.common.system.crashlogs import CrashLogs
33 from webkitpy.common.system.executive import ScriptError
34 from webkitpy.layout_tests.models.test_configuration import TestConfiguration
35 from webkitpy.port import config as port_config
36 from webkitpy.port import driver, image_diff
37 from webkitpy.port.apple import ApplePort
38 from webkitpy.port.base import Port
39 from webkitpy.port.leakdetector import LeakDetector
40 from webkitpy.xcode.simulator import Simulator, Runtime, DeviceType
41
42
43 _log = logging.getLogger(__name__)
44
45
46 class IOSPort(ApplePort):
47     port_name = "ios"
48
49     ARCHITECTURES = ['armv7', 'armv7s', 'arm64']
50     DEFAULT_ARCHITECTURE = 'armv7'
51     VERSION_FALLBACK_ORDER = ['ios-7', 'ios-8', 'ios-9']
52
53     @classmethod
54     def determine_full_port_name(cls, host, options, port_name):
55         if port_name == cls.port_name:
56             iphoneos_sdk_version = host.platform.xcode_sdk_version('iphoneos')
57             if not iphoneos_sdk_version:
58                 raise Exception("Please install the iOS SDK.")
59             major_version_number = iphoneos_sdk_version.split('.')[0]
60             port_name = port_name + '-' + major_version_number
61         return port_name
62
63     # Despite their names, these flags do not actually get passed all the way down to webkit-build.
64     def _build_driver_flags(self):
65         return ['--sdk', 'iphoneos'] + (['ARCHS=%s' % self.architecture()] if self.architecture() else [])
66
67     def operating_system(self):
68         return 'ios'
69
70
71 class IOSSimulatorPort(Port):
72     port_name = "ios-simulator"
73
74     FUTURE_VERSION = 'future'
75
76     ARCHITECTURES = ['x86_64', 'x86']
77
78     DEFAULT_ARCHITECTURE = 'x86_64'
79
80     SIMULATOR_BUNDLE_ID = 'com.apple.iphonesimulator'
81
82     relay_name = 'LayoutTestRelay'
83
84     def __init__(self, *args, **kwargs):
85         super(IOSSimulatorPort, self).__init__(*args, **kwargs)
86
87         self._leak_detector = LeakDetector(self)
88         if self.get_option("leaks"):
89             # DumpRenderTree slows down noticably if we run more than about 1000 tests in a batch
90             # with MallocStackLogging enabled.
91             self.set_option_default("batch_size", 1000)
92
93     def driver_name(self):
94         if self.get_option('driver_name'):
95             return self.get_option('driver_name')
96         if self.get_option('webkit_test_runner'):
97             return 'WebKitTestRunnerApp.app'
98         return 'DumpRenderTree.app'
99
100     @property
101     @memoized
102     def simulator_runtime(self):
103         runtime_identifier = self.get_option('runtime')
104         if runtime_identifier:
105             runtime = Runtime.from_identifier(runtime_identifier)
106         else:
107             runtime = Runtime.from_version_string(self.host.platform.xcode_sdk_version('iphonesimulator'))
108         return runtime
109
110     @property
111     @memoized
112     def simulator_device_type(self):
113         device_type_identifier = self.get_option('device_type')
114         if device_type_identifier:
115             device_type = DeviceType.from_identifier(device_type_identifier)
116         else:
117             if self.architecture() == 'x86_64':
118                 device_type = DeviceType.from_name('iPhone 5s')
119             else:
120                 device_type = DeviceType.from_name('iPhone 5')
121         return device_type
122
123     @property
124     @memoized
125     def relay_path(self):
126         mac_config = port_config.Config(self._executive, self._filesystem, 'mac')
127         return self._filesystem.join(mac_config.build_directory(self.get_option('configuration')), self.relay_name)
128
129     def default_timeout_ms(self):
130         if self.get_option('guard_malloc'):
131             return 350 * 1000
132         return super(IOSSimulatorPort, self).default_timeout_ms()
133
134     def supports_per_test_timeout(self):
135         return True
136
137     def _check_relay(self):
138         if not self._filesystem.exists(self.relay_path):
139             _log.error("%s was not found at %s" % (self.relay_name, self.relay_path))
140             return False
141         return True
142
143     def _check_build_relay(self):
144         if self.get_option('build') and not self._build_relay():
145             return False
146         if not self._check_relay():
147             return False
148         return True
149
150     def check_build(self, needs_http):
151         needs_driver = super(IOSSimulatorPort, self).check_build(needs_http)
152         return needs_driver and self._check_build_relay()
153
154     def _build_relay(self):
155         environment = self.host.copy_current_environment()
156         environment.disable_gcc_smartquotes()
157         env = environment.to_dictionary()
158
159         try:
160             # FIXME: We should be passing _arguments_for_configuration(), which respects build configuration and port,
161             # instead of hardcoding --ios-simulator.
162             self._run_script("build-layouttestrelay", args=["--ios-simulator"], env=env)
163         except ScriptError, e:
164             _log.error(e.message_with_output(output_limit=None))
165             return False
166         return True
167
168     def _build_driver(self):
169         built_tool = super(IOSSimulatorPort, self)._build_driver()
170         built_relay = self._build_relay()
171         return built_tool and built_relay
172
173     def _build_driver_flags(self):
174         archs = ['ARCHS=i386'] if self.architecture() == 'x86' else []
175         sdk = ['--sdk', 'iphonesimulator']
176         return archs + sdk
177
178     def should_retry_crashes(self):
179         return True
180
181     def _generate_all_test_configurations(self):
182         configurations = []
183         for build_type in self.ALL_BUILD_TYPES:
184             for architecture in self.ARCHITECTURES:
185                 configurations.append(TestConfiguration(version=self._version, architecture=architecture, build_type=build_type))
186         return configurations
187
188     def _driver_class(self):
189         return driver.IOSSimulatorDriver
190
191     def default_baseline_search_path(self):
192         if self.get_option('webkit_test_runner'):
193             fallback_names = [self._wk2_port_name()] + [self.port_name] + ['wk2']
194         else:
195             fallback_names = [self.port_name + '-wk1'] + [self.port_name]
196
197         return map(self._webkit_baseline_path, fallback_names)
198
199     def _port_specific_expectations_files(self):
200         return list(reversed([self._filesystem.join(self._webkit_baseline_path(p), 'TestExpectations') for p in self.baseline_search_path()]))
201
202     def setup_test_run(self):
203         device_udid = self.testing_device.udid
204         # FIXME: <rdar://problem/20916140> Switch to using CoreSimulator.framework for launching and quitting iOS Simulator
205         self._executive.run_command([
206             'open', '-b', self.SIMULATOR_BUNDLE_ID,
207             '--args', '-CurrentDeviceUDID', device_udid])
208         Simulator.wait_until_device_is_in_state(device_udid, Simulator.DeviceState.BOOTED)
209
210         # FIXME: Pause here until SpringBoard finishes launching to workaround <rdar://problem/20000383>.
211         boot_delay = 30
212         _log.debug('Waiting {seconds} seconds for iOS Simulator to finish booting ...'.format(seconds=boot_delay))
213         time.sleep(boot_delay)
214
215     def _quit_ios_simulator(self):
216         # FIXME: <rdar://problem/20916140> Switch to using CoreSimulator.framework for launching and quitting iOS Simulator
217         self._executive.run_command(['osascript', '-e', 'tell application id "{0}" to quit'.format(self.SIMULATOR_BUNDLE_ID)])
218
219     def clean_up_test_run(self):
220         super(IOSSimulatorPort, self).clean_up_test_run()
221         self._quit_ios_simulator()
222         fifos = [path for path in os.listdir('/tmp') if re.search('org.webkit.(DumpRenderTree|WebKitTestRunner).*_(IN|OUT|ERROR)', path)]
223         for fifo in fifos:
224             try:
225                 os.remove(os.path.join('/tmp', fifo))
226             except OSError:
227                 _log.warning('Unable to remove ' + fifo)
228                 pass
229
230     def setup_environ_for_server(self, server_name=None):
231         env = super(IOSSimulatorPort, self).setup_environ_for_server(server_name)
232         if server_name == self.driver_name():
233             if self.get_option('leaks'):
234                 env['MallocStackLogging'] = '1'
235             if self.get_option('guard_malloc'):
236                 self._append_value_colon_separated(env, 'DYLD_INSERT_LIBRARIES', '/usr/lib/libgmalloc.dylib')
237             self._append_value_colon_separated(env, 'DYLD_INSERT_LIBRARIES', self._build_path("libWebCoreTestShim.dylib"))
238         env['XML_CATALOG_FILES'] = ''  # work around missing /etc/catalog <rdar://problem/4292995>
239         return env
240
241     def operating_system(self):
242         return 'ios-simulator'
243
244     def check_sys_deps(self, needs_http):
245         if not self.simulator_runtime.available:
246             _log.error('The iOS Simulator runtime with identifier "{0}" cannot be used because it is unavailable.'.format(self.simulator_runtime.identifier))
247             return False
248         testing_device = self.testing_device  # May create a new simulator device
249
250         if not Simulator.check_simulator_device_and_erase_if_needed(self.host, testing_device.udid):
251             _log.error('Unable to boot the simulator device with UDID {0}.'.format(testing_device.udid))
252             return False
253         return super(IOSSimulatorPort, self).check_sys_deps(needs_http)
254
255     def check_for_leaks(self, process_name, process_pid):
256         if not self.get_option('leaks'):
257             return
258         # We could use http://code.google.com/p/psutil/ to get the process_name from the pid.
259         self._leak_detector.check_for_leaks(process_name, process_pid)
260
261     def print_leaks_summary(self):
262         if not self.get_option('leaks'):
263             return
264         # We're in the manager process, so the leak detector will not have a valid list of leak files.
265         leaks_files = self._leak_detector.leaks_files_in_directory(self.results_directory())
266         if not leaks_files:
267             return
268         total_bytes_string, unique_leaks = self._leak_detector.count_total_bytes_and_unique_leaks(leaks_files)
269         total_leaks = self._leak_detector.count_total_leaks(leaks_files)
270         _log.info("%s total leaks found for a total of %s." % (total_leaks, total_bytes_string))
271         _log.info("%s unique leaks found." % unique_leaks)
272
273     def _path_to_webcore_library(self):
274         return self._build_path('WebCore.framework/Versions/A/WebCore')
275
276     def show_results_html_file(self, results_filename):
277         # We don't use self._run_script() because we don't want to wait for the script
278         # to exit and we want the output to show up on stdout in case there are errors
279         # launching the browser.
280         self._executive.popen([self.path_to_script('run-safari')] + self._arguments_for_configuration() + ['--no-saved-state', '-NSOpen', results_filename],
281             cwd=self.webkit_base(), stdout=file(os.devnull), stderr=file(os.devnull))
282
283     def sample_file_path(self, name, pid):
284         return self._filesystem.join(self.results_directory(), "{0}-{1}-sample.txt".format(name, pid))
285
286     SUBPROCESS_CRASH_REGEX = re.compile('#CRASHED - (?P<subprocess_name>\S+) \(pid (?P<subprocess_pid>\d+)\)')
287
288     def _get_crash_log(self, name, pid, stdout, stderr, newer_than, time_fn=time.time, sleep_fn=time.sleep, wait_for_log=True):
289         time_fn = time_fn or time.time
290         sleep_fn = sleep_fn or time.sleep
291
292         # FIXME: We should collect the actual crash log for DumpRenderTree.app because it includes more
293         # information (e.g. exception codes) than is available in the stack trace written to standard error.
294         stderr_lines = []
295         crashed_subprocess_name_and_pid = None  # e.g. ('DumpRenderTree.app', 1234)
296         for line in (stderr or '').splitlines():
297             if not crashed_subprocess_name_and_pid:
298                 match = self.SUBPROCESS_CRASH_REGEX.match(line)
299                 if match:
300                     crashed_subprocess_name_and_pid = (match.group('subprocess_name'), int(match.group('subprocess_pid')))
301                     continue
302             stderr_lines.append(line)
303
304         if crashed_subprocess_name_and_pid:
305             return self._get_crash_log(crashed_subprocess_name_and_pid[0], crashed_subprocess_name_and_pid[1], stdout,
306                 '\n'.join(stderr_lines), newer_than, time_fn, sleep_fn, wait_for_log)
307
308         # LayoutTestRelay crashed
309         _log.debug('looking for crash log for %s:%s' % (name, str(pid)))
310         crash_log = ''
311         crash_logs = CrashLogs(self.host)
312         now = time_fn()
313         deadline = now + 5 * int(self.get_option('child_processes', 1))
314         while not crash_log and now <= deadline:
315             crash_log = crash_logs.find_newest_log(name, pid, include_errors=True, newer_than=newer_than)
316             if not wait_for_log:
317                 break
318             if not crash_log or not [line for line in crash_log.splitlines() if not line.startswith('ERROR')]:
319                 sleep_fn(0.1)
320                 now = time_fn()
321
322         if not crash_log:
323             return stderr, None
324         return stderr, crash_log
325
326     @property
327     @memoized
328     def testing_device(self):
329         return Simulator().lookup_or_create_device(self.simulator_device_type.name + ' WebKit Tester', self.simulator_device_type, self.simulator_runtime)
330
331     def _merge_crash_logs(self, logs, new_logs, crashed_processes):
332         for test, crash_log in new_logs.iteritems():
333             try:
334                 process_name = test.split("-")[0]
335                 pid = int(test.split("-")[1])
336             except IndexError:
337                 continue
338             if not any(entry[1] == process_name and entry[2] == pid for entry in crashed_processes):
339                 # if this is a new crash, then append the logs
340                 logs[test] = crash_log
341         return logs
342
343     def _look_for_all_crash_logs_in_log_dir(self, newer_than):
344         crash_log = CrashLogs(self.host)
345         return crash_log.find_all_logs(include_errors=True, newer_than=newer_than)
346
347     def look_for_new_crash_logs(self, crashed_processes, start_time):
348         crash_logs = {}
349         for (test_name, process_name, pid) in crashed_processes:
350             # Passing None for output.  This is a second pass after the test finished so
351             # if the output had any logging we would have already collected it.
352             crash_log = self._get_crash_log(process_name, pid, None, None, start_time, wait_for_log=False)[1]
353             if not crash_log:
354                 continue
355             crash_logs[test_name] = crash_log
356         all_crash_log = self._look_for_all_crash_logs_in_log_dir(start_time)
357         return self._merge_crash_logs(crash_logs, all_crash_log, crashed_processes)
358
359     def look_for_new_samples(self, unresponsive_processes, start_time):
360         sample_files = {}
361         for (test_name, process_name, pid) in unresponsive_processes:
362             sample_file = self.sample_file_path(process_name, pid)
363             if not self._filesystem.isfile(sample_file):
364                 continue
365             sample_files[test_name] = sample_file
366         return sample_files
367
368     def sample_process(self, name, pid):
369         try:
370             hang_report = self.sample_file_path(name, pid)
371             self._executive.run_command([
372                 "/usr/bin/sample",
373                 pid,
374                 10,
375                 10,
376                 "-file",
377                 hang_report,
378             ])
379         except ScriptError as e:
380             _log.warning('Unable to sample process:' + str(e))
381
382     def _path_to_helper(self):
383         binary_name = 'LayoutTestHelper'
384         return self._build_path(binary_name)
385
386     def diff_image(self, expected_contents, actual_contents, tolerance=None):
387         if not actual_contents and not expected_contents:
388             return (None, 0, None)
389         if not actual_contents or not expected_contents:
390             return (True, 0, None)
391         if not self._image_differ:
392             self._image_differ = image_diff.IOSSimulatorImageDiffer(self)
393         self.set_option_default('tolerance', 0.1)
394         if tolerance is None:
395             tolerance = self.get_option('tolerance')
396         return self._image_differ.diff_image(expected_contents, actual_contents, tolerance)
397
398     def reset_preferences(self):
399         # We assume that if testing_device is booted that it was booted by the iOS Simulator app
400         # (as opposed to simctl). So, quit the iOS Simulator app to shutdown testing_device.
401         self._quit_ios_simulator()
402         Simulator.wait_until_device_is_in_state(self.testing_device.udid, Simulator.DeviceState.SHUTDOWN)
403
404         data_path = os.path.join(self.testing_device.path, 'data')
405         if os.path.isdir(data_path):
406             shutil.rmtree(data_path)
407
408     def make_command(self):
409         return self.xcrun_find('make', '/usr/bin/make')
410
411     def nm_command(self):
412         return self.xcrun_find('nm')
413
414     def xcrun_find(self, command, fallback=None):
415         fallback = fallback or command
416         try:
417             return self._executive.run_command(['xcrun', '--sdk', 'iphonesimulator', '-find', command]).rstrip()
418         except ScriptError:
419             _log.warn("xcrun failed; falling back to '%s'." % fallback)
420             return fallback
421
422     @property
423     def developer_dir(self):
424         return self._executive.run_command(['xcode-select', '--print-path']).rstrip()
425
426     def logging_patterns_to_strip(self):
427         return []
428
429     def stderr_patterns_to_strip(self):
430         return []