f477410234dc06dcfe14198340188f807a305792
[WebKit-https.git] / Tools / Scripts / webkitpy / port / ios.py
1 # Copyright (C) 2014, 2015 Apple Inc. All rights reserved.
2 #
3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions
5 # are met:
6 # 1.  Redistributions of source code must retain the above copyright
7 #     notice, this list of conditions and the following disclaimer.
8 # 2.  Redistributions in binary form must reproduce the above copyright
9 #     notice, this list of conditions and the following disclaimer in the
10 #     documentation and/or other materials provided with the distribution.
11 #
12 # THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND
13 # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
14 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
15 # DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR
16 # ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
17 # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
18 # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
19 # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
20 # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
21 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
22
23 import itertools
24 import logging
25 import os
26 import re
27 import shutil
28 import subprocess
29 import time
30
31 from webkitpy.common.memoized import memoized
32 from webkitpy.common.system.crashlogs import CrashLogs
33 from webkitpy.common.system.executive import ScriptError
34 from webkitpy.layout_tests.models.test_configuration import TestConfiguration
35 from webkitpy.port import config as port_config
36 from webkitpy.port import driver, image_diff
37 from webkitpy.port.apple import ApplePort
38 from webkitpy.port.base import Port
39 from webkitpy.port.leakdetector import LeakDetector
40 from webkitpy.xcode.simulator import Simulator, Runtime, DeviceType
41
42
43 _log = logging.getLogger(__name__)
44
45
46 class IOSPort(ApplePort):
47     port_name = "ios"
48
49     ARCHITECTURES = ['armv7', 'armv7s', 'arm64']
50     DEFAULT_ARCHITECTURE = 'armv7'
51     VERSION_FALLBACK_ORDER = ['ios-7', 'ios-8', 'ios-9']
52
53     @classmethod
54     def determine_full_port_name(cls, host, options, port_name):
55         if port_name == cls.port_name:
56             iphoneos_sdk_version = host.platform.xcode_sdk_version('iphoneos')
57             if not iphoneos_sdk_version:
58                 raise Exception("Please install the iOS SDK.")
59             major_version_number = iphoneos_sdk_version.split('.')[0]
60             port_name = port_name + '-' + major_version_number
61         return port_name
62
63     def __init__(self, *args, **kwargs):
64         super(IOSPort, self).__init__(*args, **kwargs)
65
66         self._testing_device = None
67
68     # Despite their names, these flags do not actually get passed all the way down to webkit-build.
69     def _build_driver_flags(self):
70         return ['--sdk', 'iphoneos'] + (['ARCHS=%s' % self.architecture()] if self.architecture() else [])
71
72     def operating_system(self):
73         return 'ios'
74
75
76 class IOSSimulatorPort(Port):
77     port_name = "ios-simulator"
78
79     FUTURE_VERSION = 'future'
80
81     ARCHITECTURES = ['x86_64', 'x86']
82
83     DEFAULT_ARCHITECTURE = 'x86_64'
84
85     relay_name = 'LayoutTestRelay'
86
87     def __init__(self, *args, **kwargs):
88         super(IOSSimulatorPort, self).__init__(*args, **kwargs)
89
90         self._leak_detector = LeakDetector(self)
91         if self.get_option("leaks"):
92             # DumpRenderTree slows down noticably if we run more than about 1000 tests in a batch
93             # with MallocStackLogging enabled.
94             self.set_option_default("batch_size", 1000)
95
96         self._testing_device = None
97
98     def driver_name(self):
99         if self.get_option('driver_name'):
100             return self.get_option('driver_name')
101         if self.get_option('webkit_test_runner'):
102             return 'WebKitTestRunnerApp.app'
103         return 'DumpRenderTree.app'
104
105     @property
106     @memoized
107     def simulator_runtime(self):
108         runtime_identifier = self.get_option('runtime')
109         if runtime_identifier:
110             runtime = Runtime.from_identifier(runtime_identifier)
111         else:
112             runtime = Runtime.from_version_string(self.host.platform.xcode_sdk_version('iphonesimulator'))
113         return runtime
114
115     @property
116     @memoized
117     def simulator_device_type(self):
118         device_type_identifier = self.get_option('device_type')
119         if device_type_identifier:
120             device_type = DeviceType.from_identifier(device_type_identifier)
121         else:
122             if self.architecture() == 'x86_64':
123                 device_type = DeviceType.from_name('iPhone 5s')
124             else:
125                 device_type = DeviceType.from_name('iPhone 5')
126         return device_type
127
128     @property
129     @memoized
130     def relay_path(self):
131         mac_config = port_config.Config(self._executive, self._filesystem, 'mac')
132         return self._filesystem.join(mac_config.build_directory(self.get_option('configuration')), self.relay_name)
133
134     def default_timeout_ms(self):
135         if self.get_option('guard_malloc'):
136             return 350 * 1000
137         return super(IOSSimulatorPort, self).default_timeout_ms()
138
139     def supports_per_test_timeout(self):
140         return True
141
142     def _check_relay(self):
143         if not self._filesystem.exists(self.relay_path):
144             _log.error("%s was not found at %s" % (self.relay_name, self.relay_path))
145             return False
146         return True
147
148     def _check_build_relay(self):
149         if self.get_option('build') and not self._build_relay():
150             return False
151         if not self._check_relay():
152             return False
153         return True
154
155     def check_build(self, needs_http):
156         needs_driver = super(IOSSimulatorPort, self).check_build(needs_http)
157         return needs_driver and self._check_build_relay()
158
159     def _build_relay(self):
160         environment = self.host.copy_current_environment()
161         environment.disable_gcc_smartquotes()
162         env = environment.to_dictionary()
163
164         try:
165             self._run_script("build-layouttestrelay", env=env)
166         except ScriptError, e:
167             _log.error(e.message_with_output(output_limit=None))
168             return False
169         return True
170
171     def _build_driver(self):
172         built_tool = super(IOSSimulatorPort, self)._build_driver()
173         built_relay = self._build_relay()
174         return built_tool and built_relay
175
176     def _build_driver_flags(self):
177         archs = ['ARCHS=i386'] if self.architecture() == 'x86' else []
178         sdk = ['--sdk', 'iphonesimulator']
179         return archs + sdk
180
181     def should_retry_crashes(self):
182         return True
183
184     def _generate_all_test_configurations(self):
185         configurations = []
186         for build_type in self.ALL_BUILD_TYPES:
187             for architecture in self.ARCHITECTURES:
188                 configurations.append(TestConfiguration(version=self._version, architecture=architecture, build_type=build_type))
189         return configurations
190
191     def _driver_class(self):
192         return driver.IOSSimulatorDriver
193
194     def default_baseline_search_path(self):
195         if self.get_option('webkit_test_runner'):
196             fallback_names = [self._wk2_port_name()] + [self.port_name] + ['wk2']
197         else:
198             fallback_names = [self.port_name + '-wk1'] + [self.port_name]
199
200         return map(self._webkit_baseline_path, fallback_names)
201
202     def _port_specific_expectations_files(self):
203         return list(reversed([self._filesystem.join(self._webkit_baseline_path(p), 'TestExpectations') for p in self.baseline_search_path()]))
204
205     def setup_test_run(self):
206         device_udid = self.testing_device.udid
207         self._executive.run_command([
208             'open', '-a', os.path.join(self.developer_dir, 'Applications', 'iOS Simulator.app'),
209             '--args', '-CurrentDeviceUDID', device_udid])
210         Simulator.wait_until_device_is_in_state(device_udid, Simulator.DeviceState.BOOTED)
211
212         # FIXME: Pause here until SpringBoard finishes launching to workaround <rdar://problem/20000383>.
213         boot_delay = 30
214         _log.debug('Waiting {seconds} seconds for iOS Simulator to finish booting ...'.format(seconds=boot_delay))
215         time.sleep(boot_delay)
216
217     def clean_up_test_run(self):
218         super(IOSSimulatorPort, self).clean_up_test_run()
219         fifos = [path for path in os.listdir('/tmp') if re.search('org.webkit.(DumpRenderTree|WebKitTestRunner).*_(IN|OUT|ERROR)', path)]
220         for fifo in fifos:
221             try:
222                 os.remove(os.path.join('/tmp', fifo))
223             except OSError:
224                 _log.warning('Unable to remove ' + fifo)
225                 pass
226
227     def setup_environ_for_server(self, server_name=None):
228         env = super(IOSSimulatorPort, self).setup_environ_for_server(server_name)
229         if server_name == self.driver_name():
230             if self.get_option('leaks'):
231                 env['MallocStackLogging'] = '1'
232             if self.get_option('guard_malloc'):
233                 env['DYLD_INSERT_LIBRARIES'] = '/usr/lib/libgmalloc.dylib:' + self._build_path("libWebCoreTestShim.dylib")
234             else:
235                 env['DYLD_INSERT_LIBRARIES'] = self._build_path("libWebCoreTestShim.dylib")
236         env['XML_CATALOG_FILES'] = ''  # work around missing /etc/catalog <rdar://problem/4292995>
237         return env
238
239     def operating_system(self):
240         return 'ios-simulator'
241
242     def check_sys_deps(self, needs_http):
243         if not self.simulator_runtime.available:
244             _log.error('The iOS Simulator runtime with identifier "{0}" cannot be used because it is unavailable.'.format(self.simulator_runtime.identifier))
245             return False
246         testing_device = self.testing_device  # May create a new simulator device
247
248         # testing_device will fail to boot if it is already booted. We assume that if testing_device
249         # is booted that it was booted by the iOS Simulator app (as opposed to simctl). So, quit the
250         # iOS Simulator app to shutdown testing_device.
251         self._executive.run_command(['osascript', '-e', 'tell application "iOS Simulator" to quit'])
252         Simulator.wait_until_device_is_in_state(testing_device.udid, Simulator.DeviceState.SHUTDOWN)
253
254         if not Simulator.check_simulator_device_and_erase_if_needed(self.host, testing_device.udid):
255             _log.error('Unable to boot the simulator device with UDID {0}.'.format(testing_device.udid))
256             return False
257         return super(IOSSimulatorPort, self).check_sys_deps(needs_http)
258
259     def check_for_leaks(self, process_name, process_pid):
260         if not self.get_option('leaks'):
261             return
262         # We could use http://code.google.com/p/psutil/ to get the process_name from the pid.
263         self._leak_detector.check_for_leaks(process_name, process_pid)
264
265     def print_leaks_summary(self):
266         if not self.get_option('leaks'):
267             return
268         # We're in the manager process, so the leak detector will not have a valid list of leak files.
269         leaks_files = self._leak_detector.leaks_files_in_directory(self.results_directory())
270         if not leaks_files:
271             return
272         total_bytes_string, unique_leaks = self._leak_detector.count_total_bytes_and_unique_leaks(leaks_files)
273         total_leaks = self._leak_detector.count_total_leaks(leaks_files)
274         _log.info("%s total leaks found for a total of %s." % (total_leaks, total_bytes_string))
275         _log.info("%s unique leaks found." % unique_leaks)
276
277     def _path_to_webcore_library(self):
278         return self._build_path('WebCore.framework/Versions/A/WebCore')
279
280     def show_results_html_file(self, results_filename):
281         # We don't use self._run_script() because we don't want to wait for the script
282         # to exit and we want the output to show up on stdout in case there are errors
283         # launching the browser.
284         self._executive.popen([self.path_to_script('run-safari')] + self._arguments_for_configuration() + ['--no-saved-state', '-NSOpen', results_filename],
285             cwd=self.webkit_base(), stdout=file(os.devnull), stderr=file(os.devnull))
286
287     def sample_file_path(self, name, pid):
288         return self._filesystem.join(self.results_directory(), "{0}-{1}-sample.txt".format(name, pid))
289
290     SUBPROCESS_CRASH_REGEX = re.compile('#CRASHED - (?P<subprocess_name>\S+) \(pid (?P<subprocess_pid>\d+)\)')
291
292     def _get_crash_log(self, name, pid, stdout, stderr, newer_than, time_fn=time.time, sleep_fn=time.sleep, wait_for_log=True):
293         time_fn = time_fn or time.time
294         sleep_fn = sleep_fn or time.sleep
295
296         # FIXME: We should collect the actual crash log for DumpRenderTree.app because it includes more
297         # information (e.g. exception codes) than is available in the stack trace written to standard error.
298         stderr_lines = []
299         crashed_subprocess_name_and_pid = None  # e.g. ('DumpRenderTree.app', 1234)
300         for line in (stderr or '').splitlines():
301             if not crashed_subprocess_name_and_pid:
302                 match = self.SUBPROCESS_CRASH_REGEX.match(line)
303                 if match:
304                     crashed_subprocess_name_and_pid = (match.group('subprocess_name'), int(match.group('subprocess_pid')))
305                     continue
306             stderr_lines.append(line)
307
308         if crashed_subprocess_name_and_pid:
309             return self._get_crash_log(crashed_subprocess_name_and_pid[0], crashed_subprocess_name_and_pid[1], stdout,
310                 '\n'.join(stderr_lines), newer_than, time_fn, sleep_fn, wait_for_log)
311
312         # LayoutTestRelay crashed
313         _log.debug('looking for crash log for %s:%s' % (name, str(pid)))
314         crash_log = ''
315         crash_logs = CrashLogs(self.host)
316         now = time_fn()
317         deadline = now + 5 * int(self.get_option('child_processes', 1))
318         while not crash_log and now <= deadline:
319             crash_log = crash_logs.find_newest_log(name, pid, include_errors=True, newer_than=newer_than)
320             if not wait_for_log:
321                 break
322             if not crash_log or not [line for line in crash_log.splitlines() if not line.startswith('ERROR')]:
323                 sleep_fn(0.1)
324                 now = time_fn()
325
326         if not crash_log:
327             return stderr, None
328         return stderr, crash_log
329
330     @property
331     def testing_device(self):
332         if self._testing_device is not None:
333             return self._testing_device
334         self._testing_device = Simulator().lookup_or_create_device(self.simulator_device_type.name + ' WebKit Tester', self.simulator_device_type, self.simulator_runtime)
335         return self.testing_device
336
337     def look_for_new_crash_logs(self, crashed_processes, start_time):
338         crash_logs = {}
339         for (test_name, process_name, pid) in crashed_processes:
340             # Passing None for output.  This is a second pass after the test finished so
341             # if the output had any logging we would have already collected it.
342             crash_log = self._get_crash_log(process_name, pid, None, None, start_time, wait_for_log=False)[1]
343             if not crash_log:
344                 continue
345             crash_logs[test_name] = crash_log
346         return crash_logs
347
348     def look_for_new_samples(self, unresponsive_processes, start_time):
349         sample_files = {}
350         for (test_name, process_name, pid) in unresponsive_processes:
351             sample_file = self.sample_file_path(process_name, pid)
352             if not self._filesystem.isfile(sample_file):
353                 continue
354             sample_files[test_name] = sample_file
355         return sample_files
356
357     def sample_process(self, name, pid):
358         try:
359             hang_report = self.sample_file_path(name, pid)
360             self._executive.run_command([
361                 "/usr/bin/sample",
362                 pid,
363                 10,
364                 10,
365                 "-file",
366                 hang_report,
367             ])
368         except ScriptError as e:
369             _log.warning('Unable to sample process:' + str(e))
370
371     def _path_to_helper(self):
372         binary_name = 'LayoutTestHelper'
373         return self._build_path(binary_name)
374
375     def diff_image(self, expected_contents, actual_contents, tolerance=None):
376         if not actual_contents and not expected_contents:
377             return (None, 0, None)
378         if not actual_contents or not expected_contents:
379             return (True, 0, None)
380         if not self._image_differ:
381             self._image_differ = image_diff.IOSSimulatorImageDiffer(self)
382         self.set_option_default('tolerance', 0.1)
383         if tolerance is None:
384             tolerance = self.get_option('tolerance')
385         return self._image_differ.diff_image(expected_contents, actual_contents, tolerance)
386
387     def reset_preferences(self):
388         simulator_path = self.testing_device.path
389         data_path = os.path.join(simulator_path, 'data')
390         if os.path.isdir(data_path):
391             shutil.rmtree(data_path)
392
393     def make_command(self):
394         return self.xcrun_find('make', '/usr/bin/make')
395
396     def nm_command(self):
397         return self.xcrun_find('nm')
398
399     def xcrun_find(self, command, fallback=None):
400         fallback = fallback or command
401         try:
402             return self._executive.run_command(['xcrun', '--sdk', 'iphonesimulator', '-find', command]).rstrip()
403         except ScriptError:
404             _log.warn("xcrun failed; falling back to '%s'." % fallback)
405             return fallback
406
407     @property
408     def developer_dir(self):
409         return self._executive.run_command(['xcode-select', '--print-path']).rstrip()
410
411     def logging_patterns_to_strip(self):
412         return []
413
414     def stderr_patterns_to_strip(self):
415         return []