[iOS] simctl can hang if run quickly after shutting down CoreSimulator services
[WebKit-https.git] / Tools / Scripts / webkitpy / port / ios.py
1 # Copyright (C) 2014 Apple Inc. All rights reserved.
2 #
3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions
5 # are met:
6 # 1.  Redistributions of source code must retain the above copyright
7 #     notice, this list of conditions and the following disclaimer.
8 # 2.  Redistributions in binary form must reproduce the above copyright
9 #     notice, this list of conditions and the following disclaimer in the
10 #     documentation and/or other materials provided with the distribution.
11 #
12 # THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND
13 # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
14 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
15 # DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR
16 # ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
17 # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
18 # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
19 # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
20 # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
21 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
22
23 import logging
24 import os
25 import shutil
26 import time
27 import re
28 import itertools
29 import subprocess
30
31 from webkitpy.layout_tests.models.test_configuration import TestConfiguration
32 from webkitpy.common.system.crashlogs import CrashLogs
33 from webkitpy.common.system.executive import ScriptError
34 from webkitpy.port import driver
35 from webkitpy.port.base import Port
36 from webkitpy.port.leakdetector import LeakDetector
37 from webkitpy.port import config as port_config
38
39
40 _log = logging.getLogger(__name__)
41
42
43 class IOSSimulatorPort(Port):
44     port_name = "ios-simulator"
45
46     FUTURE_VERSION = 'future'
47
48     VERSION_FALLBACK_ORDER = ['ios-simulator', 'mac']
49
50     ARCHITECTURES = ['x86_64', 'x86']
51
52     relay_name = 'LayoutTestRelay'
53
54     def __init__(self, *args, **kwargs):
55         super(IOSSimulatorPort, self).__init__(*args, **kwargs)
56
57         self._architecture = self.get_option('architecture')
58
59         if not self._architecture:
60             self._architecture = 'x86_64'
61
62         self._leak_detector = LeakDetector(self)
63         if self.get_option("leaks"):
64             # DumpRenderTree slows down noticably if we run more than about 1000 tests in a batch
65             # with MallocStackLogging enabled.
66             self.set_option_default("batch_size", 1000)
67
68     def driver_name(self):
69         if self.get_option('driver_name'):
70             return self.get_option('driver_name')
71         if self.get_option('webkit_test_runner'):
72             return 'WebKitTestRunnerApp.app'
73         return 'DumpRenderTree.app'
74
75     @property
76     def relay_path(self):
77         mac_config = port_config.Config(self._executive, self._filesystem, 'mac')
78         mac_build_directory = mac_config.build_directory(self.get_option('configuration'))
79         return self._filesystem.join(mac_build_directory, self.relay_name)
80
81     def default_timeout_ms(self):
82         if self.get_option('guard_malloc'):
83             return 350 * 1000
84         return super(IOSSimulatorPort, self).default_timeout_ms()
85
86     def supports_per_test_timeout(self):
87         return True
88
89     def _check_relay(self):
90         if not self._filesystem.exists(self.relay_path):
91             _log.error("%s was not found at %s" % (self.relay_name, self.relay_path))
92             return False
93         return True
94
95     def _check_build_relay(self):
96         if self.get_option('build') and not self._build_relay():
97             return False
98         if not self._check_relay():
99             return False
100         return True
101
102     def check_build(self, needs_http):
103         needs_driver = super(IOSSimulatorPort, self).check_build(needs_http)
104         return needs_driver and self._check_build_relay() and self._check_build_image_diff()
105
106     def _path_to_image_diff(self):
107         mac_config = port_config.Config(self._executive, self._filesystem, 'mac')
108         mac_build_directory = mac_config.build_directory(self.get_option('configuration'))
109         return self._filesystem.join(mac_build_directory, 'ImageDiff')
110
111     def _build_relay(self):
112         environment = self.host.copy_current_environment()
113         environment.disable_gcc_smartquotes()
114         env = environment.to_dictionary()
115
116         try:
117             self._run_script("build-layouttestrelay", env=env)
118         except ScriptError, e:
119             _log.error(e.message_with_output(output_limit=None))
120             return False
121         return True
122
123     def _check_image_diff(self):
124         image_diff_path = self._path_to_image_diff()
125         if not self._filesystem.exists(image_diff_path):
126             _log.error("%s was not found at %s" % ('ImageDiff', image_diff_path))
127             return False
128         return True
129
130     def _check_build_image_diff(self):
131         if not self._root_was_set and self.get_option('build') and not self._build_driver():
132             return False
133         if not self._check_image_diff():
134             return False
135         return True
136
137     def _build_image_diff(self):
138         environment = self.host.copy_current_environment()
139         environment.disable_gcc_smartquotes()
140         env = environment.to_dictionary()
141
142         try:
143             self._run_script("build-imagediff", env=env)
144         except ScriptError, e:
145             _log.error(e.message_with_output(output_limit=None))
146             return False
147         return True
148
149     def _build_driver(self):
150         built_tool = super(IOSSimulatorPort, self)._build_driver()
151         built_relay = self._build_relay()
152         built_image_diff = self._build_image_diff()
153         return built_tool and built_relay and built_image_diff
154
155     def _build_driver_flags(self):
156         archs = ['ARCHS=i386'] if self.architecture() == 'x86' else []
157         sdk = ['--sdk', 'iphonesimulator']
158         return archs + sdk
159
160     def should_retry_crashes(self):
161         return True
162
163     def _generate_all_test_configurations(self):
164         configurations = []
165         for build_type in self.ALL_BUILD_TYPES:
166             configurations.append(TestConfiguration(version=self._version, architecture='x86', build_type=build_type))
167         return configurations
168
169     def _driver_class(self):
170         return driver.IOSSimulatorDriver
171
172     def default_baseline_search_path(self):
173         name = self._name.replace('-wk2', '')
174         wk_version = [] if self.get_option('webkit_test_runner') else ['mac-wk1']
175         if name.endswith(self.FUTURE_VERSION):
176             fallback_names = wk_version + [self.port_name]
177         else:
178             fallback_names = self.VERSION_FALLBACK_ORDER[self.VERSION_FALLBACK_ORDER.index(name):-1] + wk_version + [self.port_name]
179         # FIXME: mac-wk2 should appear at the same place as mac-wk1.
180         if self.get_option('webkit_test_runner'):
181             fallback_names = [self._wk2_port_name(), 'wk2'] + fallback_names
182         return map(self._webkit_baseline_path, fallback_names)
183
184     def _port_specific_expectations_files(self):
185         return list(reversed([self._filesystem.join(self._webkit_baseline_path(p), 'TestExpectations') for p in self.baseline_search_path()]))
186
187     def setup_test_run(self):
188         self._executive.run_command(['osascript', '-e', 'tell application "iOS Simulator" to quit'])
189
190     def clean_up_test_run(self):
191         super(IOSSimulatorPort, self).clean_up_test_run()
192         fifos = [path for path in os.listdir('/tmp') if re.search('org.webkit.(DumpRenderTree|WebKitTestRunner).*_(IN|OUT|ERROR)', path)]
193         for fifo in fifos:
194             try:
195                 os.remove(os.path.join('/tmp', fifo))
196             except OSError:
197                 _log.warning('Unable to remove ' + fifo)
198                 pass
199
200     def setup_environ_for_server(self, server_name=None):
201         env = super(IOSSimulatorPort, self).setup_environ_for_server(server_name)
202         if server_name == self.driver_name():
203             if self.get_option('leaks'):
204                 env['MallocStackLogging'] = '1'
205             if self.get_option('guard_malloc'):
206                 env['DYLD_INSERT_LIBRARIES'] = '/usr/lib/libgmalloc.dylib:' + self._build_path("libWebCoreTestShim.dylib")
207             else:
208                 env['DYLD_INSERT_LIBRARIES'] = self._build_path("libWebCoreTestShim.dylib")
209         env['XML_CATALOG_FILES'] = ''  # work around missing /etc/catalog <rdar://problem/4292995>
210         return env
211
212     def operating_system(self):
213         return 'ios-simulator'
214
215     def check_for_leaks(self, process_name, process_pid):
216         if not self.get_option('leaks'):
217             return
218         # We could use http://code.google.com/p/psutil/ to get the process_name from the pid.
219         self._leak_detector.check_for_leaks(process_name, process_pid)
220
221     def print_leaks_summary(self):
222         if not self.get_option('leaks'):
223             return
224         # We're in the manager process, so the leak detector will not have a valid list of leak files.
225         leaks_files = self._leak_detector.leaks_files_in_directory(self.results_directory())
226         if not leaks_files:
227             return
228         total_bytes_string, unique_leaks = self._leak_detector.count_total_bytes_and_unique_leaks(leaks_files)
229         total_leaks = self._leak_detector.count_total_leaks(leaks_files)
230         _log.info("%s total leaks found for a total of %s!" % (total_leaks, total_bytes_string))
231         _log.info("%s unique leaks found!" % unique_leaks)
232
233     def _path_to_webcore_library(self):
234         return self._build_path('WebCore.framework/Versions/A/WebCore')
235
236     def show_results_html_file(self, results_filename):
237         # We don't use self._run_script() because we don't want to wait for the script
238         # to exit and we want the output to show up on stdout in case there are errors
239         # launching the browser.
240         self._executive.popen([self.path_to_script('run-safari')] + self._arguments_for_configuration() + ['--no-saved-state', '-NSOpen', results_filename],
241             cwd=self.webkit_base(), stdout=file(os.devnull), stderr=file(os.devnull))
242
243     def acquire_http_lock(self):
244         pass
245
246     def release_http_lock(self):
247         pass
248
249     def sample_file_path(self, name, pid):
250         return self._filesystem.join(self.results_directory(), "{0}-{1}-sample.txt".format(name, pid))
251
252     def _get_crash_log(self, name, pid, stdout, stderr, newer_than, time_fn=time.time, sleep_fn=time.sleep, wait_for_log=True):
253         time_fn = time_fn or time.time
254         sleep_fn = sleep_fn or time.sleep
255         crash_log = ''
256         crash_logs = CrashLogs(self.host)
257         now = time_fn()
258
259         crash_prefix = 'CRASH: '
260         stderr_lines = []
261         crash_lines = []
262         for line in stderr.splitlines():
263             crash_lines.append(line) if line.startswith(crash_prefix) else stderr_lines.append(line)
264
265         for crash_line in crash_lines:
266             identifier, pid = crash_line[len(crash_prefix):].split(' ')
267             return self._get_crash_log(identifier, int(pid), stdout, '\n'.join(stderr_lines), newer_than, time_fn, sleep_fn, wait_for_log)
268
269         _log.debug('looking for crash log for %s:%s' % (name, str(pid)))
270         deadline = now + 5 * int(self.get_option('child_processes', 1))
271         while not crash_log and now <= deadline:
272             crash_log = crash_logs.find_newest_log(name, pid, include_errors=True, newer_than=newer_than)
273             if not wait_for_log:
274                 break
275             if not crash_log or not [line for line in crash_log.splitlines() if not line.startswith('ERROR')]:
276                 sleep_fn(0.1)
277                 now = time_fn()
278
279         if not crash_log:
280             return stderr, None
281         return stderr, crash_log
282
283     def simulator_udid(self):
284         device_name = self.get_option('device_type').split('.')[-1].replace('-', ' ') + ' WebKit Tester'
285         stdout = subprocess.check_output(['xcrun', '--sdk', 'iphonesimulator', 'simctl', 'list'])
286         lines = stdout.splitlines()
287         try:
288             devices_index = lines.index('== Devices ==')
289             device_regex = re.compile('(?P<device_name>[^(]+) \((?P<udid>[^)]+)\) \((?P<state>[^)]+)\)')
290             for device_line in itertools.takewhile(lambda line: not line.startswith('=='), lines[devices_index + 1:]):
291                 device = device_regex.match(device_line.lstrip().rstrip())
292                 if not device:
293                     continue
294                 if device.group('device_name') == device_name:
295                     return device.group('udid')
296         except ValueError:
297             pass
298
299     def simulator_path(self, udid):
300         if udid:
301             return os.path.realpath(os.path.expanduser(os.path.join('~/Library/Developer/CoreSimulator/Devices', udid)))
302
303     def look_for_new_crash_logs(self, crashed_processes, start_time):
304         crash_logs = {}
305         for (test_name, process_name, pid) in crashed_processes:
306             # Passing None for output.  This is a second pass after the test finished so
307             # if the output had any logging we would have already collected it.
308             crash_log = self._get_crash_log(process_name, pid, None, None, start_time, wait_for_log=False)[1]
309             if not crash_log:
310                 continue
311             crash_logs[test_name] = crash_log
312         return crash_logs
313
314     def look_for_new_samples(self, unresponsive_processes, start_time):
315         sample_files = {}
316         for (test_name, process_name, pid) in unresponsive_processes:
317             sample_file = self.sample_file_path(process_name, pid)
318             if not self._filesystem.isfile(sample_file):
319                 continue
320             sample_files[test_name] = sample_file
321         return sample_files
322
323     def sample_process(self, name, pid):
324         try:
325             hang_report = self.sample_file_path(name, pid)
326             self._executive.run_command([
327                 "/usr/bin/sample",
328                 pid,
329                 10,
330                 10,
331                 "-file",
332                 hang_report,
333             ])
334         except ScriptError as e:
335             _log.warning('Unable to sample process:' + str(e))
336
337     def _path_to_helper(self):
338         binary_name = 'LayoutTestHelper'
339         return self._build_path(binary_name)
340
341     def reset_preferences(self):
342         simulator_path = self.simulator_path(self.simulator_udid())
343         if not simulator_path:
344             return
345         data_path = os.path.join(simulator_path, 'data')
346         if os.path.isdir(data_path):
347             shutil.rmtree(data_path)
348
349     def make_command(self):
350         return self.xcrun_find('make', '/usr/bin/make')
351
352     def nm_command(self):
353         return self.xcrun_find('nm')
354
355     def xcrun_find(self, command, fallback=None):
356         fallback = fallback or command
357         try:
358             return self._executive.run_command(['xcrun', '--sdk', 'iphonesimulator', '-find', command]).rstrip()
359         except ScriptError:
360             _log.warn("xcrun failed; falling back to '%s'." % fallback)
361             return fallback
362
363     def logging_patterns_to_strip(self):
364         return []