Tools/Scripts/webkitpy/port/ios.py

   1 # Copyright (C) 2014, 2015 Apple Inc. All rights reserved.
   2 #
   3 # Redistribution and use in source and binary forms, with or without
   4 # modification, are permitted provided that the following conditions
   5 # are met:
   6 # 1.  Redistributions of source code must retain the above copyright
   7 #     notice, this list of conditions and the following disclaimer.
   8 # 2.  Redistributions in binary form must reproduce the above copyright
   9 #     notice, this list of conditions and the following disclaimer in the
  10 #     documentation and/or other materials provided with the distribution.
  11 #
  12 # THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND
  13 # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  14 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
  15 # DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR
  16 # ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  17 # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
  18 # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
  19 # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  20 # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  21 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  22
  23 import itertools
  24 import logging
  25 import os
  26 import re
  27 import shutil
  28 import subprocess
  29 import time
  30
  31 from webkitpy.common.memoized import memoized
  32 from webkitpy.common.system.crashlogs import CrashLogs
  33 from webkitpy.common.system.executive import ScriptError
  34 from webkitpy.layout_tests.models.test_configuration import TestConfiguration
  35 from webkitpy.port import config as port_config
  36 from webkitpy.port import driver, image_diff
  37 from webkitpy.port.apple import ApplePort
  38 from webkitpy.port.base import Port
  39 from webkitpy.port.leakdetector import LeakDetector
  40 from webkitpy.xcode.simulator import Simulator, Runtime, DeviceType
  41
  42
  43 _log = logging.getLogger(__name__)
  44
  45
  46 class IOSPort(ApplePort):
  47     port_name = "ios"
  48
  49     ARCHITECTURES = ['armv7', 'armv7s', 'arm64']
  50     DEFAULT_ARCHITECTURE = 'armv7'
  51     VERSION_FALLBACK_ORDER = ['ios-7', 'ios-8', 'ios-9']
  52
  53     @classmethod
  54     def determine_full_port_name(cls, host, options, port_name):
  55         if port_name == cls.port_name:
  56             iphoneos_sdk_version = host.platform.xcode_sdk_version('iphoneos')
  57             if not iphoneos_sdk_version:
  58                 raise Exception("Please install the iOS SDK.")
  59             major_version_number = iphoneos_sdk_version.split('.')[0]
  60             port_name = port_name + '-' + major_version_number
  61         return port_name
  62
  63     # Despite their names, these flags do not actually get passed all the way down to webkit-build.
  64     def _build_driver_flags(self):
  65         return ['--sdk', 'iphoneos'] + (['ARCHS=%s' % self.architecture()] if self.architecture() else [])
  66
  67     def operating_system(self):
  68         return 'ios'
  69
  70
  71 class IOSSimulatorPort(Port):
  72     port_name = "ios-simulator"
  73
  74     FUTURE_VERSION = 'future'
  75
  76     ARCHITECTURES = ['x86_64', 'x86']
  77
  78     DEFAULT_ARCHITECTURE = 'x86_64'
  79
  80     SIMULATOR_BUNDLE_ID = 'com.apple.iphonesimulator'
  81
  82     relay_name = 'LayoutTestRelay'
  83
  84     def __init__(self, *args, **kwargs):
  85         super(IOSSimulatorPort, self).__init__(*args, **kwargs)
  86
  87         self._leak_detector = LeakDetector(self)
  88         if self.get_option("leaks"):
  89             # DumpRenderTree slows down noticably if we run more than about 1000 tests in a batch
  90             # with MallocStackLogging enabled.
  91             self.set_option_default("batch_size", 1000)
  92
  93     def driver_name(self):
  94         if self.get_option('driver_name'):
  95             return self.get_option('driver_name')
  96         if self.get_option('webkit_test_runner'):
  97             return 'WebKitTestRunnerApp.app'
  98         return 'DumpRenderTree.app'
  99
 100     @property
 101     @memoized
 102     def simulator_runtime(self):
 103         runtime_identifier = self.get_option('runtime')
 104         if runtime_identifier:
 105             runtime = Runtime.from_identifier(runtime_identifier)
 106         else:
 107             runtime = Runtime.from_version_string(self.host.platform.xcode_sdk_version('iphonesimulator'))
 108         return runtime
 109
 110     @property
 111     @memoized
 112     def simulator_device_type(self):
 113         device_type_identifier = self.get_option('device_type')
 114         if device_type_identifier:
 115             device_type = DeviceType.from_identifier(device_type_identifier)
 116         else:
 117             if self.architecture() == 'x86_64':
 118                 device_type = DeviceType.from_name('iPhone 5s')
 119             else:
 120                 device_type = DeviceType.from_name('iPhone 5')
 121         return device_type
 122
 123     @property
 124     @memoized
 125     def relay_path(self):
 126         mac_config = port_config.Config(self._executive, self._filesystem, 'mac')
 127         return self._filesystem.join(mac_config.build_directory(self.get_option('configuration')), self.relay_name)
 128
 129     def default_timeout_ms(self):
 130         if self.get_option('guard_malloc'):
 131             return 350 * 1000
 132         return super(IOSSimulatorPort, self).default_timeout_ms()
 133
 134     def supports_per_test_timeout(self):
 135         return True
 136
 137     def _check_relay(self):
 138         if not self._filesystem.exists(self.relay_path):
 139             _log.error("%s was not found at %s" % (self.relay_name, self.relay_path))
 140             return False
 141         return True
 142
 143     def _check_build_relay(self):
 144         if self.get_option('build') and not self._build_relay():
 145             return False
 146         if not self._check_relay():
 147             return False
 148         return True
 149
 150     def check_build(self, needs_http):
 151         needs_driver = super(IOSSimulatorPort, self).check_build(needs_http)
 152         return needs_driver and self._check_build_relay()
 153
 154     def _build_relay(self):
 155         environment = self.host.copy_current_environment()
 156         environment.disable_gcc_smartquotes()
 157         env = environment.to_dictionary()
 158
 159         try:
 160             # FIXME: We should be passing _arguments_for_configuration(), which respects build configuration and port,
 161             # instead of hardcoding --ios-simulator.
 162             self._run_script("build-layouttestrelay", args=["--ios-simulator"], env=env)
 163         except ScriptError, e:
 164             _log.error(e.message_with_output(output_limit=None))
 165             return False
 166         return True
 167
 168     def _build_driver(self):
 169         built_tool = super(IOSSimulatorPort, self)._build_driver()
 170         built_relay = self._build_relay()
 171         return built_tool and built_relay
 172
 173     def _build_driver_flags(self):
 174         archs = ['ARCHS=i386'] if self.architecture() == 'x86' else []
 175         sdk = ['--sdk', 'iphonesimulator']
 176         return archs + sdk
 177
 178     def should_retry_crashes(self):
 179         return True
 180
 181     def _generate_all_test_configurations(self):
 182         configurations = []
 183         for build_type in self.ALL_BUILD_TYPES:
 184             for architecture in self.ARCHITECTURES:
 185                 configurations.append(TestConfiguration(version=self._version, architecture=architecture, build_type=build_type))
 186         return configurations
 187
 188     def _driver_class(self):
 189         return driver.IOSSimulatorDriver
 190
 191     def default_baseline_search_path(self):
 192         if self.get_option('webkit_test_runner'):
 193             fallback_names = [self._wk2_port_name()] + [self.port_name] + ['wk2']
 194         else:
 195             fallback_names = [self.port_name + '-wk1'] + [self.port_name]
 196
 197         return map(self._webkit_baseline_path, fallback_names)
 198
 199     def _port_specific_expectations_files(self):
 200         return list(reversed([self._filesystem.join(self._webkit_baseline_path(p), 'TestExpectations') for p in self.baseline_search_path()]))
 201
 202     def setup_test_run(self):
 203         device_udid = self.testing_device.udid
 204         # FIXME: <rdar://problem/20916140> Switch to using CoreSimulator.framework for launching and quitting iOS Simulator
 205         self._executive.run_command([
 206             'open', '-b', self.SIMULATOR_BUNDLE_ID,
 207             '--args', '-CurrentDeviceUDID', device_udid])
 208         Simulator.wait_until_device_is_in_state(device_udid, Simulator.DeviceState.BOOTED)
 209
 210         # FIXME: Pause here until SpringBoard finishes launching to workaround <rdar://problem/20000383>.
 211         boot_delay = 30
 212         _log.debug('Waiting {seconds} seconds for iOS Simulator to finish booting ...'.format(seconds=boot_delay))
 213         time.sleep(boot_delay)
 214
 215     def _quit_ios_simulator(self):
 216         # FIXME: <rdar://problem/20916140> Switch to using CoreSimulator.framework for launching and quitting iOS Simulator
 217         self._executive.run_command(['osascript', '-e', 'tell application id "{0}" to quit'.format(self.SIMULATOR_BUNDLE_ID)])
 218
 219     def clean_up_test_run(self):
 220         super(IOSSimulatorPort, self).clean_up_test_run()
 221         self._quit_ios_simulator()
 222         fifos = [path for path in os.listdir('/tmp') if re.search('org.webkit.(DumpRenderTree|WebKitTestRunner).*_(IN|OUT|ERROR)', path)]
 223         for fifo in fifos:
 224             try:
 225                 os.remove(os.path.join('/tmp', fifo))
 226             except OSError:
 227                 _log.warning('Unable to remove ' + fifo)
 228                 pass
 229
 230     def setup_environ_for_server(self, server_name=None):
 231         env = super(IOSSimulatorPort, self).setup_environ_for_server(server_name)
 232         if server_name == self.driver_name():
 233             if self.get_option('leaks'):
 234                 env['MallocStackLogging'] = '1'
 235             if self.get_option('guard_malloc'):
 236                 self._append_value_colon_separated(env, 'DYLD_INSERT_LIBRARIES', '/usr/lib/libgmalloc.dylib')
 237             self._append_value_colon_separated(env, 'DYLD_INSERT_LIBRARIES', self._build_path("libWebCoreTestShim.dylib"))
 238         env['XML_CATALOG_FILES'] = ''  # work around missing /etc/catalog <rdar://problem/4292995>
 239         return env
 240
 241     def operating_system(self):
 242         return 'ios-simulator'
 243
 244     def check_sys_deps(self, needs_http):
 245         if not self.simulator_runtime.available:
 246             _log.error('The iOS Simulator runtime with identifier "{0}" cannot be used because it is unavailable.'.format(self.simulator_runtime.identifier))
 247             return False
 248         testing_device = self.testing_device  # May create a new simulator device
 249
 250         if not Simulator.check_simulator_device_and_erase_if_needed(self.host, testing_device.udid):
 251             _log.error('Unable to boot the simulator device with UDID {0}.'.format(testing_device.udid))
 252             return False
 253         return super(IOSSimulatorPort, self).check_sys_deps(needs_http)
 254
 255     def check_for_leaks(self, process_name, process_pid):
 256         if not self.get_option('leaks'):
 257             return
 258         # We could use http://code.google.com/p/psutil/ to get the process_name from the pid.
 259         self._leak_detector.check_for_leaks(process_name, process_pid)
 260
 261     def print_leaks_summary(self):
 262         if not self.get_option('leaks'):
 263             return
 264         # We're in the manager process, so the leak detector will not have a valid list of leak files.
 265         leaks_files = self._leak_detector.leaks_files_in_directory(self.results_directory())
 266         if not leaks_files:
 267             return
 268         total_bytes_string, unique_leaks = self._leak_detector.count_total_bytes_and_unique_leaks(leaks_files)
 269         total_leaks = self._leak_detector.count_total_leaks(leaks_files)
 270         _log.info("%s total leaks found for a total of %s." % (total_leaks, total_bytes_string))
 271         _log.info("%s unique leaks found." % unique_leaks)
 272
 273     def _path_to_webcore_library(self):
 274         return self._build_path('WebCore.framework/Versions/A/WebCore')
 275
 276     def show_results_html_file(self, results_filename):
 277         # We don't use self._run_script() because we don't want to wait for the script
 278         # to exit and we want the output to show up on stdout in case there are errors
 279         # launching the browser.
 280         self._executive.popen([self.path_to_script('run-safari')] + self._arguments_for_configuration() + ['--no-saved-state', '-NSOpen', results_filename],
 281             cwd=self.webkit_base(), stdout=file(os.devnull), stderr=file(os.devnull))
 282
 283     def sample_file_path(self, name, pid):
 284         return self._filesystem.join(self.results_directory(), "{0}-{1}-sample.txt".format(name, pid))
 285
 286     SUBPROCESS_CRASH_REGEX = re.compile('#CRASHED - (?P<subprocess_name>\S+) \(pid (?P<subprocess_pid>\d+)\)')
 287
 288     def _get_crash_log(self, name, pid, stdout, stderr, newer_than, time_fn=time.time, sleep_fn=time.sleep, wait_for_log=True):
 289         time_fn = time_fn or time.time
 290         sleep_fn = sleep_fn or time.sleep
 291
 292         # FIXME: We should collect the actual crash log for DumpRenderTree.app because it includes more
 293         # information (e.g. exception codes) than is available in the stack trace written to standard error.
 294         stderr_lines = []
 295         crashed_subprocess_name_and_pid = None  # e.g. ('DumpRenderTree.app', 1234)
 296         for line in (stderr or '').splitlines():
 297             if not crashed_subprocess_name_and_pid:
 298                 match = self.SUBPROCESS_CRASH_REGEX.match(line)
 299                 if match:
 300                     crashed_subprocess_name_and_pid = (match.group('subprocess_name'), int(match.group('subprocess_pid')))
 301                     continue
 302             stderr_lines.append(line)
 303
 304         if crashed_subprocess_name_and_pid:
 305             return self._get_crash_log(crashed_subprocess_name_and_pid[0], crashed_subprocess_name_and_pid[1], stdout,
 306                 '\n'.join(stderr_lines), newer_than, time_fn, sleep_fn, wait_for_log)
 307
 308         # LayoutTestRelay crashed
 309         _log.debug('looking for crash log for %s:%s' % (name, str(pid)))
 310         crash_log = ''
 311         crash_logs = CrashLogs(self.host)
 312         now = time_fn()
 313         deadline = now + 5 * int(self.get_option('child_processes', 1))
 314         while not crash_log and now <= deadline:
 315             crash_log = crash_logs.find_newest_log(name, pid, include_errors=True, newer_than=newer_than)
 316             if not wait_for_log:
 317                 break
 318             if not crash_log or not [line for line in crash_log.splitlines() if not line.startswith('ERROR')]:
 319                 sleep_fn(0.1)
 320                 now = time_fn()
 321
 322         if not crash_log:
 323             return stderr, None
 324         return stderr, crash_log
 325
 326     @property
 327     @memoized
 328     def testing_device(self):
 329         return Simulator().lookup_or_create_device(self.simulator_device_type.name + ' WebKit Tester', self.simulator_device_type, self.simulator_runtime)
 330
 331     def _merge_crash_logs(self, logs, new_logs, crashed_processes):
 332         for test, crash_log in new_logs.iteritems():
 333             try:
 334                 process_name = test.split("-")[0]
 335                 pid = int(test.split("-")[1])
 336             except IndexError:
 337                 continue
 338             if not any(entry[1] == process_name and entry[2] == pid for entry in crashed_processes):
 339                 # if this is a new crash, then append the logs
 340                 logs[test] = crash_log
 341         return logs
 342
 343     def _look_for_all_crash_logs_in_log_dir(self, newer_than):
 344         crash_log = CrashLogs(self.host)
 345         return crash_log.find_all_logs(include_errors=True, newer_than=newer_than)
 346
 347     def look_for_new_crash_logs(self, crashed_processes, start_time):
 348         crash_logs = {}
 349         for (test_name, process_name, pid) in crashed_processes:
 350             # Passing None for output.  This is a second pass after the test finished so
 351             # if the output had any logging we would have already collected it.
 352             crash_log = self._get_crash_log(process_name, pid, None, None, start_time, wait_for_log=False)[1]
 353             if not crash_log:
 354                 continue
 355             crash_logs[test_name] = crash_log
 356         all_crash_log = self._look_for_all_crash_logs_in_log_dir(start_time)
 357         return self._merge_crash_logs(crash_logs, all_crash_log, crashed_processes)
 358
 359     def look_for_new_samples(self, unresponsive_processes, start_time):
 360         sample_files = {}
 361         for (test_name, process_name, pid) in unresponsive_processes:
 362             sample_file = self.sample_file_path(process_name, pid)
 363             if not self._filesystem.isfile(sample_file):
 364                 continue
 365             sample_files[test_name] = sample_file
 366         return sample_files
 367
 368     def sample_process(self, name, pid):
 369         try:
 370             hang_report = self.sample_file_path(name, pid)
 371             self._executive.run_command([
 372                 "/usr/bin/sample",
 373                 pid,
 374                 10,
 375                 10,
 376                 "-file",
 377                 hang_report,
 378             ])
 379         except ScriptError as e:
 380             _log.warning('Unable to sample process:' + str(e))
 381
 382     def _path_to_helper(self):
 383         binary_name = 'LayoutTestHelper'
 384         return self._build_path(binary_name)
 385
 386     def diff_image(self, expected_contents, actual_contents, tolerance=None):
 387         if not actual_contents and not expected_contents:
 388             return (None, 0, None)
 389         if not actual_contents or not expected_contents:
 390             return (True, 0, None)
 391         if not self._image_differ:
 392             self._image_differ = image_diff.IOSSimulatorImageDiffer(self)
 393         self.set_option_default('tolerance', 0.1)
 394         if tolerance is None:
 395             tolerance = self.get_option('tolerance')
 396         return self._image_differ.diff_image(expected_contents, actual_contents, tolerance)
 397
 398     def reset_preferences(self):
 399         # We assume that if testing_device is booted that it was booted by the iOS Simulator app
 400         # (as opposed to simctl). So, quit the iOS Simulator app to shutdown testing_device.
 401         self._quit_ios_simulator()
 402         Simulator.wait_until_device_is_in_state(self.testing_device.udid, Simulator.DeviceState.SHUTDOWN)
 403
 404         data_path = os.path.join(self.testing_device.path, 'data')
 405         if os.path.isdir(data_path):
 406             shutil.rmtree(data_path)
 407
 408     def make_command(self):
 409         return self.xcrun_find('make', '/usr/bin/make')
 410
 411     def nm_command(self):
 412         return self.xcrun_find('nm')
 413
 414     def xcrun_find(self, command, fallback=None):
 415         fallback = fallback or command
 416         try:
 417             return self._executive.run_command(['xcrun', '--sdk', 'iphonesimulator', '-find', command]).rstrip()
 418         except ScriptError:
 419             _log.warn("xcrun failed; falling back to '%s'." % fallback)
 420             return fallback
 421
 422     @property
 423     def developer_dir(self):
 424         return self._executive.run_command(['xcode-select', '--print-path']).rstrip()
 425
 426     def logging_patterns_to_strip(self):
 427         return []
 428
 429     def stderr_patterns_to_strip(self):
 430         return []