Tools/Scripts/webkitpy/port/ios.py

   1 # Copyright (C) 2014, 2015 Apple Inc. All rights reserved.
   2 #
   3 # Redistribution and use in source and binary forms, with or without
   4 # modification, are permitted provided that the following conditions
   5 # are met:
   6 # 1.  Redistributions of source code must retain the above copyright
   7 #     notice, this list of conditions and the following disclaimer.
   8 # 2.  Redistributions in binary form must reproduce the above copyright
   9 #     notice, this list of conditions and the following disclaimer in the
  10 #     documentation and/or other materials provided with the distribution.
  11 #
  12 # THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND
  13 # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  14 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
  15 # DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR
  16 # ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  17 # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
  18 # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
  19 # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  20 # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  21 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  22
  23 import itertools
  24 import logging
  25 import os
  26 import re
  27 import shutil
  28 import subprocess
  29 import time
  30
  31 from webkitpy.common.memoized import memoized
  32 from webkitpy.common.system.crashlogs import CrashLogs
  33 from webkitpy.common.system.executive import ScriptError
  34 from webkitpy.layout_tests.models.test_configuration import TestConfiguration
  35 from webkitpy.port import config as port_config
  36 from webkitpy.port import driver, image_diff
  37 from webkitpy.port.apple import ApplePort
  38 from webkitpy.port.base import Port
  39 from webkitpy.port.leakdetector import LeakDetector
  40 from webkitpy.xcode.simulator import Simulator, Runtime, DeviceType
  41
  42
  43 _log = logging.getLogger(__name__)
  44
  45
  46 class IOSPort(ApplePort):
  47     port_name = "ios"
  48
  49     ARCHITECTURES = ['armv7', 'armv7s', 'arm64']
  50     DEFAULT_ARCHITECTURE = 'armv7'
  51     VERSION_FALLBACK_ORDER = ['ios-7', 'ios-8', 'ios-9']
  52
  53     @classmethod
  54     def determine_full_port_name(cls, host, options, port_name):
  55         if port_name == cls.port_name:
  56             iphoneos_sdk_version = host.platform.xcode_sdk_version('iphoneos')
  57             if not iphoneos_sdk_version:
  58                 raise Exception("Please install the iOS SDK.")
  59             major_version_number = iphoneos_sdk_version.split('.')[0]
  60             port_name = port_name + '-' + major_version_number
  61         return port_name
  62
  63     # Despite their names, these flags do not actually get passed all the way down to webkit-build.
  64     def _build_driver_flags(self):
  65         return ['--sdk', 'iphoneos'] + (['ARCHS=%s' % self.architecture()] if self.architecture() else [])
  66
  67     def operating_system(self):
  68         return 'ios'
  69
  70
  71 class IOSSimulatorPort(Port):
  72     port_name = "ios-simulator"
  73
  74     FUTURE_VERSION = 'future'
  75
  76     ARCHITECTURES = ['x86_64', 'x86']
  77
  78     DEFAULT_ARCHITECTURE = 'x86_64'
  79
  80     SIMULATOR_BUNDLE_ID = 'com.apple.iphonesimulator'
  81
  82     relay_name = 'LayoutTestRelay'
  83
  84     def __init__(self, *args, **kwargs):
  85         super(IOSSimulatorPort, self).__init__(*args, **kwargs)
  86
  87         self._leak_detector = LeakDetector(self)
  88         if self.get_option("leaks"):
  89             # DumpRenderTree slows down noticably if we run more than about 1000 tests in a batch
  90             # with MallocStackLogging enabled.
  91             self.set_option_default("batch_size", 1000)
  92
  93     def driver_name(self):
  94         if self.get_option('driver_name'):
  95             return self.get_option('driver_name')
  96         if self.get_option('webkit_test_runner'):
  97             return 'WebKitTestRunnerApp.app'
  98         return 'DumpRenderTree.app'
  99
 100     @property
 101     @memoized
 102     def simulator_runtime(self):
 103         runtime_identifier = self.get_option('runtime')
 104         if runtime_identifier:
 105             runtime = Runtime.from_identifier(runtime_identifier)
 106         else:
 107             runtime = Runtime.from_version_string(self.host.platform.xcode_sdk_version('iphonesimulator'))
 108         return runtime
 109
 110     @property
 111     @memoized
 112     def simulator_device_type(self):
 113         device_type_identifier = self.get_option('device_type')
 114         if device_type_identifier:
 115             device_type = DeviceType.from_identifier(device_type_identifier)
 116         else:
 117             if self.architecture() == 'x86_64':
 118                 device_type = DeviceType.from_name('iPhone 5s')
 119             else:
 120                 device_type = DeviceType.from_name('iPhone 5')
 121         return device_type
 122
 123     @property
 124     @memoized
 125     def relay_path(self):
 126         if self._root_was_set:
 127             path = self._filesystem.abspath(self.get_option('root'))
 128         else:
 129             mac_config = port_config.Config(self._executive, self._filesystem, 'mac')
 130             path = mac_config.build_directory(self.get_option('configuration'))
 131         return self._filesystem.join(path, self.relay_name)
 132
 133     def default_timeout_ms(self):
 134         if self.get_option('guard_malloc'):
 135             return 350 * 1000
 136         return super(IOSSimulatorPort, self).default_timeout_ms()
 137
 138     def supports_per_test_timeout(self):
 139         return True
 140
 141     def _check_relay(self):
 142         if not self._filesystem.exists(self.relay_path):
 143             _log.error("%s was not found at %s" % (self.relay_name, self.relay_path))
 144             return False
 145         return True
 146
 147     def _check_build_relay(self):
 148         if self.get_option('build') and not self._build_relay():
 149             return False
 150         if not self._check_relay():
 151             return False
 152         return True
 153
 154     def check_build(self, needs_http):
 155         needs_driver = super(IOSSimulatorPort, self).check_build(needs_http)
 156         return needs_driver and self._check_build_relay()
 157
 158     def _build_relay(self):
 159         environment = self.host.copy_current_environment()
 160         environment.disable_gcc_smartquotes()
 161         env = environment.to_dictionary()
 162
 163         try:
 164             # FIXME: We should be passing _arguments_for_configuration(), which respects build configuration and port,
 165             # instead of hardcoding --ios-simulator.
 166             self._run_script("build-layouttestrelay", args=["--ios-simulator"], env=env)
 167         except ScriptError, e:
 168             _log.error(e.message_with_output(output_limit=None))
 169             return False
 170         return True
 171
 172     def _build_driver(self):
 173         built_tool = super(IOSSimulatorPort, self)._build_driver()
 174         built_relay = self._build_relay()
 175         return built_tool and built_relay
 176
 177     def _build_driver_flags(self):
 178         archs = ['ARCHS=i386'] if self.architecture() == 'x86' else []
 179         sdk = ['--sdk', 'iphonesimulator']
 180         return archs + sdk
 181
 182     def should_retry_crashes(self):
 183         return True
 184
 185     def _generate_all_test_configurations(self):
 186         configurations = []
 187         for build_type in self.ALL_BUILD_TYPES:
 188             for architecture in self.ARCHITECTURES:
 189                 configurations.append(TestConfiguration(version=self._version, architecture=architecture, build_type=build_type))
 190         return configurations
 191
 192     def _driver_class(self):
 193         return driver.IOSSimulatorDriver
 194
 195     def default_baseline_search_path(self):
 196         if self.get_option('webkit_test_runner'):
 197             fallback_names = [self._wk2_port_name()] + [self.port_name] + ['wk2']
 198         else:
 199             fallback_names = [self.port_name + '-wk1'] + [self.port_name]
 200
 201         return map(self._webkit_baseline_path, fallback_names)
 202
 203     def _port_specific_expectations_files(self):
 204         return list(reversed([self._filesystem.join(self._webkit_baseline_path(p), 'TestExpectations') for p in self.baseline_search_path()]))
 205
 206     def setup_test_run(self):
 207         device_udid = self.testing_device.udid
 208         # FIXME: <rdar://problem/20916140> Switch to using CoreSimulator.framework for launching and quitting iOS Simulator
 209         self._executive.run_command([
 210             'open', '-b', self.SIMULATOR_BUNDLE_ID,
 211             '--args', '-CurrentDeviceUDID', device_udid])
 212         Simulator.wait_until_device_is_in_state(device_udid, Simulator.DeviceState.BOOTED)
 213
 214         # FIXME: Pause here until SpringBoard finishes launching to workaround <rdar://problem/20000383>.
 215         boot_delay = 30
 216         _log.debug('Waiting {seconds} seconds for iOS Simulator to finish booting ...'.format(seconds=boot_delay))
 217         time.sleep(boot_delay)
 218
 219     def _quit_ios_simulator(self):
 220         # FIXME: <rdar://problem/20916140> Switch to using CoreSimulator.framework for launching and quitting iOS Simulator
 221         self._executive.run_command(['osascript', '-e', 'tell application id "{0}" to quit'.format(self.SIMULATOR_BUNDLE_ID)])
 222
 223     def clean_up_test_run(self):
 224         super(IOSSimulatorPort, self).clean_up_test_run()
 225         self._quit_ios_simulator()
 226         fifos = [path for path in os.listdir('/tmp') if re.search('org.webkit.(DumpRenderTree|WebKitTestRunner).*_(IN|OUT|ERROR)', path)]
 227         for fifo in fifos:
 228             try:
 229                 os.remove(os.path.join('/tmp', fifo))
 230             except OSError:
 231                 _log.warning('Unable to remove ' + fifo)
 232                 pass
 233
 234     def setup_environ_for_server(self, server_name=None):
 235         env = super(IOSSimulatorPort, self).setup_environ_for_server(server_name)
 236         if server_name == self.driver_name():
 237             if self.get_option('leaks'):
 238                 env['MallocStackLogging'] = '1'
 239             if self.get_option('guard_malloc'):
 240                 self._append_value_colon_separated(env, 'DYLD_INSERT_LIBRARIES', '/usr/lib/libgmalloc.dylib')
 241             self._append_value_colon_separated(env, 'DYLD_INSERT_LIBRARIES', self._build_path("libWebCoreTestShim.dylib"))
 242         env['XML_CATALOG_FILES'] = ''  # work around missing /etc/catalog <rdar://problem/4292995>
 243         return env
 244
 245     def operating_system(self):
 246         return 'ios-simulator'
 247
 248     def check_sys_deps(self, needs_http):
 249         if not self.simulator_runtime.available:
 250             _log.error('The iOS Simulator runtime with identifier "{0}" cannot be used because it is unavailable.'.format(self.simulator_runtime.identifier))
 251             return False
 252         testing_device = self.testing_device  # May create a new simulator device
 253
 254         if not Simulator.check_simulator_device_and_erase_if_needed(self.host, testing_device.udid):
 255             _log.error('Unable to boot the simulator device with UDID {0}.'.format(testing_device.udid))
 256             return False
 257         return super(IOSSimulatorPort, self).check_sys_deps(needs_http)
 258
 259     def check_for_leaks(self, process_name, process_pid):
 260         if not self.get_option('leaks'):
 261             return
 262         # We could use http://code.google.com/p/psutil/ to get the process_name from the pid.
 263         self._leak_detector.check_for_leaks(process_name, process_pid)
 264
 265     def print_leaks_summary(self):
 266         if not self.get_option('leaks'):
 267             return
 268         # We're in the manager process, so the leak detector will not have a valid list of leak files.
 269         leaks_files = self._leak_detector.leaks_files_in_directory(self.results_directory())
 270         if not leaks_files:
 271             return
 272         total_bytes_string, unique_leaks = self._leak_detector.count_total_bytes_and_unique_leaks(leaks_files)
 273         total_leaks = self._leak_detector.count_total_leaks(leaks_files)
 274         _log.info("%s total leaks found for a total of %s." % (total_leaks, total_bytes_string))
 275         _log.info("%s unique leaks found." % unique_leaks)
 276
 277     def _path_to_webcore_library(self):
 278         return self._build_path('WebCore.framework/Versions/A/WebCore')
 279
 280     def show_results_html_file(self, results_filename):
 281         # We don't use self._run_script() because we don't want to wait for the script
 282         # to exit and we want the output to show up on stdout in case there are errors
 283         # launching the browser.
 284         self._executive.popen([self.path_to_script('run-safari')] + self._arguments_for_configuration() + ['--no-saved-state', '-NSOpen', results_filename],
 285             cwd=self.webkit_base(), stdout=file(os.devnull), stderr=file(os.devnull))
 286
 287     def sample_file_path(self, name, pid):
 288         return self._filesystem.join(self.results_directory(), "{0}-{1}-sample.txt".format(name, pid))
 289
 290     SUBPROCESS_CRASH_REGEX = re.compile('#CRASHED - (?P<subprocess_name>\S+) \(pid (?P<subprocess_pid>\d+)\)')
 291
 292     def _get_crash_log(self, name, pid, stdout, stderr, newer_than, time_fn=time.time, sleep_fn=time.sleep, wait_for_log=True):
 293         time_fn = time_fn or time.time
 294         sleep_fn = sleep_fn or time.sleep
 295
 296         # FIXME: We should collect the actual crash log for DumpRenderTree.app because it includes more
 297         # information (e.g. exception codes) than is available in the stack trace written to standard error.
 298         stderr_lines = []
 299         crashed_subprocess_name_and_pid = None  # e.g. ('DumpRenderTree.app', 1234)
 300         for line in (stderr or '').splitlines():
 301             if not crashed_subprocess_name_and_pid:
 302                 match = self.SUBPROCESS_CRASH_REGEX.match(line)
 303                 if match:
 304                     crashed_subprocess_name_and_pid = (match.group('subprocess_name'), int(match.group('subprocess_pid')))
 305                     continue
 306             stderr_lines.append(line)
 307
 308         if crashed_subprocess_name_and_pid:
 309             return self._get_crash_log(crashed_subprocess_name_and_pid[0], crashed_subprocess_name_and_pid[1], stdout,
 310                 '\n'.join(stderr_lines), newer_than, time_fn, sleep_fn, wait_for_log)
 311
 312         # LayoutTestRelay crashed
 313         _log.debug('looking for crash log for %s:%s' % (name, str(pid)))
 314         crash_log = ''
 315         crash_logs = CrashLogs(self.host)
 316         now = time_fn()
 317         deadline = now + 5 * int(self.get_option('child_processes', 1))
 318         while not crash_log and now <= deadline:
 319             crash_log = crash_logs.find_newest_log(name, pid, include_errors=True, newer_than=newer_than)
 320             if not wait_for_log:
 321                 break
 322             if not crash_log or not [line for line in crash_log.splitlines() if not line.startswith('ERROR')]:
 323                 sleep_fn(0.1)
 324                 now = time_fn()
 325
 326         if not crash_log:
 327             return stderr, None
 328         return stderr, crash_log
 329
 330     @property
 331     @memoized
 332     def testing_device(self):
 333         return Simulator().lookup_or_create_device(self.simulator_device_type.name + ' WebKit Tester', self.simulator_device_type, self.simulator_runtime)
 334
 335     def _merge_crash_logs(self, logs, new_logs, crashed_processes):
 336         for test, crash_log in new_logs.iteritems():
 337             try:
 338                 process_name = test.split("-")[0]
 339                 pid = int(test.split("-")[1])
 340             except IndexError:
 341                 continue
 342             if not any(entry[1] == process_name and entry[2] == pid for entry in crashed_processes):
 343                 # if this is a new crash, then append the logs
 344                 logs[test] = crash_log
 345         return logs
 346
 347     def _look_for_all_crash_logs_in_log_dir(self, newer_than):
 348         crash_log = CrashLogs(self.host)
 349         return crash_log.find_all_logs(include_errors=True, newer_than=newer_than)
 350
 351     def look_for_new_crash_logs(self, crashed_processes, start_time):
 352         crash_logs = {}
 353         for (test_name, process_name, pid) in crashed_processes:
 354             # Passing None for output.  This is a second pass after the test finished so
 355             # if the output had any logging we would have already collected it.
 356             crash_log = self._get_crash_log(process_name, pid, None, None, start_time, wait_for_log=False)[1]
 357             if not crash_log:
 358                 continue
 359             crash_logs[test_name] = crash_log
 360         all_crash_log = self._look_for_all_crash_logs_in_log_dir(start_time)
 361         return self._merge_crash_logs(crash_logs, all_crash_log, crashed_processes)
 362
 363     def look_for_new_samples(self, unresponsive_processes, start_time):
 364         sample_files = {}
 365         for (test_name, process_name, pid) in unresponsive_processes:
 366             sample_file = self.sample_file_path(process_name, pid)
 367             if not self._filesystem.isfile(sample_file):
 368                 continue
 369             sample_files[test_name] = sample_file
 370         return sample_files
 371
 372     def sample_process(self, name, pid):
 373         try:
 374             hang_report = self.sample_file_path(name, pid)
 375             self._executive.run_command([
 376                 "/usr/bin/sample",
 377                 pid,
 378                 10,
 379                 10,
 380                 "-file",
 381                 hang_report,
 382             ])
 383         except ScriptError as e:
 384             _log.warning('Unable to sample process:' + str(e))
 385
 386     def _path_to_helper(self):
 387         binary_name = 'LayoutTestHelper'
 388         return self._build_path(binary_name)
 389
 390     def diff_image(self, expected_contents, actual_contents, tolerance=None):
 391         if not actual_contents and not expected_contents:
 392             return (None, 0, None)
 393         if not actual_contents or not expected_contents:
 394             return (True, 0, None)
 395         if not self._image_differ:
 396             self._image_differ = image_diff.IOSSimulatorImageDiffer(self)
 397         self.set_option_default('tolerance', 0.1)
 398         if tolerance is None:
 399             tolerance = self.get_option('tolerance')
 400         return self._image_differ.diff_image(expected_contents, actual_contents, tolerance)
 401
 402     def reset_preferences(self):
 403         # We assume that if testing_device is booted that it was booted by the iOS Simulator app
 404         # (as opposed to simctl). So, quit the iOS Simulator app to shutdown testing_device.
 405         self._quit_ios_simulator()
 406         Simulator.wait_until_device_is_in_state(self.testing_device.udid, Simulator.DeviceState.SHUTDOWN)
 407
 408         data_path = os.path.join(self.testing_device.path, 'data')
 409         if os.path.isdir(data_path):
 410             shutil.rmtree(data_path)
 411
 412     def make_command(self):
 413         return self.xcrun_find('make', '/usr/bin/make')
 414
 415     def nm_command(self):
 416         return self.xcrun_find('nm')
 417
 418     def xcrun_find(self, command, fallback=None):
 419         fallback = fallback or command
 420         try:
 421             return self._executive.run_command(['xcrun', '--sdk', 'iphonesimulator', '-find', command]).rstrip()
 422         except ScriptError:
 423             _log.warn("xcrun failed; falling back to '%s'." % fallback)
 424             return fallback
 425
 426     @property
 427     def developer_dir(self):
 428         return self._executive.run_command(['xcode-select', '--print-path']).rstrip()
 429
 430     def logging_patterns_to_strip(self):
 431         return []
 432
 433     def stderr_patterns_to_strip(self):
 434         return []