Allow a port to run tests with a custom device setup
authorsimon.fraser@apple.com <simon.fraser@apple.com@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Mon, 15 Aug 2016 20:50:13 +0000 (20:50 +0000)
committersimon.fraser@apple.com <simon.fraser@apple.com@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Mon, 15 Aug 2016 20:50:13 +0000 (20:50 +0000)
https://bugs.webkit.org/show_bug.cgi?id=160833

Reviewed by Daniel Bates.

These changes allow the IOSSimulator port to run tests in iPad mode.

This is made possible by allowing a platform to define CUSTOM_DEVICE_CLASSES,
in this case 'ipad'. When specified, any test in a directory with a suffix that matches
a custom device will be collected into a set, and run in that device's environment after
the other tests have run.

* Scripts/webkitpy/layout_tests/controllers/manager.py:
(Manager._custom_device_for_test): If the test contains a directory matching a
custom device suffix, return that custom device.
(Manager._set_up_run): Push the custom device class, if any, into options so
that the Worker can get to it.
(Manager.run): Go through the list of tests, and break it down into device-generic
tests, and tests for each device class. _run_test_subset is then called for
each collection of tests, and the results merged.
(Manager._run_test_subset): Some lines unwrapped.
(Manager._end_test_run):
(Manager._run_tests):
* Scripts/webkitpy/layout_tests/controllers/single_test_runner.py:
(SingleTestRunner.__init__): Unwrapped a line.
* Scripts/webkitpy/layout_tests/models/test_run_results.py:
(TestRunResults.merge): Add this function to merge TestRunResults
* Scripts/webkitpy/layout_tests/views/printing.py:
(Printer.print_workers_and_shards): Print the custom device, if any.
* Scripts/webkitpy/port/base.py:
(Port): Base port has empty array of custom devices.
(Port.setup_test_run): Add device_class argument.
* Scripts/webkitpy/port/driver.py:
(DriverInput.__repr__):
(Driver.check_driver.implementation):
* Scripts/webkitpy/port/efl.py:
(EflPort.setup_test_run):
* Scripts/webkitpy/port/gtk.py:
(GtkPort.setup_test_run):
* Scripts/webkitpy/port/ios.py:
(IOSSimulatorPort): Add CUSTOM_DEVICE_CLASSES for ipad.
(IOSSimulatorPort.__init__):
(IOSSimulatorPort.simulator_device_type): Use a device name from the DEVICE_CLASS_MAP
based on the custom device class.
(IOSSimulatorPort._set_device_class):
(IOSSimulatorPort._create_simulators): Factor some code into this function.
(IOSSimulatorPort.setup_test_run):
(IOSSimulatorPort.testing_device):
(IOSSimulatorPort.reset_preferences): This used to create the simulator apps, but that
seemed wrong for this function. That was moved to setup_test_run().
(IOSSimulatorPort.check_sys_deps): This function used to create testing devices,
but this happened too early, before we knew which kind of devices to create. Devices
are now created in setup_test_run().
* Scripts/webkitpy/port/win.py:
(WinPort.setup_test_run):

git-svn-id: https://svn.webkit.org/repository/webkit/trunk@204477 268f45cc-cd09-0410-ab3c-d52691b4dbfc

13 files changed:
Tools/ChangeLog
Tools/Scripts/webkitpy/layout_tests/controllers/manager.py
Tools/Scripts/webkitpy/layout_tests/controllers/manager_unittest.py
Tools/Scripts/webkitpy/layout_tests/controllers/single_test_runner.py
Tools/Scripts/webkitpy/layout_tests/models/test_run_results.py
Tools/Scripts/webkitpy/layout_tests/views/printing.py
Tools/Scripts/webkitpy/port/base.py
Tools/Scripts/webkitpy/port/driver.py
Tools/Scripts/webkitpy/port/efl.py
Tools/Scripts/webkitpy/port/gtk.py
Tools/Scripts/webkitpy/port/ios.py
Tools/Scripts/webkitpy/port/test.py
Tools/Scripts/webkitpy/port/win.py

index 734145bfbe22716ea4d54afc54e203ba7d824ea8..23863361f9591f0e8e2d23b80aad891ace0b7d47 100644 (file)
@@ -1,3 +1,61 @@
+2016-08-15  Simon Fraser  <simon.fraser@apple.com>
+
+        Allow a port to run tests with a custom device setup
+        https://bugs.webkit.org/show_bug.cgi?id=160833
+
+        Reviewed by Daniel Bates.
+
+        These changes allow the IOSSimulator port to run tests in iPad mode.
+
+        This is made possible by allowing a platform to define CUSTOM_DEVICE_CLASSES,
+        in this case 'ipad'. When specified, any test in a directory with a suffix that matches
+        a custom device will be collected into a set, and run in that device's environment after
+        the other tests have run.
+
+        * Scripts/webkitpy/layout_tests/controllers/manager.py:
+        (Manager._custom_device_for_test): If the test contains a directory matching a
+        custom device suffix, return that custom device.
+        (Manager._set_up_run): Push the custom device class, if any, into options so
+        that the Worker can get to it.
+        (Manager.run): Go through the list of tests, and break it down into device-generic
+        tests, and tests for each device class. _run_test_subset is then called for
+        each collection of tests, and the results merged.
+        (Manager._run_test_subset): Some lines unwrapped.
+        (Manager._end_test_run):
+        (Manager._run_tests):
+        * Scripts/webkitpy/layout_tests/controllers/single_test_runner.py:
+        (SingleTestRunner.__init__): Unwrapped a line.
+        * Scripts/webkitpy/layout_tests/models/test_run_results.py:
+        (TestRunResults.merge): Add this function to merge TestRunResults
+        * Scripts/webkitpy/layout_tests/views/printing.py:
+        (Printer.print_workers_and_shards): Print the custom device, if any.
+        * Scripts/webkitpy/port/base.py:
+        (Port): Base port has empty array of custom devices.
+        (Port.setup_test_run): Add device_class argument.
+        * Scripts/webkitpy/port/driver.py:
+        (DriverInput.__repr__):
+        (Driver.check_driver.implementation):
+        * Scripts/webkitpy/port/efl.py:
+        (EflPort.setup_test_run):
+        * Scripts/webkitpy/port/gtk.py:
+        (GtkPort.setup_test_run):
+        * Scripts/webkitpy/port/ios.py:
+        (IOSSimulatorPort): Add CUSTOM_DEVICE_CLASSES for ipad.
+        (IOSSimulatorPort.__init__):
+        (IOSSimulatorPort.simulator_device_type): Use a device name from the DEVICE_CLASS_MAP
+        based on the custom device class.
+        (IOSSimulatorPort._set_device_class):
+        (IOSSimulatorPort._create_simulators): Factor some code into this function.
+        (IOSSimulatorPort.setup_test_run):
+        (IOSSimulatorPort.testing_device):
+        (IOSSimulatorPort.reset_preferences): This used to create the simulator apps, but that
+        seemed wrong for this function. That was moved to setup_test_run().
+        (IOSSimulatorPort.check_sys_deps): This function used to create testing devices,
+        but this happened too early, before we knew which kind of devices to create. Devices
+        are now created in setup_test_run().
+        * Scripts/webkitpy/port/win.py:
+        (WinPort.setup_test_run):
+
 2016-08-15  Daniel Bates  <dabates@apple.com>
 
         Cannot build WebKit for iOS device using Xcode 7.3/iOS 9.3 public SDK due to missing
index 001cfdfcd8c8bc3662bd5bab83d9bf07bf6dbfda..0ab28c7640c09794e08168e68a8a9a0adc5341b1 100644 (file)
@@ -39,6 +39,7 @@ import logging
 import random
 import sys
 import time
+from collections import defaultdict
 
 from webkitpy.common.checkout.scm.detection import SCMDetector
 from webkitpy.common.net.file_uploader import FileUploader
@@ -97,6 +98,13 @@ class Manager(object):
     def _is_web_platform_test(self, test):
         return self.web_platform_test_subdir in test
 
+    def _custom_device_for_test(self, test):
+        for device_class in self._port.CUSTOM_DEVICE_CLASSES:
+            directory_suffix = device_class + self._port.TEST_PATH_SEPARATOR
+            if directory_suffix in test:
+                return device_class
+        return None
+
     def _http_tests(self, test_names):
         return set(test for test in test_names if self._is_http_test(test))
 
@@ -141,12 +149,14 @@ class Manager(object):
         worker_count = self._runner.get_worker_count(test_inputs, int(self._options.child_processes))
         self._options.child_processes = worker_count
 
-    def _set_up_run(self, test_names):
+    def _set_up_run(self, test_names, device_class=None):
         self._printer.write_update("Checking build ...")
         if not self._port.check_build(self.needs_servers(test_names)):
             _log.error("Build check failed")
             return False
 
+        self._options.device_class = device_class
+
         # This must be started before we check the system dependencies,
         # since the helper may do things to make the setup correct.
         self._printer.write_update("Starting helper ...")
@@ -169,7 +179,7 @@ class Manager(object):
         # Create the output directory if it doesn't already exist.
         self._port.host.filesystem.maybe_make_directory(self._results_directory)
 
-        self._port.setup_test_run()
+        self._port.setup_test_run(self._options.device_class)
         return True
 
     def run(self, args):
@@ -194,13 +204,56 @@ class Manager(object):
             _log.critical('No tests to run.')
             return test_run_results.RunDetails(exit_code=-1)
 
-        try:
+        default_device_tests = []
+
+        # Look for tests with custom device requirements.
+        custom_device_tests = defaultdict(list)
+        for test_file in tests_to_run:
+            custom_device = self._custom_device_for_test(test_file)
+            if custom_device:
+                custom_device_tests[custom_device].append(test_file)
+            else:
+                default_device_tests.append(test_file)
+
+        if custom_device_tests:
+            for device_class in custom_device_tests:
+                _log.debug('{} tests use device {}'.format(len(custom_device_tests[device_class]), device_class))
+
+        initial_results = None
+        retry_results = None
+        enabled_pixel_tests_in_retry = False
+
+        if default_device_tests:
+            _log.info('')
+            _log.info("Running %s", pluralize(len(tests_to_run), "test"))
+            _log.info('')
             if not self._set_up_run(tests_to_run):
                 return test_run_results.RunDetails(exit_code=-1)
 
+            initial_results, retry_results, enabled_pixel_tests_in_retry = self._run_test_subset(default_device_tests, tests_to_skip)
+
+        for device_class in custom_device_tests:
+            device_tests = custom_device_tests[device_class]
+            if device_tests:
+                _log.info('')
+                _log.info('Running %s for %s', pluralize(len(device_tests), "test"), device_class)
+                _log.info('')
+                if not self._set_up_run(device_tests, device_class):
+                    return test_run_results.RunDetails(exit_code=-1)
+
+                device_initial_results, device_retry_results, device_enabled_pixel_tests_in_retry = self._run_test_subset(device_tests, tests_to_skip)
+
+                initial_results = initial_results.merge(device_initial_results) if initial_results else device_initial_results
+                retry_results = retry_results.merge(device_retry_results) if retry_results else device_retry_results
+                enabled_pixel_tests_in_retry |= device_enabled_pixel_tests_in_retry
+
+        end_time = time.time()
+        return self._end_test_run(start_time, end_time, initial_results, retry_results, enabled_pixel_tests_in_retry)
+
+    def _run_test_subset(self, tests_to_run, tests_to_skip):
+        try:
             enabled_pixel_tests_in_retry = False
-            initial_results = self._run_tests(tests_to_run, tests_to_skip, self._options.repeat_each, self._options.iterations,
-                int(self._options.child_processes), retrying=False)
+            initial_results = self._run_tests(tests_to_run, tests_to_skip, self._options.repeat_each, self._options.iterations, int(self._options.child_processes), retrying=False)
 
             tests_to_retry = self._tests_to_retry(initial_results, include_crashes=self._port.should_retry_crashes())
             # Don't retry failures when interrupted by user or failures limit exception.
@@ -211,8 +264,7 @@ class Manager(object):
                 _log.info('')
                 _log.info("Retrying %s ..." % pluralize(len(tests_to_retry), "unexpected failure"))
                 _log.info('')
-                retry_results = self._run_tests(tests_to_retry, tests_to_skip=set(), repeat_each=1, iterations=1,
-                    num_workers=1, retrying=True)
+                retry_results = self._run_tests(tests_to_retry, tests_to_skip=set(), repeat_each=1, iterations=1, num_workers=1, retrying=True)
 
                 if enabled_pixel_tests_in_retry:
                     self._options.pixel_tests = False
@@ -221,10 +273,12 @@ class Manager(object):
         finally:
             self._clean_up_run()
 
-        end_time = time.time()
+        return (initial_results, retry_results, enabled_pixel_tests_in_retry)
 
+    def _end_test_run(self, start_time, end_time, initial_results, retry_results, enabled_pixel_tests_in_retry):
         # Some crash logs can take a long time to be written out so look
         # for new logs after the test run finishes.
+
         _log.debug("looking for new crash logs")
         self._look_for_new_crash_logs(initial_results, start_time)
         if retry_results:
@@ -259,6 +313,7 @@ class Manager(object):
         needs_websockets = any(self._is_websocket_test(test) for test in tests_to_run)
 
         test_inputs = self._get_test_inputs(tests_to_run, repeat_each, iterations)
+
         return self._runner.run_tests(self._expectations, test_inputs, tests_to_skip, num_workers, needs_http, needs_websockets, needs_web_platform_test_server, retrying)
 
     def _clean_up_run(self):
index 817a6cdfb50e3cd8269e20d403c6feb4ea9a2b98..6f8664b39332af5c936a723bb63ff319fdab08ba 100644 (file)
@@ -37,6 +37,7 @@ from webkitpy.common.host_mock import MockHost
 from webkitpy.layout_tests.controllers.manager import Manager
 from webkitpy.layout_tests.models import test_expectations
 from webkitpy.layout_tests.models.test_run_results import TestRunResults
+from webkitpy.port.test import TestPort
 from webkitpy.thirdparty.mock import Mock
 from webkitpy.tool.mocktool import MockOptions
 
@@ -99,3 +100,19 @@ class ManagerTest(unittest.TestCase):
         run_results = TestRunResults(expectations, len(tests))
         manager = get_manager()
         manager._look_for_new_crash_logs(run_results, time.time())
+
+    def test_uses_custom_device(self):
+        class MockCustomDevicePort(TestPort):
+            CUSTOM_DEVICE_CLASSES = ['starship']            
+            def __init__(self, host):
+                super(MockCustomDevicePort, self).__init__(host)
+
+        def get_manager():
+            host = MockHost()
+            port = MockCustomDevicePort(host)
+            manager = Manager(port, options=MockOptions(test_list=['fast/test-starship/lasers.html'], http=True), printer=Mock())
+            return manager
+
+        manager = get_manager()
+        self.assertTrue(manager._custom_device_for_test('fast/test-starship/lasers.html') == 'starship')
+        
index 79c6fc1b94b7b1117e9cb05d924b45566645a9a1..e8583524703e2bf386aed8dfdf666570de8f8a2e 100644 (file)
@@ -70,8 +70,7 @@ class SingleTestRunner(object):
             for suffix in ('.txt', '.png', '.wav'):
                 expected_filename = self._port.expected_filename(self._test_name, suffix)
                 if self._filesystem.exists(expected_filename):
-                    _log.error('%s is a reftest, but has an unused expectation file. Please remove %s.',
-                        self._test_name, expected_filename)
+                    _log.error('%s is a reftest, but has an unused expectation file. Please remove %s.', self._test_name, expected_filename)
 
     def _expected_driver_output(self):
         return DriverOutput(self._port.expected_text(self._test_name),
index f1396619934142c07e05a40cbfddc0d45e1e09e4..9e72ed03ac5e83e9f96d37a268f1bd833390601c 100644 (file)
@@ -93,6 +93,30 @@ class TestRunResults(object):
         if test_is_slow:
             self.slow_tests.add(test_result.test_name)
 
+    def merge(self, test_run_results):
+        # self.expectations should be the same for both
+        self.total += test_run_results.total
+        self.remaining += test_run_results.remaining
+        self.expected += test_run_results.expected
+        self.unexpected += test_run_results.unexpected
+        self.unexpected_failures += test_run_results.unexpected_failures
+        self.unexpected_crashes += test_run_results.unexpected_crashes
+        self.unexpected_timeouts += test_run_results.unexpected_timeouts
+        self.tests_by_expectation.update(test_run_results.tests_by_expectation)
+        self.tests_by_timeline.update(test_run_results.tests_by_timeline)
+        self.results_by_name.update(test_run_results.results_by_name)
+        self.all_results += test_run_results.all_results
+        self.unexpected_results_by_name.update(test_run_results.unexpected_results_by_name)
+        self.failures_by_name.update(test_run_results.failures_by_name)
+        self.total_failures += test_run_results.total_failures
+        self.expected_skips += test_run_results.expected_skips
+        self.tests_by_expectation.update(test_run_results.tests_by_expectation)
+        self.tests_by_timeline.update(test_run_results.tests_by_timeline)
+        self.slow_tests.update(test_run_results.slow_tests)
+
+        self.interrupted |= test_run_results.interrupted
+        self.keyboard_interrupted |= test_run_results.keyboard_interrupted
+        return self
 
 class RunDetails(object):
     def __init__(self, exit_code, summarized_results=None, initial_results=None, retry_results=None, enabled_pixel_tests_in_retry=False):
index 701cefe80590ff1b8f77357a172faf46ed8e3f1f..65c795718bc0f85dbff8cc557ac452bdc218efd7 100644 (file)
@@ -112,12 +112,14 @@ class Printer(object):
 
     def print_workers_and_shards(self, num_workers, num_shards):
         driver_name = self._port.driver_name()
+
+        device_suffix = ' for device "{}"'.format(self._options.device_class) if self._options.device_class else ''
         if num_workers == 1:
-            self._print_default("Running 1 %s." % driver_name)
-            self._print_debug("(%s)." % grammar.pluralize(num_shards, "shard"))
+            self._print_default('Running 1 {}{}.'.format(driver_name, device_suffix))
+            self._print_debug('({}).'.format(grammar.pluralize(num_shards, "shard")))
         else:
-            self._print_default("Running %s in parallel." % (grammar.pluralize(num_workers, driver_name)))
-            self._print_debug("(%d shards)." % num_shards)
+            self._print_default('Running {} in parallel{}.'.format(grammar.pluralize(num_workers, driver_name), device_suffix))
+            self._print_debug('({} shards).'.format(num_shards))
         self._print_default('')
 
     def _print_expected_results_of_type(self, run_results, result_type, result_type_str, tests_with_result_type_callback):
index a0483a0a91edebd5d899959e7fefa9332d4189ef..b645a53e9173005bd649d0b290ffb11032212cb2 100644 (file)
@@ -82,6 +82,8 @@ class Port(object):
 
     DEFAULT_ARCHITECTURE = 'x86'
 
+    CUSTOM_DEVICE_CLASSES = []
+
     @classmethod
     def determine_full_port_name(cls, host, options, port_name):
         """Return a fully-specified port name that can be used to construct objects."""
@@ -803,7 +805,7 @@ class Port(object):
         # to have multiple copies of webkit checked out and built.
         return self._build_path('layout-test-results')
 
-    def setup_test_run(self):
+    def setup_test_run(self, device_class=None):
         """Perform port-specific work at the beginning of a test run."""
         pass
 
index d649508bff9a621a86dd583d0cfd540603287512..7a7c9d9aa58c954d22c4b562ff87ceee7391e959 100644 (file)
@@ -52,6 +52,9 @@ class DriverInput(object):
         self.should_run_pixel_test = should_run_pixel_test
         self.args = args or []
 
+    def __repr__(self):
+        return "DriverInput(test_name='{}', timeout={}, image_hash={}, should_run_pixel_test={}'".format(self.test_name, self.timeout, self.image_hash, self.should_run_pixel_test)
+
 
 class DriverOutput(object):
     """Groups information about a output from driver for easy passing
@@ -587,6 +590,7 @@ class Driver(object):
         return True
 
 
+# FIXME: this should be abstracted out via the Port subclass somehow.
 class IOSSimulatorDriver(Driver):
     def cmd_line(self, pixel_tests, per_test_args):
         cmd = super(IOSSimulatorDriver, self).cmd_line(pixel_tests, per_test_args)
index a900b3c732642cc4b0b18990ceadfa3d5daeed34..556e329357f4a8a7aa00aa09dee485151a85987e 100644 (file)
@@ -54,8 +54,8 @@ class EflPort(Port):
     def _port_flag_for_scripts(self):
         return "--efl"
 
-    def setup_test_run(self):
-        super(EflPort, self).setup_test_run()
+    def setup_test_run(self, device_class=None):
+        super(EflPort, self).setup_test_run(device_class)
         self._pulseaudio_sanitizer.unload_pulseaudio_module()
 
     def setup_environ_for_server(self, server_name=None):
index c6302f18ae9917ba60b491343a17af94a91412c5..0333280d62a0531ef09ea9039e8170f58fb7647c 100644 (file)
@@ -100,8 +100,8 @@ class GtkPort(Port):
             return self.default_timeout_ms()
         return super(GtkPort, self).driver_stop_timeout()
 
-    def setup_test_run(self):
-        super(GtkPort, self).setup_test_run()
+    def setup_test_run(self, device_class=None):
+        super(GtkPort, self).setup_test_run(device_class)
         self._pulseaudio_sanitizer.unload_pulseaudio_module()
 
         if self.get_option("leaks"):
index b6c4d819dd72bcf52e00be5959e993fbfd506957..0985ab45b79998bd9330072b5c920c56d9575b13 100644 (file)
@@ -74,15 +74,33 @@ class IOSSimulatorPort(ApplePort):
     ARCHITECTURES = ['x86_64', 'x86']
     DEFAULT_ARCHITECTURE = 'x86_64'
 
+    DEFAULT_DEVICE_CLASS = 'iphone'
+    CUSTOM_DEVICE_CLASSES = ['ipad']
+
     SIMULATOR_BUNDLE_ID = 'com.apple.iphonesimulator'
     relay_name = 'LayoutTestRelay'
     SIMULATOR_DIRECTORY = "/tmp/WebKitTestingSimulators/"
     LSREGISTER_PATH = "/System/Library/Frameworks/CoreServices.framework/Frameworks/LaunchServices.framework/Versions/Current/Support/lsregister"
     PROCESS_COUNT_ESTIMATE_PER_SIMULATOR_INSTANCE = 100
 
+    DEVICE_CLASS_MAP = {
+        'x86_64': {
+            'iphone': 'iPhone 5s',
+            'ipad': 'iPad Air'
+        },
+        'x86': {
+            'iphone': 'iPhone 5',
+            'ipad': 'iPad Retina'
+        },
+    }
+
     def __init__(self, host, port_name, **kwargs):
         super(IOSSimulatorPort, self).__init__(host, port_name, **kwargs)
 
+        optional_device_class = self.get_option('device_class')
+        self._device_class = optional_device_class if optional_device_class else self.DEFAULT_DEVICE_CLASS
+        _log.debug('IOSSimulatorPort _device_class is %s', self._device_class)
+
     def driver_name(self):
         if self.get_option('driver_name'):
             return self.get_option('driver_name')
@@ -100,17 +118,17 @@ class IOSSimulatorPort(ApplePort):
             runtime = Runtime.from_version_string(self.host.platform.xcode_sdk_version('iphonesimulator'))
         return runtime
 
-    @property
-    @memoized
     def simulator_device_type(self):
         device_type_identifier = self.get_option('device_type')
         if device_type_identifier:
+            _log.debug('simulator_device_type for device identifier %s', device_type_identifier)
             device_type = DeviceType.from_identifier(device_type_identifier)
         else:
-            if self.architecture() == 'x86_64':
-                device_type = DeviceType.from_name('iPhone 5s')
-            else:
-                device_type = DeviceType.from_name('iPhone 5')
+            _log.debug('simulator_device_type for device %s', self._device_class)
+            device_name = self.DEVICE_CLASS_MAP[self.architecture()][self._device_class]
+            if not device_name:
+                raise Exception('Failed to find device for architecture {} and device class {}'.format(self.architecture()), self._device_class)
+            device_type = DeviceType.from_name(device_name)
         return device_type
 
     @property
@@ -205,12 +223,37 @@ class IOSSimulatorPort(ApplePort):
     def _port_specific_expectations_files(self):
         return list(reversed([self._filesystem.join(self._webkit_baseline_path(p), 'TestExpectations') for p in self.baseline_search_path()]))
 
-    def setup_test_run(self):
+    def _set_device_class(self, device_class):
+        # Ideally we'd ensure that no simulators are running when this is called.
+        self._device_class = device_class if device_class else self.DEFAULT_DEVICE_CLASS
+
+    def _create_simulators(self):
+        if (self.default_child_processes() < self.child_processes()):
+                _log.warn("You have specified very high value({0}) for --child-processes".format(self.child_processes()))
+                _log.warn("maximum child-processes which can be supported on this system are: {0}".format(self.default_child_processes()))
+                _log.warn("This is very likely to fail.")
+
+        self._createSimulatorApps()
+
+        for i in xrange(self.child_processes()):
+            Simulator.wait_until_device_is_in_state(self.testing_device(i).udid, Simulator.DeviceState.SHUTDOWN)
+            Simulator.reset_device(self.testing_device(i).udid)
+
+    def setup_test_run(self, device_class=None):
         mac_os_version = self.host.platform.os_version
+
+        self._set_device_class(device_class)
+
+        _log.debug('')
+        _log.debug('setup_test_run for %s', self._device_class)
+
+        self._create_simulators()
+
         for i in xrange(self.child_processes()):
             device_udid = self.testing_device(i).udid
-            # FIXME: <rdar://problem/20916140> Switch to using CoreSimulator.framework for launching and quitting iOS Simulator
+            _log.debug('testing device %s has udid %s', i, device_udid)
 
+            # FIXME: <rdar://problem/20916140> Switch to using CoreSimulator.framework for launching and quitting iOS Simulator
             self._executive.run_command([
                 'open', '-g', '-b', self.SIMULATOR_BUNDLE_ID + str(i),
                 '--args', '-CurrentDeviceUDID', device_udid])
@@ -284,9 +327,6 @@ class IOSSimulatorPort(ApplePort):
         if not self.simulator_runtime.available:
             _log.error('The iOS Simulator runtime with identifier "{0}" cannot be used because it is unavailable.'.format(self.simulator_runtime.identifier))
             return False
-        for i in xrange(self.child_processes()):
-            # FIXME: This creates the devices sequentially, doing this in parallel can improve performance.
-            testing_device = self.testing_device(i)
         return super(IOSSimulatorPort, self).check_sys_deps(needs_http)
 
     SUBPROCESS_CRASH_REGEX = re.compile('#CRASHED - (?P<subprocess_name>\S+) \(pid (?P<subprocess_pid>\d+)\)')
@@ -329,9 +369,12 @@ class IOSSimulatorPort(ApplePort):
             return stderr, None
         return stderr, crash_log
 
-    @memoized
     def testing_device(self, number):
-        return Simulator().lookup_or_create_device(self.simulator_device_type.name + ' WebKit Tester' + str(number), self.simulator_device_type, self.simulator_runtime)
+        # FIXME: rather than calling lookup_or_create_device every time, we should just store a mapping of
+        # number to device_udid.
+        device_type = self.simulator_device_type()
+        _log.debug(' testing_device %s using device_type %s', number, device_type)
+        return Simulator().lookup_or_create_device(device_type.name + ' WebKit Tester' + str(number), device_type, self.simulator_runtime)
 
     def get_simulator_path(self, suffix=""):
         return os.path.join(self.SIMULATOR_DIRECTORY, "Simulator" + str(suffix) + ".app")
@@ -350,17 +393,8 @@ class IOSSimulatorPort(ApplePort):
 
     def reset_preferences(self):
         _log.debug("reset_preferences")
-        if (self.default_child_processes() < self.child_processes()):
-                _log.warn("You have specified very high value({0}) for --child-processes".format(self.child_processes()))
-                _log.warn("maximum child-processes which can be supported on this system are: {0}".format(self.default_child_processes()))
-                _log.warn("This is very likely to fail.")
-
         self._quit_ios_simulator()
-        self._createSimulatorApps()
-
-        for i in xrange(self.child_processes()):
-            Simulator.wait_until_device_is_in_state(self.testing_device(i).udid, Simulator.DeviceState.SHUTDOWN)
-            Simulator.reset_device(self.testing_device(i).udid)
+        # Maybe this should delete all devices that we've created?
 
     def nm_command(self):
         return self.xcrun_find('nm')
index 83f858fedd2024b262546ff837df6ea8584ff9f3..ab31d9323d09c29c0d7296c518f15cd88817a81f 100644 (file)
@@ -457,7 +457,7 @@ class TestPort(Port):
     def default_results_directory(self):
         return '/tmp/layout-test-results'
 
-    def setup_test_run(self):
+    def setup_test_run(self, device_class=None):
         pass
 
     def _driver_class(self):
index 713446c7bbd56443eb88f667ad5bfe13b62c56b9..fbcc33893f17146d8fe4c706d5a9bc1aafe668ab 100644 (file)
@@ -345,12 +345,12 @@ class WinPort(ApplePort):
     def delete_sem_locks(self):
         os.system("rm -rf /dev/shm/sem.*")
 
-    def setup_test_run(self):
+    def setup_test_run(self, device_class=None):
         atexit.register(self.restore_crash_log_saving)
         self.setup_crash_log_saving()
         self.prevent_error_dialogs()
         self.delete_sem_locks()
-        super(WinPort, self).setup_test_run()
+        super(WinPort, self).setup_test_run(device_class)
 
     def clean_up_test_run(self):
         self.allow_error_dialogs()