Create iOS-EWS client queue
authorddkilzer@apple.com <ddkilzer@apple.com@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Sat, 17 Jan 2015 08:42:20 +0000 (08:42 +0000)
committerddkilzer@apple.com <ddkilzer@apple.com@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Sat, 17 Jan 2015 08:42:20 +0000 (08:42 +0000)
https://bugs.webkit.org/show_bug.cgi?id=140476

Patch by Jake Nielsen <jacob_nielsen@apple.com> on 2015-01-17
Reviewed by Alexey Proskuryakov.

* EWSTools/start-queue-mac.sh:
Adds call to configure-xcode-for-ios-development for iOS EWS.
* Scripts/webkitpy/common/config/ews.json:
Adds the iOS EWS config entry.
* Scripts/webkitpy/common/config/ports.py:
(DeprecatedPort.port):
Adds IOSPort to the port map.
(IOSPort):
(IOSPort.build_webkit_command):
Adds --sdk=iphoneos to the webkit build command.
* Scripts/webkitpy/port/base.py:
(Port.architecture):
Adds an accessor for the architecture member variable.
* Scripts/webkitpy/port/factory.py:
(PortFactory):
Adds IOSPort to the port class list.
* Scripts/webkitpy/port/ios.py:
Adds the IOSPort to encapsulate iOS-specific things.
(IOSPort):
(IOSPort.determine_full_port_name):
(IOSPort.__init__):
(IOSPort._build_driver_flags):
(IOSPort.operating_system):
* Scripts/webkitpy/tool/commands/download_unittest.py:
Adds the archetecture option to the mock options.
(DownloadCommandsTest._default_options):
* Scripts/webkitpy/tool/commands/earlywarningsystem.py:
Appends the archetecture option into calls to webkit-patch.
(AbstractEarlyWarningSystem.review_patch):
(AbstractEarlyWarningSystem.run_command):
(AbstractEarlyWarningSystem.load_ews_classes):
* Scripts/webkitpy/tool/commands/earlywarningsystem_unittest.py:
Adds logic to soothe the unittest now that --architecture is being
thrown around everywhere.
(EarlyWarningSystemTest._default_expected_logs):
* Scripts/webkitpy/tool/commands/queues.py:
Adds glue code to ferry the --architecture option around.
(AbstractQueue.__init__):
(PatchProcessingQueue.begin_work_queue):
* Scripts/webkitpy/tool/steps/abstractstep.py:
Adds the archetecture option to all AbstractSteps so that
EarlyWarningSystem doesn't start shoving --architecture into
unsuspecting commands.
(AbstractStep.options):
* Scripts/webkitpy/tool/steps/build.py:
Translates the --architecture option into an ARCHS= format to pass to
build-webkit.
(Build.build):
* Scripts/webkitpy/tool/steps/options.py:
Adds a definition for the --architecture option.
(Options):

git-svn-id: https://svn.webkit.org/repository/webkit/trunk@178622 268f45cc-cd09-0410-ab3c-d52691b4dbfc

14 files changed:
Tools/ChangeLog
Tools/EWSTools/start-queue-mac.sh
Tools/Scripts/webkitpy/common/config/ews.json
Tools/Scripts/webkitpy/common/config/ports.py
Tools/Scripts/webkitpy/port/base.py
Tools/Scripts/webkitpy/port/factory.py
Tools/Scripts/webkitpy/port/ios.py
Tools/Scripts/webkitpy/tool/commands/download_unittest.py
Tools/Scripts/webkitpy/tool/commands/earlywarningsystem.py
Tools/Scripts/webkitpy/tool/commands/earlywarningsystem_unittest.py
Tools/Scripts/webkitpy/tool/commands/queues.py
Tools/Scripts/webkitpy/tool/steps/abstractstep.py
Tools/Scripts/webkitpy/tool/steps/build.py
Tools/Scripts/webkitpy/tool/steps/options.py

index eb00e1a..c05b090 100644 (file)
@@ -1,3 +1,62 @@
+2015-01-17  Jake Nielsen  <jacob_nielsen@apple.com>
+
+        Create iOS-EWS client queue
+        https://bugs.webkit.org/show_bug.cgi?id=140476
+
+        Reviewed by Alexey Proskuryakov.
+
+        * EWSTools/start-queue-mac.sh:
+        Adds call to configure-xcode-for-ios-development for iOS EWS.
+        * Scripts/webkitpy/common/config/ews.json:
+        Adds the iOS EWS config entry.
+        * Scripts/webkitpy/common/config/ports.py:
+        (DeprecatedPort.port):
+        Adds IOSPort to the port map.
+        (IOSPort):
+        (IOSPort.build_webkit_command):
+        Adds --sdk=iphoneos to the webkit build command.
+        * Scripts/webkitpy/port/base.py:
+        (Port.architecture):
+        Adds an accessor for the architecture member variable.
+        * Scripts/webkitpy/port/factory.py:
+        (PortFactory):
+        Adds IOSPort to the port class list.
+        * Scripts/webkitpy/port/ios.py:
+        Adds the IOSPort to encapsulate iOS-specific things.
+        (IOSPort):
+        (IOSPort.determine_full_port_name):
+        (IOSPort.__init__):
+        (IOSPort._build_driver_flags):
+        (IOSPort.operating_system):
+        * Scripts/webkitpy/tool/commands/download_unittest.py:
+        Adds the archetecture option to the mock options.
+        (DownloadCommandsTest._default_options):
+        * Scripts/webkitpy/tool/commands/earlywarningsystem.py:
+        Appends the archetecture option into calls to webkit-patch.
+        (AbstractEarlyWarningSystem.review_patch):
+        (AbstractEarlyWarningSystem.run_command):
+        (AbstractEarlyWarningSystem.load_ews_classes):
+        * Scripts/webkitpy/tool/commands/earlywarningsystem_unittest.py:
+        Adds logic to soothe the unittest now that --architecture is being
+        thrown around everywhere.
+        (EarlyWarningSystemTest._default_expected_logs):
+        * Scripts/webkitpy/tool/commands/queues.py:
+        Adds glue code to ferry the --architecture option around.
+        (AbstractQueue.__init__):
+        (PatchProcessingQueue.begin_work_queue):
+        * Scripts/webkitpy/tool/steps/abstractstep.py:
+        Adds the archetecture option to all AbstractSteps so that
+        EarlyWarningSystem doesn't start shoving --architecture into
+        unsuspecting commands.
+        (AbstractStep.options):
+        * Scripts/webkitpy/tool/steps/build.py:
+        Translates the --architecture option into an ARCHS= format to pass to
+        build-webkit.
+        (Build.build):
+        * Scripts/webkitpy/tool/steps/options.py:
+        Adds a definition for the --architecture option.
+        (Options):
+
 2015-01-16  Sam Weinig  <sam@webkit.org>
 
         Add a basic configuration object for WKView
index 040651d..8828513 100755 (executable)
@@ -1,6 +1,6 @@
 #!/bin/sh
 # Copyright (c) 2012 Google Inc. All rights reserved.
-# Copyright (c) 2014 Apple Inc. All rights reserved.
+# Copyright (c) 2014, 2015 Apple Inc. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
@@ -39,20 +39,28 @@ RESET_AFTER_ITERATION=$3
 shift 3
 QUEUE_PARAMS="$@"
 
+EWS_HOME=/Volumes/Data/EWS
+WEBKIT_HOME=$EWS_HOME/WebKit
+
+# If building for iOS, make sure we run this script to make it possible to build frameworks.
+if [ "$QUEUE_NAME" == "ios-ews" ]; then
+    (cd $WEBKIT_HOME; sudo ./Tools/Scripts/configure-xcode-for-ios-development)
+fi
+
 # We reboot every night between 1 and 6 to reduce the likelihood of unexpected reboots while people are looking into things.
 TIME_TO_REBOOT=$(( $(date +%s) + 3600 * 12))
 
 while [ $TIME_TO_REBOOT -gt $(date +%s) ] || [ $(date +%H) -lt 1 ] || [ $(date +%H) -ge 6 ]; do
-    # Delete log files older than 14 days, move aside the main mac-ews.log file to prevent it from growing extra large.
-    cd /Volumes/Data/EWS/$QUEUE_NAME-logs
+    # Delete log files older than 14 days, move aside the main $QUEUE_NAME-ews.log file to prevent it from growing extra large.
+    cd $EWS_HOME/$QUEUE_NAME-logs
     find . -mtime +14 -delete
     if [ -s $QUEUE_NAME.log ]; then
         mv -f $QUEUE_NAME.log ${QUEUE_NAME}_$(date +%Y-%m-%d_%H-%m).log
     fi
-    cd /Volumes/Data/EWS/Webkit
+    cd $WEBKIT_HOME
     
     # Delete WebKitBuild to force a clean build
-    rm -rf /Volumes/Data/EWS/WebKit/WebKitBuild
+    rm -rf $WEBKIT_HOME/WebKitBuild
     
     # This somewhat quirky sequence of steps seems to clear up all the broken
     # git situations we've gotten ourself into in the past.
index bfa4bc6..b2ecda3 100644 (file)
     "Win EWS": {
         "port": "win"
     },
+    "iOS EWS": {
+        "port": "ios-device",
+        "architecture": "armv7"
+    },
     "Mac EWS": {
         "port": "mac",
         "watchers": [
index b70b83c..042e896 100644 (file)
@@ -1,5 +1,6 @@
 # Copyright (C) 2009, Google Inc. All rights reserved.
 # Copyright (C) 2013 Nokia Corporation and/or its subsidiary(-ies).
+# Copyright (C) 2015 Apple Inc. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
@@ -63,6 +64,7 @@ class DeprecatedPort(object):
     def port(port_name):
         ports = {
             "gtk-wk2": GtkWK2Port,
+            "ios-device": IOSPort,
             "mac": MacPort,
             "mac-wk2": MacWK2Port,
             "win": WinPort,
@@ -118,6 +120,15 @@ class DeprecatedPort(object):
         return self.script_shell_command("run-bindings-tests")
 
 
+class IOSPort(DeprecatedPort):
+    port_flag_name = "ios-device"
+
+    def build_webkit_command(self, build_style=None):
+        command = super(IOSPort, self).build_webkit_command(build_style=build_style)
+        command.append("--sdk=iphoneos")
+        return command
+
+
 class MacPort(DeprecatedPort):
     port_flag_name = "mac"
 
index c9e849d..a6d7330 100644 (file)
@@ -127,6 +127,9 @@ class Port(object):
         self._root_was_set = hasattr(options, 'root') and options.root
         self._jhbuild_wrapper = []
 
+    def architecture(self):
+        return self._architecture
+
     def additional_drt_flag(self):
         return []
 
index db7ab4a..542379c 100644 (file)
@@ -83,6 +83,7 @@ class PortFactory(object):
     PORT_CLASSES = (
         'efl.EflPort',
         'gtk.GtkPort',
+        'ios.IOSPort',
         'ios.IOSSimulatorPort',
         'mac.MacPort',
         'mock_drt.MockDRTPort',
index 9a60532..77adb7c 100644 (file)
@@ -1,4 +1,4 @@
-# Copyright (C) 2014 Apple Inc. All rights reserved.
+# Copyright (C) 2014, 2015 Apple Inc. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions
@@ -31,6 +31,7 @@ import subprocess
 from webkitpy.layout_tests.models.test_configuration import TestConfiguration
 from webkitpy.common.system.crashlogs import CrashLogs
 from webkitpy.common.system.executive import ScriptError
+from webkitpy.port.apple import ApplePort
 from webkitpy.port import driver, image_diff
 from webkitpy.port.base import Port
 from webkitpy.port.leakdetector import LeakDetector
@@ -41,6 +42,42 @@ from webkitpy.xcode import simulator
 _log = logging.getLogger(__name__)
 
 
+class IOSPort(ApplePort):
+    port_name = "ios-device"
+
+    ARCHITECTURES = ['armv7', 'armv7s', 'arm64']
+    VERSION_FALLBACK_ORDER = ['ios-device-8']
+
+    @classmethod
+    def determine_full_port_name(cls, host, options, port_name):
+        if port_name == cls.port_name:
+            sdk_command_process = subprocess.Popen('xcrun --sdk iphoneos --show-sdk-version', stdout=subprocess.PIPE, stderr=None, shell=True)
+            sdk_command_stdout = sdk_command_process.communicate()[0].strip()
+
+            assert sdk_command_stdout, "Xcode is not installed, and hence we cannot construct an iOS port object!"
+
+            port_name = port_name + '-' + re.match('^([0-9]+).*', sdk_command_stdout).group(1)
+
+        return port_name
+
+    def __init__(self, *args, **kwargs):
+        super(IOSPort, self).__init__(*args, **kwargs)
+
+        self._architecture = self.get_option('architecture')
+
+        if not self._architecture:
+            self._architecture = 'armv7'
+
+        self._testing_device = None
+
+    # Despite their names, these flags do not actually get passed all the way down to webkit-build.
+    def _build_driver_flags(self):
+        return ['--sdk', 'iphoneos'] + (['ARCHS=%s' % self._architecture] if self._architecture else [])
+
+    def operating_system(self):
+        return 'ios'
+
+
 class IOSSimulatorPort(Port):
     port_name = "ios-simulator"
 
index 59b52ea..6c8cd54 100644 (file)
@@ -87,6 +87,7 @@ class DownloadCommandsTest(CommandsTest):
         options.quiet = False
         options.test = True
         options.update = True
+        options.architecture = 'MOCK ARCH'
         return options
 
     def test_build(self):
@@ -162,7 +163,7 @@ MOCK run_command: ['ruby', '-I', '/mock-checkout/Websites/bugs.webkit.org/Pretty
 MOCK: user.open_url: file://...
 Was that diff correct?
 Building WebKit
-MOCK run_and_throw_if_fail: ['mock-build-webkit'], cwd=/mock-checkout, env={'LC_ALL': 'C', 'TERM': 'none', 'MOCK_ENVIRON_COPY': '1'}
+MOCK run_and_throw_if_fail: ['mock-build-webkit', 'ARCHS=MOCK ARCH'], cwd=/mock-checkout, env={'LC_ALL': 'C', 'TERM': 'none', 'MOCK_ENVIRON_COPY': '1'}
 Running Python unit tests
 MOCK run_and_throw_if_fail: ['mock-test-webkitpy'], cwd=/mock-checkout
 Running Perl unit tests
index 34b67c7..75250d8 100644 (file)
@@ -108,7 +108,7 @@ class AbstractEarlyWarningSystem(AbstractReviewQueue, EarlyWarningSystemTaskDele
         return self.name
 
     def run_command(self, command):
-        self.run_webkit_patch(command + [self._deprecated_port.flag()])
+        self.run_webkit_patch(command + [self._deprecated_port.flag()] + (['--architecture=%s' % self._port.architecture()] if self._port.architecture() else []))
 
     def command_passed(self, message, patch):
         pass
@@ -153,6 +153,7 @@ class AbstractEarlyWarningSystem(AbstractReviewQueue, EarlyWarningSystemTaskDele
             classes.append(type(str(name.replace(' ', '')), (AbstractEarlyWarningSystem,), {
                 'name': config['port'] + '-ews',
                 'port_name': config['port'],
+                'architecture': config.get('architecture', None),
                 'watchers': config.get('watchers', []),
                 'run_tests': config.get('runTests', cls.run_tests),
             }))
index c6de13a..6f7d4ab 100644 (file)
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 from webkitpy.thirdparty.mock import Mock
+from webkitpy.common.host import Host
 from webkitpy.common.net.layouttestresults import LayoutTestResults
 from webkitpy.common.system.outputcapture import OutputCapture
 from webkitpy.layout_tests.models import test_results
 from webkitpy.layout_tests.models import test_failures
+from webkitpy.port.factory import PortFactory
 from webkitpy.tool.bot.queueengine import QueueEngine
 from webkitpy.tool.commands.earlywarningsystem import *
+from webkitpy.tool.commands.queues import PatchProcessingQueue
 from webkitpy.tool.commands.queuestest import QueuesTest
 from webkitpy.tool.mocktool import MockTool, MockOptions
 
@@ -57,22 +60,26 @@ class AbstractEarlyWarningSystemTest(QueuesTest):
 
 class EarlyWarningSystemTest(QueuesTest):
     def _default_expected_logs(self, ews):
+        host = Host()
+        real_port_name = PatchProcessingQueue()._new_port_name_from_old(ews.port_name, host.platform)
+        real_port = Host().port_factory.get(real_port_name)
         string_replacements = {
             "name": ews.name,
             "port": ews.port_name,
+            "architecture": real_port.architecture(),
         }
         if ews.run_tests:
-            run_tests_line = "Running: webkit-patch --status-host=example.com build-and-test --no-clean --no-update --test --non-interactive --port=%(port)s\n" % string_replacements
+            run_tests_line = "Running: webkit-patch --status-host=example.com build-and-test --no-clean --no-update --test --non-interactive --port=%(port)s --architecture=%(architecture)s\n" % string_replacements
         else:
             run_tests_line = ""
         string_replacements['run_tests_line'] = run_tests_line
 
         expected_logs = {
             "begin_work_queue": self._default_begin_work_queue_logs(ews.name),
-            "process_work_item": """Running: webkit-patch --status-host=example.com clean --port=%(port)s
-Running: webkit-patch --status-host=example.com update --port=%(port)s
-Running: webkit-patch --status-host=example.com apply-attachment --no-update --non-interactive 10000 --port=%(port)s
-Running: webkit-patch --status-host=example.com build --no-clean --no-update --build-style=release --port=%(port)s
+            "process_work_item": """Running: webkit-patch --status-host=example.com clean --port=%(port)s --architecture=%(architecture)s
+Running: webkit-patch --status-host=example.com update --port=%(port)s --architecture=%(architecture)s
+Running: webkit-patch --status-host=example.com apply-attachment --no-update --non-interactive 10000 --port=%(port)s --architecture=%(architecture)s
+Running: webkit-patch --status-host=example.com build --no-clean --no-update --build-style=release --port=%(port)s --architecture=%(architecture)s
 %(run_tests_line)sMOCK: update_status: %(name)s Pass
 MOCK: release_work_item: %(name)s 10000
 """ % string_replacements,
index 1ac43e9..2a62415 100644 (file)
@@ -29,6 +29,7 @@
 
 import codecs
 import logging
+import optparse
 import os
 import re
 import sys
@@ -74,6 +75,7 @@ class AbstractQueue(Command, QueueEngineDelegate):
         self.help_text = "Run the %s" % self.name
         Command.__init__(self, options=options_list)
         self._iteration_count = 0
+        self.architecture = None
 
     def _cc_watchers(self, bug_id):
         try:
@@ -275,7 +277,10 @@ class PatchProcessingQueue(AbstractPatchQueue):
         self._deprecated_port = DeprecatedPort.port(self.port_name)
         # FIXME: This violates abstraction
         self._tool._deprecated_port = self._deprecated_port
-        self._port = self._tool.port_factory.get(self._new_port_name_from_old(self.port_name, self._tool.platform))
+        port_options = optparse.Values()
+        if self.architecture:
+            setattr(port_options, 'architecture', self.architecture)
+        self._port = self._tool.port_factory.get(self._new_port_name_from_old(self.port_name, self._tool.platform), port_options)
 
     def _upload_results_archive_for_patch(self, patch, results_archive_zip):
         if not self._port:
index 2a5fea6..7306b69 100644 (file)
@@ -72,6 +72,7 @@ class AbstractStep(object):
         return [
             # We need this option here because cached_lookup uses it.  :(
             Options.git_commit,
+            Options.architecture,
         ]
 
     def run(self, state):
index b66a35f..0356b40 100644 (file)
@@ -50,6 +50,10 @@ class Build(AbstractStep):
         env = environment.to_dictionary()
 
         build_webkit_command = self._tool.deprecated_port().build_webkit_command(build_style=build_style)
+
+        if self._options.architecture:
+            build_webkit_command += ['ARCHS=%s' % self._options.architecture]
+
         self._tool.executive.run_and_throw_if_fail(build_webkit_command, self._options.quiet,
             cwd=self._tool.scm().checkout_root, env=env)
 
index 7eda614..dc5fe2c 100644 (file)
@@ -29,6 +29,7 @@
 from optparse import make_option
 
 class Options(object):
+    architecture = make_option("--architecture", action="store", type="string", dest="architecture", default=None, help="Target architecture")
     blocks = make_option("--blocks", action="store", type="string", dest="blocks", default=None, help="Bug number which the created bug blocks.")
     build = make_option("--build", action="store_true", dest="build", default=False, help="Build and run run-webkit-tests before committing.")
     build_style = make_option("--build-style", action="store", dest="build_style", default=None, help="Whether to build debug, release, or both.")