[ews-build] Add support for compiling WebKit
authoraakash_jain@apple.com <aakash_jain@apple.com@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Wed, 27 Jun 2018 01:07:27 +0000 (01:07 +0000)
committeraakash_jain@apple.com <aakash_jain@apple.com@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Wed, 27 Jun 2018 01:07:27 +0000 (01:07 +0000)
https://bugs.webkit.org/show_bug.cgi?id=187019

Reviewed by Lucas Forschler.

* BuildSlaveSupport/ews-build/steps.py:
(CompileWebKit): Added, class to compile WebKit.
(CleanBuild): Added, class to clean up the build.
(KillOldProcesses): Added, class to kill old processes.
* BuildSlaveSupport/ews-build/steps_unittest.py: Added unit-tests.
* BuildSlaveSupport/ews-build/config.json: Renamed ios-11-simulator to ios-simulator-11, to match with build.webkit.org code.
Added configuration, architecture which is required for compiling. Renamed Release to release as the tools expect lower-case release.
* BuildSlaveSupport/ews-build/factories.py: Added BuildFactory.
* BuildSlaveSupport/ews-build/loadConfig.py: Renamed configuraton value to lower-case as tools expect lower-case values.
* BuildSlaveSupport/ews-build/loadConfig_unittest.py: Ditto.

git-svn-id: https://svn.webkit.org/repository/webkit/trunk@233235 268f45cc-cd09-0410-ab3c-d52691b4dbfc

Tools/BuildSlaveSupport/ews-build/config.json
Tools/BuildSlaveSupport/ews-build/factories.py
Tools/BuildSlaveSupport/ews-build/loadConfig.py
Tools/BuildSlaveSupport/ews-build/loadConfig_unittest.py
Tools/BuildSlaveSupport/ews-build/steps.py
Tools/BuildSlaveSupport/ews-build/steps_unittest.py
Tools/ChangeLog

index ca8d922..ea005eb 100644 (file)
     },
     {
       "name": "ews123",
-      "platform": "ios-11-simulator"
+      "platform": "ios-simulator-11"
     },
     {
       "name": "ews124",
-      "platform": "ios-11-simulator"
+      "platform": "ios-simulator-11"
     },
     {
       "name": "ews125",
-      "platform": "ios-11-simulator"
+      "platform": "ios-simulator-11"
     },
     {
       "name": "ews126",
-      "platform": "ios-11-simulator"
+      "platform": "ios-simulator-11"
     },
     {
       "name": "ews127",
       "name": "iOS-11-EWS",
       "factory": "iOSFactory",
       "platform": "ios-11",
+      "configuration": "release",
+      "architectures": ["arm64"],
       "workernames": ["ews108", "ews109"]
     },
     {
       "name": "iOS-11-Simulator-EWS",
       "factory": "iOSSimulatorFactory",
-      "platform": "ios-11-simulator",
+      "platform": "ios-simulator-11",
+      "configuration": "release",
+      "architectures": ["x86_64"],
       "workernames": ["ews123", "ews124", "ews125", "ews126"]
     },
     {
       "name": "macOS-Sierra-Release-WK1-EWS",
       "factory": "MacWK1Factory",
       "platform": "mac-sierra",
-      "configuration": "Release",
+      "configuration": "release",
+      "architectures": ["x86_64"],
       "workernames": ["ews100", "ews101", "ews102", "ews103"]
     },
     {
       "name": "macOS-Sierra-Release-WK2-EWS",
       "factory": "MacWK2Factory",
       "platform": "mac-sierra",
-      "configuration": "Release",
+      "configuration": "release",
+      "architectures": ["x86_64"],
       "workernames": ["ews104", "ews105", "ews106", "ews107"]
     },
     {
       "name": "macOS-Sierra-Debug-WK1-EWS",
       "factory": "MacWK1Factory",
       "platform": "mac-sierra",
-      "configuration": "Debug",
+      "configuration": "debug",
+      "architectures": ["x86_64"],
       "workernames": ["ews112", "ews113", "ews114", "ews115", "ews116"]
     },
     {
       "name": "macOS-High-Sierra-Release-32bit-WK2-EWS",
       "factory": "MacWK2Factory",
       "platform": "mac-high-sierra",
-      "configuration": "Release",
+      "configuration": "release",
+      "architectures": ["i386"],
       "workernames": ["ews118", "ews119", "ews120"]
     },
     {
index ecf8a9c..0d89c35 100644 (file)
@@ -51,23 +51,31 @@ class WebKitPerlFactory(Factory):
         self.addStep(RunWebKitPerlTests())
 
 
+class BuildFactory(Factory):
+    def __init__(self, platform, configuration=None, architectures=None, additionalArguments=None, **kwargs):
+        Factory.__init__(self, platform, configuration, architectures, False, additionalArguments)
+        self.addStep(KillOldProcesses())
+        self.addStep(CleanBuild())
+        self.addStep(CompileWebKit())
+
+
 class GTKFactory(Factory):
     pass
 
 
-class iOSFactory(Factory):
+class iOSFactory(BuildFactory):
     pass
 
 
-class iOSSimulatorFactory(Factory):
+class iOSSimulatorFactory(BuildFactory):
     pass
 
 
-class MacWK1Factory(Factory):
+class MacWK1Factory(BuildFactory):
     pass
 
 
-class MacWK2Factory(Factory):
+class MacWK2Factory(BuildFactory):
     pass
 
 
index 71b1492..3d80d79 100644 (file)
@@ -106,7 +106,7 @@ def checkValidBuilder(builder):
     if len(builder['name']) > BUILDER_NAME_LENGTH_LIMIT:
         raise Exception('Builder name {} is longer than maximum allowed by Buildbot ({} characters).'.format(builder['name'], BUILDER_NAME_LENGTH_LIMIT))
 
-    if 'configuration' in builder and builder['configuration'] not in ['Debug', 'Production', 'Release']:
+    if 'configuration' in builder and builder['configuration'] not in ['debug', 'production', 'release']:
         raise Exception('Invalid configuration: {} for builder: {}'.format(builder.get('configuration'), builder.get('name')))
 
     if not builder.get('factory'):
index d3815fd..2757bc1 100644 (file)
@@ -114,16 +114,16 @@ class TestcheckValidBuilder(unittest.TestCase):
 
     def test_builder_with_missing_factory(self):
         with self.assertRaises(Exception) as context:
-            loadConfig.checkValidBuilder({'name': 'mac-wk2', 'configuration': 'Release'})
+            loadConfig.checkValidBuilder({'name': 'mac-wk2', 'configuration': 'release'})
         self.assertEqual(context.exception.args, ('Builder mac-wk2 does not have factory defined.',))
 
     def test_builder_with_missing_platform(self):
         with self.assertRaises(Exception) as context:
-            loadConfig.checkValidBuilder({'name': 'mac-wk2', 'configuration': 'Release', 'factory': 'WK2Factory'})
+            loadConfig.checkValidBuilder({'name': 'mac-wk2', 'configuration': 'release', 'factory': 'WK2Factory'})
         self.assertEqual(context.exception.args, ('Builder mac-wk2 does not have platform defined.',))
 
     def test_valid_builder(self):
-        loadConfig.checkValidBuilder({'name': 'mac-wk2', 'configuration': 'Release', 'factory': 'WK2Factory', 'platform': 'mac-sierra'})
+        loadConfig.checkValidBuilder({'name': 'mac-wk2', 'configuration': 'release', 'factory': 'WK2Factory', 'platform': 'mac-sierra'})
 
 
 class TestcheckWorkersAndBuildersForConsistency(unittest.TestCase):
index e48014c..cc28b0d 100644 (file)
 # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-from buildbot.process import buildstep
+from buildbot.process import buildstep, properties
 from buildbot.process.results import Results, SUCCESS, FAILURE, WARNINGS, SKIPPED, EXCEPTION, RETRY
 from buildbot.steps import shell
 from buildbot.steps.source import svn
 from twisted.internet import defer
 
+WithProperties = properties.WithProperties
 
 class ConfigureBuild(buildstep.BuildStep):
     name = "configure-build"
@@ -87,3 +88,63 @@ class RunWebKitPerlTests(shell.ShellCommand):
     descriptionDone = ['webkitperl-tests']
     flunkOnFailure = True
     command = ['Tools/Scripts/test-webkitperl']
+
+
+def appendCustomBuildFlags(step, platform, fullPlatform):
+    # FIXME: Make a common 'supported platforms' list.
+    if platform not in ('gtk', 'wincairo', 'ios', 'jsc-only', 'wpe'):
+        return
+    if fullPlatform.startswith('ios-simulator'):
+        platform = 'ios-simulator'
+    elif platform == 'ios':
+        platform = 'device'
+    step.setCommand(step.command + ['--' + platform])
+
+
+class CompileWebKit(shell.Compile):
+    name = "compile-webkit"
+    description = ["compiling webkit"]
+    descriptionDone = ["compiled webkit"]
+    env = {'MFLAGS': ''}
+    warningPattern = ".*arning: .*"
+    command = ["perl", "Tools/Scripts/build-webkit", WithProperties("--%(configuration)s")]
+
+    def start(self):
+        platform = self.getProperty('platform')
+        buildOnly = self.getProperty('buildOnly')
+        architecture = self.getProperty('architecture')
+        additionalArguments = self.getProperty('additionalArguments')
+
+        if additionalArguments:
+            self.setCommand(self.command + additionalArguments)
+        if platform in ('mac', 'ios') and architecture:
+            self.setCommand(self.command + ['ARCHS=' + architecture])
+            if platform == 'ios':
+                self.setCommand(self.command + ['ONLY_ACTIVE_ARCH=NO'])
+        if platform in ('mac', 'ios') and buildOnly:
+            # For build-only bots, the expectation is that tests will be run on separate machines,
+            # so we need to package debug info as dSYMs. Only generating line tables makes
+            # this much faster than full debug info, and crash logs still have line numbers.
+            self.setCommand(self.command + ['DEBUG_INFORMATION_FORMAT=dwarf-with-dsym'])
+            self.setCommand(self.command + ['CLANG_DEBUG_INFORMATION_LEVEL=line-tables-only'])
+
+        appendCustomBuildFlags(self, platform, self.getProperty('fullPlatform'))
+
+        return shell.Compile.start(self)
+
+
+class CleanBuild(shell.Compile):
+    name = "delete-WebKitBuild-directory"
+    description = ["deleting WebKitBuild directory"]
+    descriptionDone = ["deleted WebKitBuild directory"]
+    command = ["python", "Tools/BuildSlaveSupport/clean-build", WithProperties("--platform=%(fullPlatform)s"), WithProperties("--%(configuration)s")]
+
+
+class KillOldProcesses(shell.Compile):
+    name = "kill-old-processes"
+    description = ["killing old processes"]
+    descriptionDone = ["killed old processes"]
+    command = ["python", "Tools/BuildSlaveSupport/kill-old-processes", "buildbot"]
+
+    def __init__(self, **kwargs):
+        super(KillOldProcesses, self).__init__(timeout=60, **kwargs)
index 3eeaf58..2364e02 100644 (file)
@@ -295,5 +295,111 @@ Failed 1/40 test programs. 10/630 subtests failed.''')
         return self.runStep()
 
 
+class TestKillOldProcesses(BuildStepMixinAdditions, unittest.TestCase):
+    def setUp(self):
+        self.longMessage = True
+        return self.setUpBuildStep()
+
+    def tearDown(self):
+        return self.tearDownBuildStep()
+
+    def test_success(self):
+        self.setupStep(KillOldProcesses())
+        self.expectRemoteCommands(
+            ExpectShell(workdir='wkdir',
+                        command=['python', 'Tools/BuildSlaveSupport/kill-old-processes', 'buildbot'],
+                        timeout=60,
+                        )
+            + 0,
+        )
+        self.expectOutcome(result=SUCCESS, state_string='killed old processes')
+        return self.runStep()
+
+    def test_failure(self):
+        self.setupStep(KillOldProcesses())
+        self.expectRemoteCommands(
+            ExpectShell(workdir='wkdir',
+                        command=['python', 'Tools/BuildSlaveSupport/kill-old-processes', 'buildbot'],
+                        timeout=60,
+                        )
+            + ExpectShell.log('stdio', stdout='Unexpected error.')
+            + 2,
+        )
+        self.expectOutcome(result=FAILURE, state_string='killed old processes (failure)')
+        return self.runStep()
+
+
+class TestCleanBuild(BuildStepMixinAdditions, unittest.TestCase):
+    def setUp(self):
+        self.longMessage = True
+        return self.setUpBuildStep()
+
+    def tearDown(self):
+        return self.tearDownBuildStep()
+
+    def test_success(self):
+        self.setupStep(CleanBuild())
+        self.setProperty('fullPlatform', 'ios-11')
+        self.setProperty('configuration', 'Release')
+        self.expectRemoteCommands(
+            ExpectShell(workdir='wkdir',
+                        command=['python', 'Tools/BuildSlaveSupport/clean-build', '--platform=ios-11', '--Release'],
+                        )
+            + 0,
+        )
+        self.expectOutcome(result=SUCCESS, state_string='deleted WebKitBuild directory')
+        return self.runStep()
+
+    def test_failure(self):
+        self.setupStep(CleanBuild())
+        self.setProperty('fullPlatform', 'ios-simulator-11')
+        self.setProperty('configuration', 'Debug')
+        self.expectRemoteCommands(
+            ExpectShell(workdir='wkdir',
+                        command=['python', 'Tools/BuildSlaveSupport/clean-build', '--platform=ios-simulator-11', '--Debug'],
+                        )
+            + ExpectShell.log('stdio', stdout='Unexpected error.')
+            + 2,
+        )
+        self.expectOutcome(result=FAILURE, state_string='deleted WebKitBuild directory (failure)')
+        return self.runStep()
+
+
+class TestCompileWebKit(BuildStepMixinAdditions, unittest.TestCase):
+    def setUp(self):
+        self.longMessage = True
+        return self.setUpBuildStep()
+
+    def tearDown(self):
+        return self.tearDownBuildStep()
+
+    def test_success(self):
+        self.setupStep(CompileWebKit())
+        self.setProperty('fullPlatform', 'ios-simulator-11')
+        self.setProperty('configuration', 'Release')
+        self.expectRemoteCommands(
+            ExpectShell(workdir='wkdir',
+                        command=["perl", "Tools/Scripts/build-webkit", '--Release'],
+                        )
+            + 0,
+        )
+        self.expectOutcome(result=SUCCESS, state_string='compiled webkit')
+        return self.runStep()
+
+    def test_failure(self):
+        self.setupStep(CompileWebKit())
+        self.setProperty('fullPlatform', 'mac-sierra')
+        self.setProperty('configuration', 'Debug')
+        self.expectRemoteCommands(
+            ExpectShell(workdir='wkdir',
+                        command=["perl", "Tools/Scripts/build-webkit", '--Debug'],
+                        )
+            + ExpectShell.log('stdio', stdout='1 error generated.')
+            + 2,
+        )
+        self.expectOutcome(result=FAILURE, state_string='compiled webkit (failure)')
+        return self.runStep()
+
+
 if __name__ == '__main__':
     unittest.main()
index 45f68e2..18a83e1 100644 (file)
@@ -1,5 +1,23 @@
 2018-06-26  Aakash Jain  <aakash_jain@apple.com>
 
+        [ews-build] Add support for compiling WebKit
+        https://bugs.webkit.org/show_bug.cgi?id=187019
+
+        Reviewed by Lucas Forschler.
+
+        * BuildSlaveSupport/ews-build/steps.py:
+        (CompileWebKit): Added, class to compile WebKit.
+        (CleanBuild): Added, class to clean up the build.
+        (KillOldProcesses): Added, class to kill old processes.
+        * BuildSlaveSupport/ews-build/steps_unittest.py: Added unit-tests.
+        * BuildSlaveSupport/ews-build/config.json: Renamed ios-11-simulator to ios-simulator-11, to match with build.webkit.org code.
+        Added configuration, architecture which is required for compiling. Renamed Release to release as the tools expect lower-case release.
+        * BuildSlaveSupport/ews-build/factories.py: Added BuildFactory.
+        * BuildSlaveSupport/ews-build/loadConfig.py: Renamed configuraton value to lower-case as tools expect lower-case values.
+        * BuildSlaveSupport/ews-build/loadConfig_unittest.py: Ditto.
+
+2018-06-26  Aakash Jain  <aakash_jain@apple.com>
+
         [ews-build] Add support for WebKitPerl-Tests-EWS
         https://bugs.webkit.org/show_bug.cgi?id=187023