Commit the current configuration of build.webkit.org.
authormrowe@apple.com <mrowe@apple.com@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Mon, 10 Nov 2008 21:45:32 +0000 (21:45 +0000)
committermrowe@apple.com <mrowe@apple.com@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Mon, 10 Nov 2008 21:45:32 +0000 (21:45 +0000)
git-svn-id: https://svn.webkit.org/repository/webkit/trunk@38271 268f45cc-cd09-0410-ab3c-d52691b4dbfc

WebKitTools/BuildSlaveSupport/build.webkit.org-config/webkit/builders.py
WebKitTools/BuildSlaveSupport/build.webkit.org-config/webkit/factories.py
WebKitTools/BuildSlaveSupport/build.webkit.org-config/webkit/schedulers.py
WebKitTools/BuildSlaveSupport/build.webkit.org-config/webkit/status.py
WebKitTools/BuildSlaveSupport/build.webkit.org-config/webkit/steps.py

index b4e087e..a79011c 100644 (file)
@@ -1,35 +1,51 @@
 from webkit.factories import *
-from buildbot import locks
-
-# There are four build slaves that take care of the majority of builds, with two other specialist slaves at Apple
-# Slave 1 isn older G4 PowerMac dedicated to the PLT builds, as it needs extra configuration
-# Slave 2 is a Windows PC dedicated to the Windows builds
-# Slaves 3 and 4 are older G4 PowerMacs with relatively low amounts of RAM which leads to insanely slow leaks tests
-# Slaves 4 and 5 are newer G5 PowerMacs with ATI graphics cards that lead to kernel panics during pixel tests
-
-nonATIPowerPCBuilders = ['apple-slave-%d' % i for i in (3, 4)]
-ATIPowerPCBuilders = ['apple-slave-%d' % i for i in (1, 5, 6)]
-allPowerPCBuilders = nonATIPowerPCBuilders + ATIPowerPCBuilders
-allIntelBuilders = ['bdash-slave-1', 'bdash-slave-2']
-
-_builders = [('post-commit-powerpc-mac-os-x', StandardBuildFactory, allPowerPCBuilders),
-             ('post-commit-intel-mac-os-x', StandardBuildFactory, allIntelBuilders),
-             ('post-commit-leaks-powerpc-mac-os-x', LeakBuildFactory, allPowerPCBuilders),
-             ('post-commit-leaks-intel-mac-os-x', LeakBuildFactory, allIntelBuilders),
-#             ('page-layout-test-mac-os-x', PageLoadTestBuildFactory, ['apple-slave-1']),
-#             ('post-commit-pixel-powerpc-mac-os-x', PixelTestBuildFactory, nonATIPowerPCBuilders),
-             ('post-commit-win32', Win32BuildFactory, ['apple-slave-2']),
-             ('post-commit-linux-qt', StandardBuildFactory, ['webtroll-slave-1']),
-             ('post-commit-linux-gtk', GtkBuildFactory, ['zecke-slave-1']),
-             ('periodic-powerpc-mac-os-x-no-svg', NoSVGBuildFactory, allPowerPCBuilders),
-             ('periodic-intel-mac-os-x-coverage', CoverageDataBuildFactory, allIntelBuilders),
+
+allWinBuilders = ['apple-slave-7', 'apple-slave-2'] # pewtermoose-slave-1
+allMacPPCBuilders = ['apple-slave-1', 'apple-slave-3', 'apple-slave-6']
+allMacIntelBuilders = ['bdash-slave-1', 'bdash-slave-2']
+macIntelPixelBuilders = ['apple-slave-8']
+allQtLinuxBuilders = ['webtroll-slave-1']
+allQtWinBuilders = ['qt-slave-2']
+allGtkLinuxBuilders = ['zecke-slave-1']
+allWxMacBuilders = ['kollivier-slave-1']
+
+# apple-slave-4 is currently giving incomprehensible ICEs when compiling:
+# WebKit/History/WebBackForwardList.mm: In function 'WebHistoryItem* -[WebBackForwardList currentItem](WebBackForwardList*, objc_selector*)':
+# WebKit/History/WebBackForwardList.mm:178: internal compiler error: Bus error
+
+# apple-slave-5 is currently giving incomprehensible link errors:
+# WebKitBuild/JavaScriptCore.build/Release/JavaScriptCore.build/Objects-normal/ppc/pcre_tables.o r_address (0x34a10d) field of relocation entry 12 in section (__DWARFA,__debug_info) out of range
+# /usr/bin/libtool: internal link edit command failed
+
+
+_builders = [('trunk-mac-ppc-release', StandardBuildFactory, allMacPPCBuilders, False),
+             ('trunk-mac-intel-release', StandardBuildFactory, allMacIntelBuilders, False),
+#             ('trunk-mac-ppc-debug', LeakBuildFactory, allMacPPCBuilders, False),
+             ('trunk-mac-intel-debug', LeakBuildFactory, allMacIntelBuilders, False),
+             ('trunk-mac-intel-pixel', PixelTestBuildFactory, macIntelPixelBuilders, False),
+#             ('trunk-win-release', Win32BuildFactory, allWinBuilders, False),
+             ('trunk-win-debug', Win32BuildFactory, allWinBuilders, False),
+             ('trunk-qt-linux-release', QtBuildFactory, allQtLinuxBuilders, False),
+             ('trunk-qt-win-release', QtBuildFactory, allQtWinBuilders, False),
+             ('trunk-gtk-linux-release', GtkBuildFactory, allGtkLinuxBuilders, False),
+             ('trunk-wx-mac-debug', WxBuildFactory, allWxMacBuilders, False),
+
+#             ('trunk-mac-intel-coverage', CoverageDataBuildFactory, ['bdash-slave-1'], True),
+#             ('trunk-mac-intel-nosvg', NoSVGBuildFactory, ['bdash-slave-2'], True),
+
+#             ('stable-mac-ppc-release', StandardBuildFactory, allMacPPCBuilders, False),
+#             ('stable-mac-intel-release', StandardBuildFactory, allMacIntelBuilders, False),
+#             ('stable-mac-ppc-debug', LeakBuildFactory, allMacPPCBuilders, False),
+#             ('stable-mac-intel-debug', LeakBuildFactory, allMacIntelBuilders, False),
+#             ('stable-win-release', Win32BuildFactory, allWinBuilders, False),
              ]
 
 def getBuilders():
     result = []
-    for name, factory, slaves in _builders:
+    for name, factory, slaves, periodic in _builders:
         result.append({'name': name,
                        'slavenames': slaves,
                        'builddir': name,
-                       'factory': factory()})
+                       'factory': factory(),
+                       'periodic': periodic})
     return result
index 781375f..59eb9cd 100644 (file)
@@ -12,22 +12,25 @@ class StandardBuildFactory(BuildFactory):
     def __init__(self):
         BuildFactory.__init__(self)
         self.steps.append(s(SetConfiguration, configuration="release"))
-        self.steps.append(s(self.getCompileStep(), configuration="release"))
-        self.steps.append(s(self.getJavaScriptCoreTestStep()))
-        self.steps.append(s(LayoutTest))
+        self.addCompileStep()
+        self.addJavaScriptCoreTestStep()
+        self.addLayoutTestStep()
         self.steps.append(s(UploadLayoutResults))
 #        self.steps.append(s(UploadDiskImage))
 
-    def getCompileStep(self):
-        return CompileWebKit
+    def addCompileStep(self):
+        self.steps.append(s(CompileWebKit, configuration="release"))
 
-    def getJavaScriptCoreTestStep(self):
-        return JavaScriptCoreTest
+    def addJavaScriptCoreTestStep(self):
+        self.steps.append(s(JavaScriptCoreTest))
+
+    def addLayoutTestStep(self):
+        self.steps.append(s(LayoutTest))
 
 
 class NoSVGBuildFactory(StandardBuildFactory):
-    def getCompileStep(self):
-        return CompileWebKitNoSVG
+    def addCompileStep(self):
+        self.steps.append(s(CompileWebKitNoSVG, configuration="release"))
 
 
 class PixelTestBuildFactory(BuildFactory):
@@ -38,6 +41,7 @@ class PixelTestBuildFactory(BuildFactory):
         self.steps.append(s(PixelLayoutTest))
         self.steps.append(s(UploadLayoutResults))
 
+
 class LeakBuildFactory(BuildFactory):
     def __init__(self):
         BuildFactory.__init__(self)
@@ -48,20 +52,55 @@ class LeakBuildFactory(BuildFactory):
         self.steps.append(s(UploadLayoutResults))
 #        self.steps.append(s(UploadDiskImage))
 
+
 class PageLoadTestBuildFactory(BuildFactory):
     def __init__(self):
         BuildFactory.__init__(self)
         self.steps.append(s(CompileWebKit, configuration="release"))
         self.steps.append(s(PageLoadTest))
 
-Win32BuildFactory = StandardBuildFactory
+
+class Win32BuildFactory(StandardBuildFactory):
+    def addCompileStep(self):
+        self.steps.append(s(InstallWin32Dependencies))
+        self.steps.append(s(SetConfiguration, configuration="debug"))
+        self.steps.append(s(CompileWebKitWindows, configuration="debug"))
+
+    def addLayoutTestStep(self):
+        self.steps.append(s(LayoutTestWindows))
 
 class GtkBuildFactory(StandardBuildFactory):
-    def getCompileStep(self):
-        return CompileWebKitGtk
+    def addCompileStep(self):
+#        self.steps.append(s(CleanWebKitGtk, configuration="release"))
+        self.steps.append(s(CompileWebKitGtk, configuration="release"))
+
+    def addJavaScriptCoreTestStep(self):
+        self.steps.append(s(JavaScriptCoreTestGtk))
+
+    def addLayoutTestStep(self):
+        pass
+
+
+class WxBuildFactory(StandardBuildFactory):
+    def addCompileStep(self):
+        self.steps.append(s(CleanWebKitWx, configuration="release"))
+        self.steps.append(s(CompileWebKitWx, configuration="release"))
+
+    def addJavaScriptCoreTestStep(self):
+        self.steps.append(s(JavaScriptCoreTestWx))
+
+    def addLayoutTestStep(self):
+        pass
+
+
+class QtBuildFactory(StandardBuildFactory):
+    def addCompileStep(self):
+        self.steps.append(s(CleanWebKit, configuration="release"))
+        self.steps.append(s(CompileWebKit, configuration="release"))
+
+    def addLayoutTestStep(self):
+        pass # self.steps.append(s(LayoutTestQt))
 
-    def getJavaScriptCoreTestStep(self):
-        return JavaScriptCoreTestGtk
 
 class CoverageDataBuildFactory(BuildFactory):
     def __init__(self):
index 11f9d41..4bece4f 100644 (file)
@@ -1,15 +1,16 @@
-from buildbot.scheduler import Scheduler, Periodic
+from buildbot.scheduler import AnyBranchScheduler, Periodic
 
 def getSchedulers(builders):
-    builder_names = map(lambda builder: builder['name'], builders)
-    post_commit_builders = [name for name in builder_names if name.startswith('post-commit-')] + ['page-layout-test-mac-os-x']
-    post_commit_builders = [name for name in post_commit_builders if name in builder_names]
-    post_commit_builders.sort()
-    
-    periodic_builders = [b['name'] for b in builders if b['name'].startswith('periodic-')]
-    periodic_builders.sort()
-    
-    post_commit = Scheduler(name="post-commit", branch=None, treeStableTimer=90, builderNames=post_commit_builders)
-    periodic = Periodic("periodic", periodic_builders, 6 * 60 * 60)
+    builders = map(lambda builder: (builder['name'], builder['periodic']), builders)
+    trunkBuilders = [name for name, periodic in builders if name.startswith('trunk-') and not periodic]
+    trunkBuilders.sort()
+    stableBuilders = [name for name, periodic in builders if name.startswith('stable-') and not periodic]
+    stableBuilders.sort()
+    periodicBuilders = [name for name, periodic in builders if periodic]
+    periodicBuilders.sort()
 
-    return [post_commit, periodic]
+    trunk = AnyBranchScheduler(name="trunk", branches=['trunk'], treeStableTimer=90, builderNames=trunkBuilders)
+#    stable = AnyBranchScheduler(name="stable", branches=['branches/Safari-3-1-branch'], treeStableTimer=90, builderNames=stableBuilders)
+    periodic = Periodic("periodic", periodicBuilders, 6 * 60 * 60)
+
+    return [trunk, periodic]
index ad5519e..a7228ef 100644 (file)
@@ -16,4 +16,4 @@ IRC = words.IRC(host="irc.freenode.net",
                 )
 
 def getStatusListeners():
-    return [web, allBuildsEmail, breakageEmail, IRC]
+    return [web, allBuildsEmail, IRC]
index c1933a3..e518c2c 100644 (file)
@@ -2,13 +2,13 @@ from webkit.basesteps import ShellCommand, SVN, Test, Compile, UploadCommand
 from buildbot.status.builder import SUCCESS, FAILURE, WARNINGS
 
 class CheckOutSource(SVN):
-    svnurl = "http://svn.webkit.org/repository/webkit/trunk"
+    baseURL = "http://svn.webkit.org/repository/webkit/"
     mode = "update"
     def __init__(self, *args, **kwargs):
-        SVN.__init__(self, svnurl=self.svnurl, mode=self.mode, *args, **kwargs)
+        SVN.__init__(self, baseURL=self.baseURL, defaultBranch="trunk", mode=self.mode, *args, **kwargs)
 
 class SetConfiguration(ShellCommand):
-    command = ["./WebKitTools/Scripts/set-webkit-configuration"]
+    command = ["perl", "./WebKitTools/Scripts/set-webkit-configuration"]
     
     def __init__(self, *args, **kwargs):
         configuration = kwargs.pop('configuration')
@@ -23,17 +23,20 @@ class LayoutTest(Test):
     name = "layout-test"
     description = ["layout-tests running"]
     descriptionDone = ["layout-tests"]
-    command = ["./WebKitTools/Scripts/run-webkit-tests", "--no-launch-safari", "--no-new-test-results", "--results-directory", "layout-test-results"]
+    command = ["perl", "./WebKitTools/Scripts/run-webkit-tests", "--no-launch-safari", "--no-new-test-results", "--no-sample-on-timeout", "--results-directory", "layout-test-results"]
 
     def commandComplete(self, cmd):
         Test.commandComplete(self, cmd)
         
         logText = cmd.logs['stdio'].getText()
-        incorrectLayoutLines = [line for line in logText.splitlines() if line.find('had incorrect layout') >= 0 or (line.find('test case') >= 0 and line.find(' crashed') >= 0)]
-        if incorrectLayoutLines:
-            self.incorrectLayoutLines = incorrectLayoutLines
-        else:
-            self.incorrectLayoutLines = None
+        incorrectLayoutLines = [line for line in logText.splitlines() if line.find('had incorrect layout') >= 0 or (line.find('test case') >= 0 and (line.find(' crashed') >= 0 or line.find(' timed out') >= 0))]
+        self.incorrectLayoutLines = incorrectLayoutLines
+
+    def evaluateCommand(self, cmd):
+        if self.incorrectLayoutLines or cmd.rc != 0:
+            return FAILURE
+
+        return SUCCESS
 
     def getText(self, cmd, results):
         return self.getText2(cmd, results)
@@ -49,7 +52,7 @@ class JavaScriptCoreTest(Test):
     name = "jscore-test"
     description = ["jscore-tests running"]
     descriptionDone = ["jscore-tests"]
-    command = ["./WebKitTools/Scripts/run-javascriptcore-tests"]
+    command = ["perl", "./WebKitTools/Scripts/run-javascriptcore-tests"]
     logfiles = {'results': 'JavaScriptCore/tests/mozilla/actual.html'}
 
     def commandComplete(self, cmd):
@@ -63,10 +66,10 @@ class JavaScriptCoreTest(Test):
             self.regressionLine = None
 
     def evaluateCommand(self, cmd):
-        if cmd.rc != 0:
+        if self.regressionLine:
             return FAILURE
 
-        if self.regressionLine:
+        if cmd.rc != 0:
             return FAILURE
 
         return SUCCESS
@@ -84,42 +87,34 @@ class PixelLayoutTest(LayoutTest):
     name = "pixel-layout-test"
     description = ["pixel-layout-tests running"]
     descriptionDone = ["pixel-layout-tests"]
-    command = LayoutTest.command + ["--pixel"]
+    command = LayoutTest.command + ["--pixel", "--tolerance", "0.1"]
 
     
-class LeakTest(Test):
-    name = "leak-test"
-    description = ["leak-tests running"]
-    descriptionDone = ["leak-tests"]
-    command = ["./WebKitTools/Scripts/run-webkit-tests", "--no-launch-safari", "--leaks", "--results-directory", "layout-test-results"]
+class LeakTest(LayoutTest):
+    command = ["perl", "./WebKitTools/Scripts/run-webkit-tests", "--no-launch-safari", "--no-sample-on-timeout", "--leaks", "--results-directory", "layout-test-results"]
 
     def commandComplete(self, cmd):
-        Test.commandComplete(self, cmd)
+        LayoutTest.commandComplete(self, cmd)
 
         logText = cmd.logs['stdio'].getText()
         lines = logText.splitlines()
-        self.totalLeakLines = [line for line in lines if line.find('total leaks found!') >= 0]
-        self.totalLeakLines += [line for line in lines if line.find('LEAK: ') >= 0]
-        self.totalLeakLines = [' '.join(x.split()[1:]) for x in self.totalLeakLines]
-        self.totalLeakLines += [line for line in lines if line.find('test case') >= 0 and line.find('crashed') >= 0]
+        leakLines = [line for line in lines if line.find('total leaks found!') >= 0]
+        leakLines += [line for line in lines if line.find('LEAK: ') >= 0]
+        leakLines = [' '.join(x.split()[1:]) for x in leakLines]
 
+        leakSummary = {}
+        for line in leakLines:
+            count, key = line.split(' ', 1)
+            if key.find('total leaks found!') >= 0:
+                key = 'allocations found by "leaks" tool'
 
-    def evaluateCommand(self, cmd):
-        if cmd.rc != 0:
-            return FAILURE
+            leakSummary[key] = leakSummary.get(key, 0) + int(count)
 
-        if self.totalLeakLines:
-            return FAILURE
+        leakSummaryLines = []
+        for key in sorted(leakSummary.keys()):
+            leakSummaryLines.append('%s %s' % (leakSummary[key], key))
 
-        return SUCCESS
-
-    def getText(self, cmd, results):
-        return self.getText2(cmd, results)
-
-    def getText2(self, cmd, results):
-        if results != SUCCESS and self.totalLeakLines:
-            return self.totalLeakLines
-        return [self.name]
+        self.incorrectLayoutLines += leakSummaryLines
 
 
 class UploadLayoutResults(UploadCommand, ShellCommand):
@@ -135,24 +130,24 @@ class UploadLayoutResults(UploadCommand, ShellCommand):
         ShellCommand.setBuild(self, build)
         self.initializeForUpload()
 
-        self.command = '''\
+        self.command = '''
         if [[ -d layout-test-results ]]; then \
             find layout-test-results -type d -print0 | xargs -0 chmod ug+rx; \
             find layout-test-results -type f -print0 | xargs -0 chmod ug+r; \
-            rsync -rlvzP --rsync-path="/home/buildresults/bin/rsync" layout-test-results/ %s && rm -rf layout-test-results; \
+            rsync -rlvzP --rsync-path=/home/buildresults/bin/rsync layout-test-results/ %s && rm -rf layout-test-results; \
         fi; \
         CRASH_LOG=~/Library/Logs/CrashReporter/DumpRenderTree*.crash*; \
         if [[ -f $(ls -1 $CRASH_LOG | head -n 1 ) ]]; then \
             chmod ug+r $CRASH_LOG; \
-            rsync -rlvzP --rsync-path="/home/buildresults/bin/rsync" $CRASH_LOG %s && rm -rf $CRASH_LOG; \
-        fi;''' % (self.getRemotePath(), self.getRemotePath())
+            rsync -rlvzP --rsync-path=/home/buildresults/bin/rsync $CRASH_LOG %s && rm -rf $CRASH_LOG; \
+        fi; ''' % (self.getRemotePath(), self.getRemotePath())
 
         self.addFactoryArguments(command=self.command)
 
 
 class CompileWebKit(Compile):
-    command = ["./WebKitTools/Scripts/build-webkit", "--no-color"]
-    env = {'WEBKITSUPPORTLIBRARIESZIPDIR': 'C:\\cygwin\\home\\buildbot'}
+    command = ["perl", "./WebKitTools/Scripts/build-webkit"]
+    env = {'WEBKITSUPPORTLIBRARIESZIPDIR': 'C:\\cygwin\\home\\buildbot', 'MFLAGS':''}
     def __init__(self, *args, **kwargs):
         configuration = kwargs.pop('configuration')
         
@@ -160,22 +155,69 @@ class CompileWebKit(Compile):
         self.description = ["compiling " + configuration]
         self.descriptionDone = ["compiled " + configuration]
 
-        Compile.__init__(self, *args, **kwargs)
+        Compile.__init__(self, env=self.env, *args, **kwargs)
 
+class CleanWebKit(CompileWebKit):
+    command = CompileWebKit.command + ['--clean']
+    description = ['cleaning']
+    descriptionDone = ['cleaned']
 
 class CompileWebKitNoSVG(CompileWebKit):
-    command = 'rm -rf WebKitBuild && ./WebKitTools/Scripts/build-webkit --no-svg --no-color'
+    command = 'rm -rf WebKitBuild && perl ./WebKitTools/Scripts/build-webkit --no-svg'
 
 class CompileWebKitGtk(CompileWebKit):
-    command = CompileWebKit.command + ['--gtk']
+    command = ['perl', './WebKitTools/Scripts/build-webkit', '--gtk', '--qmake=qmake-qt4']
+
+class CleanWebKitGtk(CompileWebKitGtk):
+    command = CompileWebKitGtk.command + ['--clean']
+    description = ['cleaning']
+    descriptionDone = ['cleaned']
+
+class CompileWebKitWx(CompileWebKit):
+    command = ['perl', './WebKitTools/Scripts/build-webkit', '--wx']
+
+class CleanWebKitWx(CompileWebKitWx):
+    command = CompileWebKitWx.command + ['--clean']
+    description = ['cleaning']
+    descriptionDone = ['cleaned']
+
+class CompileWebKitWindows(UploadCommand, CompileWebKit):
+    def setBuild(self, build):
+        CompileWebKit.setBuild(self, build)
+        self.initializeForUpload()
+
+        self.command = '''\
+        ./WebKitTools/Scripts/build-webkit; \
+        RESULT=$?
+        for log in $(find WebKitBuild/*/*/*/*.htm); do \
+            chmod ug+r $log; \
+            REMOTE_NAME=$(echo $log | sed -e 's|WebKitBuild/obj/||' -e 's|/Release/|-|' -e 's|/Debug/|-|'); \
+            rsync -rlvzP --rsync-path="/home/buildresults/bin/rsync" $log %s/$REMOTE_NAME && rm $log; \
+        done; \
+        exit $RESULT;''' % (self.getRemotePath(), )
+
+        self.addFactoryArguments(command=self.command)
+
+class LayoutTestWindows(LayoutTest):
+    env = {'WEBKIT_TESTFONTS': 'C:\\cygwin\\home\\buildbot\\WebKitTestFonts'}
+
+    def __init__(self, *args, **kwargs):
+        return LayoutTest.__init__(self, env=self.env, *args, **kwargs)
+
 
 class JavaScriptCoreTestGtk(JavaScriptCoreTest):
     command = JavaScriptCoreTest.command + ['--gtk']
 
+class JavaScriptCoreTestWx(JavaScriptCoreTest):
+    command = JavaScriptCoreTest.command + ['--wx']
+
+class LayoutTestQt(LayoutTest):
+    command  = LayoutTest.command + ['--qt']
+
 class InstallWin32Dependencies(ShellCommand):
     description = ["installing Windows dependencies"]
     descriptionDone = ["installed Windows dependencies"]
-    command = ["WebKitTools/Scripts/update-webkit-auxiliary-libs"]
+    command = ["perl", "./WebKitTools/Scripts/update-webkit-auxiliary-libs"]
 
 
 # class UploadDiskImage(UploadCommand, ShellCommand):
@@ -189,7 +231,7 @@ class InstallWin32Dependencies(ShellCommand):
 #         ShellCommand.__init__(self, *args, **kwargs)
 
 class GenerateCoverageData(Compile):
-    command = ["./WebKitTools/Scripts/generate-coverage-data"]
+    command = ["perl", "./WebKitTools/Scripts/generate-coverage-data"]
     description = ["generating coverage data"]
     descriptionDone = ["generated coverage data"]