96e471b07fe060f7dbd38f91fdbd11fd374684cb
[WebKit-https.git] / Tools / BuildSlaveSupport / build.webkit.org-config / master.cfg
1 # -*- python -*-
2 # ex: set syntax=python:
3
4 from buildbot.buildslave import BuildSlave
5 from buildbot.changes.pb import PBChangeSource
6 from buildbot.scheduler import AnyBranchScheduler, Triggerable
7 from buildbot.schedulers.filter import ChangeFilter
8 from buildbot.status import html
9 from buildbot.status.web.authz import Authz
10 from buildbot.process import buildstep, factory, properties
11 from buildbot.steps import master, shell, source, transfer, trigger
12 from buildbot.status.builder import SUCCESS, FAILURE, WARNINGS, SKIPPED
13
14 from twisted.internet import defer
15
16 import os
17 import re
18 import simplejson
19 import urllib
20
21 from webkitpy.common.config import build as wkbuild
22 from webkitpy.common.net.buildbot import BuildBot as wkbuildbot
23
24
25 c = BuildmasterConfig = {}
26
27 c['change_source'] = PBChangeSource()
28
29 # permissions for WebStatus
30 authz = Authz(
31     forceBuild=False,
32     forceAllBuilds=False,
33     pingBuilder=True,
34     gracefulShutdown=False,
35     stopBuild=True,
36     stopAllBuilds=True,
37     cancelPendingBuild=True,
38     stopChange=True,
39     cleanShutdown=False)
40
41 c['status'] = []
42 c['status'].append(html.WebStatus(http_port=8710, 
43                                   revlink="http://trac.webkit.org/changeset/%s", 
44                                   authz=authz))
45
46 c['slavePortnum'] = 17000
47 c['projectName'] = "WebKit"
48 c['projectURL'] = "http://webkit.org"
49 c['buildbotURL'] = "http://build.webkit.org/"
50
51 c['buildHorizon'] = 1000
52 c['logHorizon'] = 500
53 c['eventHorizon'] = 200
54 c['buildCacheSize'] = 60
55
56 WithProperties = properties.WithProperties
57
58
59 class ConfigureBuild(buildstep.BuildStep):
60     name = "configure build"
61     description = ["configuring build"]
62     descriptionDone = ["configured build"]
63     def __init__(self, platform, configuration, architecture, buildOnly, features, *args, **kwargs):
64         buildstep.BuildStep.__init__(self, *args, **kwargs)
65         self.platform = platform.split('-', 1)[0]
66         self.fullPlatform = platform
67         self.configuration = configuration
68         self.architecture = architecture
69         self.buildOnly = buildOnly
70         self.features = features;
71         self.addFactoryArguments(platform=platform, configuration=configuration, architecture=architecture, buildOnly=buildOnly, features=features)
72
73     def start(self):
74         self.setProperty("platform", self.platform)
75         self.setProperty("fullPlatform", self.fullPlatform)
76         self.setProperty("configuration", self.configuration)
77         self.setProperty("architecture", self.architecture)
78         self.setProperty("buildOnly", self.buildOnly)
79         self.setProperty("features", self.features)
80         self.finished(SUCCESS)
81         return defer.succeed(None)
82
83
84 class CheckOutSource(source.SVN):
85     baseURL = "http://svn.webkit.org/repository/webkit/"
86     mode = "update"
87     def __init__(self, *args, **kwargs):
88         source.SVN.__init__(self, baseURL=self.baseURL, defaultBranch="trunk", mode=self.mode, *args, **kwargs)
89
90
91 class InstallWin32Dependencies(shell.Compile):
92     description = ["installing dependencies"]
93     descriptionDone = ["installed dependencies"]
94     command = ["perl", "./Tools/Scripts/update-webkit-auxiliary-libs"]
95
96 class KillOldProcesses(shell.Compile):
97     name = "kill old processes"
98     description = ["killing old processes"]
99     descriptionDone = ["killed old processes"]
100     command = ["python", "./Tools/BuildSlaveSupport/kill-old-processes"]
101
102 class InstallGtkDependencies(shell.ShellCommand):
103     name = "jhbuild"
104     description = ["updating gtk dependencies"]
105     descriptionDone = ["updated gtk dependencies"]
106     command = ["perl", "./Tools/Scripts/update-webkitgtk-libs"]
107     haltOnFailure = True
108
109 class InstallChromiumDependencies(shell.ShellCommand):
110     name = "gclient"
111     description = ["updating chromium dependencies"]
112     descriptionDone = ["updated chromium dependencies"]
113     command = ["perl", "./Tools/Scripts/update-webkit-chromium", "--force"]
114     haltOnFailure = True
115     def start(self):
116         if self.getProperty('fullPlatform') == "chromium-android":
117             self.setCommand(self.command + ['--chromium-android'])
118
119         return shell.ShellCommand.start(self)
120
121 class CleanupChromiumCrashLogs(shell.ShellCommand):
122     name = "cleanup crash logs"
123     description = ["removing crash logs"]
124     descriptionDone = ["removed crash logs"]
125     command = ["python", "./Tools/BuildSlaveSupport/chromium/remove-crash-logs"]
126     haltOnFailure = False
127
128
129 def appendCustomBuildFlags(step, platform, fullPlatform="", useFullPlatformForChromiumCGMac=False):
130     if fullPlatform == "chromium-android":
131         step.setCommand(step.command + ['--chromium-android'])
132     elif useFullPlatformForChromiumCGMac and fullPlatform.find('-cg-mac') > 0:
133         step.setCommand(step.command + ['--platform', fullPlatform])
134         # Remove this code once Skia transition is done
135     elif platform in ('chromium', 'efl', 'gtk', 'qt', 'wincairo', 'wince', 'wx'):
136         step.setCommand(step.command + ['--' + platform])
137
138
139 class CompileWebKit(shell.Compile):
140     command = ["perl", "./Tools/Scripts/build-webkit", WithProperties("--%(configuration)s")]
141     env = {'MFLAGS':''}
142     name = "compile-webkit"
143     description = ["compiling"]
144     descriptionDone = ["compiled"]
145     warningPattern = ".*arning: .*"
146
147     def start(self):
148         platform = self.getProperty('platform')
149         buildOnly = self.getProperty('buildOnly')
150         if platform == 'mac' and buildOnly:
151             self.setCommand(self.command + ['DEBUG_INFORMATION_FORMAT=dwarf-with-dsym'])
152
153         appendCustomBuildFlags(self, platform, self.getProperty('fullPlatform'))
154
155         features = self.getProperty('features')
156         for feature in features:
157             self.setCommand(self.command + ['--' + feature])
158
159         return shell.Compile.start(self)
160
161
162 class ArchiveBuiltProduct(shell.ShellCommand):
163     command = ["python", "./Tools/BuildSlaveSupport/built-product-archive",
164                WithProperties("--platform=%(platform)s"), WithProperties("--%(configuration)s"), "archive"]
165     name = "archive-built-product"
166     description = ["archiving built product"]
167     descriptionDone = ["archived built product"]
168     haltOnFailure = True
169
170
171 class ExtractBuiltProduct(shell.ShellCommand):
172     command = ["python", "./Tools/BuildSlaveSupport/built-product-archive",
173                WithProperties("--platform=%(platform)s"), WithProperties("--%(configuration)s"), "extract"]
174     name = "extract-built-product"
175     description = ["extracting built product"]
176     descriptionDone = ["extracted built product"]
177     haltOnFailure = True
178
179
180 class UploadBuiltProduct(transfer.FileUpload):
181     slavesrc = WithProperties("WebKitBuild/%(configuration)s.zip")
182     masterdest = WithProperties("archives/%(fullPlatform)s-%(architecture)s-%(configuration)s%(extraFeatures)s/%(got_revision)s.zip",
183                                 extraFeatures=lambda properties: UploadBuiltProduct.determineExtraFeatures(properties))
184     haltOnFailure = True
185
186     def __init__(self):
187         transfer.FileUpload.__init__(self, self.slavesrc, self.masterdest, mode=0644)
188
189     @staticmethod
190     def determineExtraFeatures(properties):
191         if not properties.has_key("features"):
192             return ''
193         features = properties.getProperty("features")
194         if not len(features):
195             return ''
196         result = '-with-' + '-and-'.join(features)
197         # Just to make sure no feature will add invalid chars, we allow only letters here.
198         result = re.sub('[^a-zA-Z]', '-', result)
199         return result
200
201
202 class DownloadBuiltProduct(shell.ShellCommand):
203     command = ["curl", "--output", WithProperties("WebKitBuild/%(configuration)s.zip"), WithProperties(c["buildbotURL"] + "archives/%(fullPlatform)s-%(architecture)s-%(configuration)s/%(got_revision)s.zip")]
204     name = "download-built-product"
205     description = ["downloading built product"]
206     descriptionDone = ["downloaded built product"]
207     haltOnFailure = True
208     flunkOnFailure = True
209
210
211 class CreateWebKitBuildDirectory(shell.ShellCommand):
212     command = ["mkdir", "-p", "WebKitBuild"]
213     name = "create-webkitbuild-dir"
214     description = ["creating WebKitBuild directory"]
215     descriptionDone = ["created WebKitBuild directory"]
216     haltOnFailure = True
217     flunkOnFailure = True
218
219
220 class RunJavaScriptCoreTests(shell.Test):
221     name = "jscore-test"
222     description = ["jscore-tests running"]
223     descriptionDone = ["jscore-tests"]
224     command = ["perl", "./Tools/Scripts/run-javascriptcore-tests", WithProperties("--%(configuration)s")]
225     logfiles = {'actual.html (source)': 'Source/JavaScriptCore/tests/mozilla/actual.html'}
226
227     def __init__(self, buildJSCTool=True, *args, **kwargs):
228         self.buildJSCTool = buildJSCTool
229         shell.Test.__init__(self, *args, **kwargs)
230         self.addFactoryArguments(buildJSCTool=buildJSCTool)
231
232     def start(self):
233         appendCustomBuildFlags(self, self.getProperty('platform'))
234         if not self.buildJSCTool:
235             self.setCommand(self.command + ['--no-build'])
236         return shell.Test.start(self)
237
238     def commandComplete(self, cmd):
239         shell.Test.commandComplete(self, cmd)
240
241         logText = cmd.logs['stdio'].getText()
242         statusLines = [line for line in logText.splitlines() if line.find('regression') >= 0 and line.find(' found.') >= 0]
243         if statusLines and statusLines[0].split()[0] != '0':
244             self.regressionLine = statusLines[0]
245         else:
246             self.regressionLine = None
247
248         if 'actual.html (source)' in cmd.logs:
249             self.addHTMLLog('actual.html', cmd.logs['actual.html (source)'].getText())
250
251     def evaluateCommand(self, cmd):
252         if self.regressionLine:
253             return FAILURE
254
255         if cmd.rc != 0:
256             return FAILURE
257
258         return SUCCESS
259
260     def getText(self, cmd, results):
261         return self.getText2(cmd, results)
262
263     def getText2(self, cmd, results):
264         if results != SUCCESS and self.regressionLine:
265             return [self.name, self.regressionLine]
266
267         return [self.name]
268
269
270 class RunWebKitTests(shell.Test):
271     name = "layout-test"
272     description = ["layout-tests running"]
273     descriptionDone = ["layout-tests"]
274     command = ["perl", "./Tools/Scripts/run-webkit-tests",
275                "--no-launch-safari",
276                "--no-new-test-results",
277                "--no-sample-on-timeout",
278                "--results-directory", "layout-test-results",
279                "--use-remote-links-to-tests",
280                "--builder-name", WithProperties("%(buildername)s"),
281                "--build-number", WithProperties("%(buildnumber)s"),
282                "--master-name", "webkit.org",
283                "--test-results-server", "test-results.appspot.com",
284                WithProperties("--%(configuration)s"),
285                "--exit-after-n-crashes-or-timeouts", "20",
286                "--exit-after-n-failures", "500"]
287
288     def __init__(self, buildJSCTool=True, *args, **kwargs):
289         self.buildJSCTool = buildJSCTool
290         shell.Test.__init__(self, *args, **kwargs)
291         self.addFactoryArguments(buildJSCTool=buildJSCTool)
292
293     def start(self):
294         platform = self.getProperty('platform')
295         appendCustomBuildFlags(self, platform, self.getProperty('fullPlatform'), True)
296         if platform == "win":
297             rootArgument = ['--root=' + os.path.join("WebKitBuild", self.getProperty('configuration'), "bin")]
298         else:
299             rootArgument = ['--root=WebKitBuild/bin']
300         if not self.buildJSCTool:
301             self.setCommand(self.command + rootArgument)
302         return shell.Test.start(self)
303
304     def _parseOldRunWebKitTestsOutput(self, logText):
305         incorrectLayoutLines = []
306         for line in logText.splitlines():
307             if line.find('had incorrect layout') >= 0 or line.find('were new') >= 0 or line.find('was new') >= 0:
308                 incorrectLayoutLines.append(line)
309             elif line.find('test case') >= 0 and (line.find(' crashed') >= 0 or line.find(' timed out') >= 0):
310                 incorrectLayoutLines.append(line)
311             elif line.startswith("WARNING:") and line.find(' leak') >= 0:
312                 incorrectLayoutLines.append(line.replace('WARNING: ', ''))
313             elif line.find('Exiting early') >= 0:
314                 incorrectLayoutLines.append(line)
315
316             # FIXME: Detect and summarize leaks of RefCounted objects
317
318         self.incorrectLayoutLines = incorrectLayoutLines
319
320     # FIXME: This will break if new-run-webkit-tests changes its default log formatter.
321     nrwt_log_message_regexp = re.compile(r'(?P<log_prefix>.*) (?P<log_level>DEBUG|INFO) (?P<message>.*)')
322
323     def _strip_python_logging_prefix(self, line):
324         match_object = self.nrwt_log_message_regexp.match(line)
325         if match_object:
326             return match_object.group('message')
327         return line
328
329     def _parseNewRunWebKitTestsOutput(self, logText):
330         incorrectLayoutLines = []
331         expressions = [
332             ('flakes', re.compile(r'Unexpected flakiness.+:?\s*\((\d+)\)')),
333             ('new passes', re.compile(r'Expected to .+, but passed:\s+\((\d+)\)')),
334             ('missing results', re.compile(r'no expected results found\s*:\s+\((\d+)\)')),
335             ('failures', re.compile(r'Regressions: Unexpected.+:?\s*\((\d+)\)')),
336         ]
337         testFailures = {}
338
339         for line in logText.splitlines():
340             if line.find('Exiting early') >= 0 or line.find('leaks found') >= 0:
341                 incorrectLayoutLines.append(self._strip_python_logging_prefix(line))
342                 continue
343             for name, expression in expressions:
344                 match = expression.search(line)
345
346                 if match:
347                     testFailures[name] = testFailures.get(name, 0) + int(match.group(1))
348                     break
349
350                 # FIXME: Parse file names and put them in results
351
352         for name in testFailures:
353             incorrectLayoutLines.append(str(testFailures[name]) + ' ' + name)
354
355         self.incorrectLayoutLines = incorrectLayoutLines
356
357     def commandComplete(self, cmd):
358         shell.Test.commandComplete(self, cmd)
359
360         logText = cmd.logs['stdio'].getText()
361         if logText.find("Collecting tests ...") >= 0:
362             self._parseNewRunWebKitTestsOutput(logText)
363         else:
364             self._parseOldRunWebKitTestsOutput(logText)
365
366     def evaluateCommand(self, cmd):
367         result = SUCCESS
368
369         if self.incorrectLayoutLines:
370             if len(self.incorrectLayoutLines) == 1:
371                 line = self.incorrectLayoutLines[0]
372                 if line.find('were new') >= 0 or line.find('was new') >= 0 or line.find(' leak') >= 0:
373                     return WARNINGS
374
375             for line in self.incorrectLayoutLines:
376                 if line.find('flakes') >= 0 or line.find('new passes') >= 0 or line.find('missing results') >= 0:
377                     result = WARNINGS
378                 else:
379                     return FAILURE
380
381         if cmd.rc != 0:
382             return FAILURE
383
384         return result
385
386     def getText(self, cmd, results):
387         return self.getText2(cmd, results)
388
389     def getText2(self, cmd, results):
390         if results != SUCCESS and self.incorrectLayoutLines:
391             return self.incorrectLayoutLines
392
393         return [self.name]
394
395
396 class NewRunWebKitTests(RunWebKitTests):
397     command = ["python", "./Tools/Scripts/new-run-webkit-tests",
398                "--no-show-results", "--no-new-test-results",
399                "--verbose", "--results-directory", "layout-test-results",
400                "--builder-name", WithProperties("%(buildername)s"),
401                "--build-number", WithProperties("%(buildnumber)s"),
402                "--master-name", "webkit.org",
403                "--test-results-server", "test-results.appspot.com",
404                WithProperties("--%(configuration)s")]
405
406     def commandComplete(self, cmd):
407         shell.Test.commandComplete(self, cmd)
408
409         logText = cmd.logs['stdio'].getText()
410         self._parseNewRunWebKitTestsOutput(logText)
411
412 class RunUnitTests(shell.Test):
413     name = "run-api-tests"
414     description = ["unit tests running"]
415     descriptionDone = ["unit-tests"]
416     command = ["perl", "./Tools/Scripts/run-api-tests", WithProperties("--%(configuration)s"), "--verbose"]
417
418     def start(self):
419         platform = self.getProperty('platform')
420         if platform == 'win':
421             self.setCommand(self.command + ['--no-build'])
422         if platform.startswith('chromium'):
423             self.setCommand(self.command + ['--chromium'])
424         return shell.Test.start(self)
425
426
427 class TestWithFailureCount(shell.Test):
428     failedTestsFormatString = "%d tests failed"
429
430     def countFailures(self, cmd):
431         return 0
432
433     def commandComplete(self, cmd):
434         shell.Test.commandComplete(self, cmd)
435         self.failedTestCount = self.countFailures(cmd)
436
437     def evaluateCommand(self, cmd):
438         if self.failedTestCount:
439             return FAILURE
440
441         if cmd.rc != 0:
442             return FAILURE
443
444         return SUCCESS
445
446     def getText(self, cmd, results):
447         return self.getText2(cmd, results)
448
449     def getText2(self, cmd, results):
450         if results != SUCCESS and self.failedTestCount:
451             return [self.failedTestsFormatString % self.failedTestCount]
452
453         return [self.name]
454
455
456 class RunPythonTests(TestWithFailureCount):
457     name = "webkitpy-test"
458     description = ["python-tests running"]
459     descriptionDone = ["python-tests"]
460     command = ["python", "./Tools/Scripts/test-webkitpy"]
461     failedTestsFormatString = "%d python tests failed"
462
463     def countFailures(self, cmd):
464         logText = cmd.logs['stdio'].getText()
465         # We're looking for the line that looks like this: FAILED (failures=2, errors=1)
466         regex = re.compile(r'^FAILED \((?P<counts>[^)]+)\)')
467         for line in logText.splitlines():
468             match = regex.match(line)
469             if not match:
470                 continue
471             return sum(int(component.split('=')[1]) for component in match.group('counts').split(', '))
472         return 0
473
474
475 class RunPerlTests(TestWithFailureCount):
476     name = "webkitperl-test"
477     description = ["perl-tests running"]
478     descriptionDone = ["perl-tests"]
479     command = ["perl", "./Tools/Scripts/test-webkitperl"]
480     failedTestsFormatString = "%d perl tests failed"
481
482     def countFailures(self, cmd):
483         logText = cmd.logs['stdio'].getText()
484         # We're looking for the line that looks like this: Failed 2/19 test programs. 5/363 subtests failed.
485         regex = re.compile(r'^Failed \d+/\d+ test programs\. (?P<count>\d+)/\d+ subtests failed\.')
486         for line in logText.splitlines():
487             match = regex.match(line)
488             if not match:
489                 continue
490             return int(match.group('count'))
491         return 0
492
493
494 class RunBindingsTests(shell.Test):
495     name = "bindings-generation-tests"
496     description = ["bindings-tests running"]
497     descriptionDone = ["bindings-tests"]
498     command = ["python", "./Tools/Scripts/run-bindings-tests"]
499
500
501 class RunGtkAPITests(shell.Test):
502     name = "API tests"
503     description = ["API tests running"]
504     descriptionDone = ["API tests"]
505     command = ["perl", "./Tools/Scripts/run-gtk-tests", WithProperties("--%(configuration)s")]
506
507     def commandComplete(self, cmd):
508         shell.Test.commandComplete(self, cmd)
509
510         logText = cmd.logs['stdio'].getText()
511         incorrectLines = []
512         for line in logText.splitlines():
513             if line.startswith('ERROR'):
514                 incorrectLines.append(line)
515
516         self.incorrectLines = incorrectLines
517
518     def evaluateCommand(self, cmd):
519         if self.incorrectLines:
520             return FAILURE
521
522         if cmd.rc != 0:
523             return FAILURE
524
525         return SUCCESS
526
527     def getText(self, cmd, results):
528         return self.getText2(cmd, results)
529
530     def getText2(self, cmd, results):
531         if results != SUCCESS and self.incorrectLines:
532             return ["%d API tests failed" % len(self.incorrectLines)]
533
534         return [self.name]
535
536 class RunQtAPITests(shell.Test):
537     name = "API tests"
538     description = ["API tests running"]
539     descriptionDone = ["API tests"]
540     command = ["python", "./Tools/Scripts/run-qtwebkit-tests",
541                "--output-file=qt-unit-tests.html", "--do-not-open-results", "--timeout=120",
542                WithProperties("WebKitBuild/%(configuration_pretty)s/Source/WebKit/qt/tests/")]
543
544     def start(self):
545         self.setProperty("configuration_pretty", self.getProperty("configuration").title())
546         return shell.Test.start(self)
547
548     def commandComplete(self, cmd):
549         shell.Test.commandComplete(self, cmd)
550
551         logText = cmd.logs['stdio'].getText()
552         foundItems = re.findall("TOTALS: (?P<passed>\d+) passed, (?P<failed>\d+) failed, (?P<skipped>\d+) skipped", logText)
553
554         self.incorrectTests = 0
555         self.statusLine = []
556
557         if foundItems:
558             self.incorrectTests = int(foundItems[0][1])
559             if self.incorrectTests > 0:
560                 self.statusLine = [
561                     "%s passed, %s failed, %s skipped" % (foundItems[0][0], foundItems[0][1], foundItems[0][2])
562                 ]
563
564     def evaluateCommand(self, cmd):
565         if self.incorrectTests:
566             return WARNINGS
567
568         if cmd.rc != 0:
569             return FAILURE
570
571         return SUCCESS
572
573     def getText(self, cmd, results):
574         return self.getText2(cmd, results)
575
576     def getText2(self, cmd, results):
577         if results != SUCCESS and self.incorrectTests:
578             return self.statusLine
579
580         return [self.name]
581
582 class RunWebKitLeakTests(RunWebKitTests):
583     warnOnWarnings = True
584     def start(self):
585         self.setCommand(self.command + ["--leaks"])
586         return RunWebKitTests.start(self)
587
588
589 class RunWebKit2Tests(RunWebKitTests):
590     def start(self):
591         self.setCommand(self.command + ["--webkit-test-runner"])
592         return RunWebKitTests.start(self)
593
594
595 class RunChromiumWebKitUnitTests(shell.Test):
596     name = "webkit-unit-tests"
597     description = ["webkit-unit-tests running"]
598     descriptionDone = ["webkit-unit-tests"]
599     command = ["perl", "./Tools/Scripts/run-chromium-webkit-unit-tests",
600                WithProperties("--%(configuration)s")]
601
602
603 class ArchiveTestResults(shell.ShellCommand):
604     command = ["python", "./Tools/BuildSlaveSupport/test-result-archive",
605                WithProperties("--platform=%(platform)s"), WithProperties("--%(configuration)s"), "archive"]
606     name = "archive-test-results"
607     description = ["archiving test results"]
608     descriptionDone = ["archived test results"]
609     haltOnFailure = True
610
611
612 class UploadTestResults(transfer.FileUpload):
613     slavesrc = "layout-test-results.zip"
614     masterdest = WithProperties("public_html/results/%(buildername)s/r%(got_revision)s (%(buildnumber)s).zip")
615
616     def __init__(self):
617         transfer.FileUpload.__init__(self, self.slavesrc, self.masterdest, mode=0644)
618
619
620 class ExtractTestResults(master.MasterShellCommand):
621     zipFile = WithProperties("public_html/results/%(buildername)s/r%(got_revision)s (%(buildnumber)s).zip")
622     resultDirectory = WithProperties("public_html/results/%(buildername)s/r%(got_revision)s (%(buildnumber)s)")
623     descriptionDone = ["uploaded results"]
624
625     def __init__(self):
626         master.MasterShellCommand.__init__(self, "")
627
628     def resultDirectoryURL(self):
629         return self.build.getProperties().render(self.resultDirectory).replace("public_html/", "/") + "/"
630
631     def start(self):
632         self.command = ["ditto", "-k", "-x", "-V", self.build.getProperties().render(self.zipFile), self.build.getProperties().render(self.resultDirectory)]
633         return master.MasterShellCommand.start(self)
634
635     def addCustomURLs(self):
636         url = self.resultDirectoryURL() + "results.html"
637         self.addURL("view results", url)
638
639     def finished(self, result):
640         self.addCustomURLs()
641         return master.MasterShellCommand.finished(self, result)
642
643
644 class ExtractTestResultsAndLeaks(ExtractTestResults):
645     def addCustomURLs(self):
646         ExtractTestResults.addCustomURLs(self)
647         url = "/LeaksViewer/?url=" + urllib.quote(self.resultDirectoryURL(), safe="")
648         self.addURL("view leaks", url)
649
650
651 class Factory(factory.BuildFactory):
652     def __init__(self, platform, configuration, architectures, buildOnly, features=None, **kwargs):
653         factory.BuildFactory.__init__(self)
654         self.addStep(ConfigureBuild, platform=platform, configuration=configuration, architecture=" ".join(architectures), buildOnly=buildOnly, features=features)
655         self.addStep(CheckOutSource)
656         self.addStep(KillOldProcesses)
657         if platform == "win":
658             self.addStep(InstallWin32Dependencies)
659         if platform.startswith("chromium"):
660             self.addStep(InstallChromiumDependencies)
661         if platform == "gtk":
662             self.addStep(InstallGtkDependencies)
663
664
665 class BuildFactory(Factory):
666     def __init__(self, platform, configuration, architectures, triggers=None, upload=False, **kwargs):
667         Factory.__init__(self, platform, configuration, architectures, True, **kwargs)
668         self.addStep(CompileWebKit)
669         if triggers or upload:
670             self.addStep(ArchiveBuiltProduct)
671             self.addStep(UploadBuiltProduct)
672         if triggers:
673             self.addStep(trigger.Trigger, schedulerNames=triggers)
674
675 def unitTestsSupported(configuration, platform):
676     return (platform == 'win' or platform.startswith('mac')
677             or (platform.startswith('chromium') and platform != 'chromium-android'))
678
679 class TestFactory(Factory):
680     TestClass = RunWebKitTests
681     ExtractTestResultsClass = ExtractTestResults
682     def __init__(self, platform, configuration, architectures, **kwargs):
683         Factory.__init__(self, platform, configuration, architectures, False, **kwargs)
684         self.addStep(CreateWebKitBuildDirectory)
685         self.addStep(DownloadBuiltProduct)
686         self.addStep(ExtractBuiltProduct)
687         self.addStep(RunJavaScriptCoreTests, buildJSCTool=False)
688         self.addStep(self.TestClass, buildJSCTool=(platform != 'win'))
689
690         if unitTestsSupported(configuration, platform): 
691             self.addStep(RunUnitTests)
692         self.addStep(RunPythonTests)
693         self.addStep(RunPerlTests)
694         self.addStep(RunBindingsTests)
695         self.addStep(ArchiveTestResults)
696         self.addStep(UploadTestResults)
697         self.addStep(self.ExtractTestResultsClass)
698
699 class BuildAndTestFactory(Factory):
700     TestClass = RunWebKitTests
701     ExtractTestResultsClass = ExtractTestResults
702     def __init__(self, platform, configuration, architectures, **kwargs):
703         Factory.__init__(self, platform, configuration, architectures, False, **kwargs)
704         if platform.startswith("chromium"):
705             self.addStep(CleanupChromiumCrashLogs)
706         self.addStep(CompileWebKit)
707         if not platform.startswith("chromium"):
708             self.addStep(RunJavaScriptCoreTests)
709         if platform.startswith("chromium"):
710             self.addStep(RunChromiumWebKitUnitTests)
711         self.addStep(self.TestClass)
712         if unitTestsSupported(configuration, platform): 
713             self.addStep(RunUnitTests)
714         self.addStep(RunPythonTests)
715         # Chromium Win runs in non-Cygwin environment, which is not yet fit
716         # for running tests. This can be removed once bug 48166 is fixed.
717         if platform != "chromium-win":
718             self.addStep(RunPerlTests)
719             self.addStep(RunBindingsTests)
720         self.addStep(ArchiveTestResults)
721         self.addStep(UploadTestResults)
722         self.addStep(self.ExtractTestResultsClass)
723         if platform == "gtk":
724             self.addStep(RunGtkAPITests)
725         if platform == "qt":
726             self.addStep(RunQtAPITests)
727
728 class BuildAndTestLeaksFactory(BuildAndTestFactory):
729     TestClass = RunWebKitLeakTests
730     ExtractTestResultsClass = ExtractTestResultsAndLeaks
731
732 class NewBuildAndTestFactory(BuildAndTestFactory):
733     TestClass = NewRunWebKitTests
734
735 class TestWebKit2Factory(TestFactory):
736     TestClass = RunWebKit2Tests
737
738 class PlatformSpecificScheduler(AnyBranchScheduler):
739     def __init__(self, platform, branch, **kwargs):
740         self.platform = platform
741         filter = ChangeFilter(branch=[branch, None], filter_fn=self.filter)
742         AnyBranchScheduler.__init__(self, name=platform, change_filter=filter, **kwargs)
743
744     def filter(self, change):
745         return wkbuild.should_build(self.platform, change.files)
746
747 trunk_filter = ChangeFilter(branch=["trunk", None])
748
749 def loadBuilderConfig(c):
750     # FIXME: These file handles are leaked.
751     passwords = simplejson.load(open('passwords.json'))
752     config = simplejson.load(open('config.json'))
753
754     # use webkitpy's buildbot module to test for core builders
755     wkbb = wkbuildbot()
756
757     c['slaves'] = [BuildSlave(slave['name'], passwords[slave['name']], max_builds=1) for slave in config['slaves']]
758
759     c['schedulers'] = []
760     for scheduler in config['schedulers']:
761         if "change_filter" in scheduler:
762             scheduler["change_filter"] = globals()[scheduler["change_filter"]]
763         kls = globals()[scheduler.pop('type')]
764         # Python 2.6 can't handle unicode keys as keyword arguments:
765         # http://bugs.python.org/issue2646.  Modern versions of simplejson return
766         # unicode strings from simplejson.load, so we map all keys to str objects.
767         scheduler = dict(map(lambda key_value_pair: (str(key_value_pair[0]), key_value_pair[1]), scheduler.items()))
768
769         # BaseScheduler asserts if given unicode objects instead of strs.
770         # http://trac.buildbot.net/ticket/2075
771         scheduler['builderNames'] = map(str, scheduler['builderNames'])
772         c['schedulers'].append(kls(**scheduler))
773
774     c['builders'] = []
775     for builder in config['builders']:
776         for slaveName in builder['slavenames']:
777             for slave in config['slaves']:
778                 if slave['name'] != slaveName or slave['platform'] == '*':
779                     continue
780
781                 if slave['platform'] != builder['platform']:
782                     raise Exception, "Builder %r is for platform %r but has slave %r for platform %r!" % (builder['name'], builder['platform'], slave['name'], slave['platform'])
783
784                 break
785
786         factory = globals()["%sFactory" % builder.pop('type')]
787         factoryArgs = []
788         for key in "platform", "configuration", "architectures", "triggers":
789             value = builder.pop(key, None)
790             if value:
791                 factoryArgs.append(value)
792
793         factoryKwArgs = {
794             "features": builder.pop("features", []),
795             "upload": builder.pop("upload", False)
796         }
797
798         builder["factory"] = factory(*factoryArgs, **factoryKwArgs)
799
800         builder["category"] = "noncore"
801         if wkbb._is_core_builder(builder['name']):
802             builder["category"] = "core"
803
804         c['builders'].append(builder)
805
806 loadBuilderConfig(c)