Allow committers to use their Trac credentials to force builds on the buildbots
[WebKit-https.git] / Tools / BuildSlaveSupport / build.webkit.org-config / master.cfg
1 # -*- python -*-
2 # ex: set syntax=python:
3
4 from buildbot.buildslave import BuildSlave
5 from buildbot.changes.pb import PBChangeSource
6 from buildbot.scheduler import AnyBranchScheduler, Triggerable
7 from buildbot.schedulers.filter import ChangeFilter
8 from buildbot.status import html
9 from buildbot.status.web.authz import Authz
10 from buildbot.process import buildstep, factory, properties
11 from buildbot.steps import master, shell, source, transfer, trigger
12 from buildbot.status.builder import SUCCESS, FAILURE, WARNINGS, SKIPPED
13
14 from twisted.internet import defer
15
16 import os
17 import re
18 import simplejson
19 import urllib
20
21 from committer_auth import CommitterAuth
22 from webkitpy.common.config import build as wkbuild
23 from webkitpy.common.net.buildbot import BuildBot as wkbuildbot
24
25
26 c = BuildmasterConfig = {}
27
28 c['change_source'] = PBChangeSource()
29
30 # permissions for WebStatus
31 authz = Authz(
32     auth=CommitterAuth('auth.json'),
33     forceBuild='auth',
34     forceAllBuilds='auth',
35     pingBuilder=True,
36     gracefulShutdown=False,
37     stopBuild=True,
38     stopAllBuilds=True,
39     cancelPendingBuild=True,
40     stopChange=True,
41     cleanShutdown=False)
42
43 c['status'] = []
44 c['status'].append(html.WebStatus(http_port=8710, 
45                                   revlink="http://trac.webkit.org/changeset/%s", 
46                                   authz=authz))
47
48 c['slavePortnum'] = 17000
49 c['projectName'] = "WebKit"
50 c['projectURL'] = "http://webkit.org"
51 c['buildbotURL'] = "http://build.webkit.org/"
52
53 c['buildHorizon'] = 1000
54 c['logHorizon'] = 500
55 c['eventHorizon'] = 200
56 c['buildCacheSize'] = 60
57
58 WithProperties = properties.WithProperties
59
60
61 class ConfigureBuild(buildstep.BuildStep):
62     name = "configure build"
63     description = ["configuring build"]
64     descriptionDone = ["configured build"]
65     def __init__(self, platform, configuration, architecture, buildOnly, features, *args, **kwargs):
66         buildstep.BuildStep.__init__(self, *args, **kwargs)
67         self.platform = platform.split('-', 1)[0]
68         self.fullPlatform = platform
69         self.configuration = configuration
70         self.architecture = architecture
71         self.buildOnly = buildOnly
72         self.features = features;
73         self.addFactoryArguments(platform=platform, configuration=configuration, architecture=architecture, buildOnly=buildOnly, features=features)
74
75     def start(self):
76         self.setProperty("platform", self.platform)
77         self.setProperty("fullPlatform", self.fullPlatform)
78         self.setProperty("configuration", self.configuration)
79         self.setProperty("architecture", self.architecture)
80         self.setProperty("buildOnly", self.buildOnly)
81         self.setProperty("features", self.features)
82         self.finished(SUCCESS)
83         return defer.succeed(None)
84
85
86 class CheckOutSource(source.SVN):
87     baseURL = "http://svn.webkit.org/repository/webkit/"
88     mode = "update"
89     def __init__(self, *args, **kwargs):
90         source.SVN.__init__(self, baseURL=self.baseURL, defaultBranch="trunk", mode=self.mode, *args, **kwargs)
91
92
93 class InstallWin32Dependencies(shell.Compile):
94     description = ["installing dependencies"]
95     descriptionDone = ["installed dependencies"]
96     command = ["perl", "./Tools/Scripts/update-webkit-auxiliary-libs"]
97
98 class KillOldProcesses(shell.Compile):
99     name = "kill old processes"
100     description = ["killing old processes"]
101     descriptionDone = ["killed old processes"]
102     command = ["python", "./Tools/BuildSlaveSupport/kill-old-processes"]
103
104 class InstallGtkDependencies(shell.ShellCommand):
105     name = "jhbuild"
106     description = ["updating gtk dependencies"]
107     descriptionDone = ["updated gtk dependencies"]
108     command = ["perl", "./Tools/Scripts/update-webkitgtk-libs"]
109     haltOnFailure = True
110
111 class InstallChromiumDependencies(shell.ShellCommand):
112     name = "gclient"
113     description = ["updating chromium dependencies"]
114     descriptionDone = ["updated chromium dependencies"]
115     command = ["perl", "./Tools/Scripts/update-webkit-chromium", "--force"]
116     haltOnFailure = True
117     def start(self):
118         if self.getProperty('fullPlatform') == "chromium-android":
119             self.setCommand(self.command + ['--chromium-android'])
120
121         return shell.ShellCommand.start(self)
122
123 class CleanupChromiumCrashLogs(shell.ShellCommand):
124     name = "cleanup crash logs"
125     description = ["removing crash logs"]
126     descriptionDone = ["removed crash logs"]
127     command = ["python", "./Tools/BuildSlaveSupport/chromium/remove-crash-logs"]
128     haltOnFailure = False
129
130
131 def appendCustomBuildFlags(step, platform, fullPlatform="", useFullPlatformForChromiumCGMac=False):
132     if fullPlatform == "chromium-android":
133         step.setCommand(step.command + ['--chromium-android'])
134     elif useFullPlatformForChromiumCGMac and fullPlatform.find('-cg-mac') > 0:
135         step.setCommand(step.command + ['--platform', fullPlatform])
136         # Remove this code once Skia transition is done
137     elif platform in ('chromium', 'efl', 'gtk', 'qt', 'wincairo', 'wince', 'wx'):
138         step.setCommand(step.command + ['--' + platform])
139
140
141 class CompileWebKit(shell.Compile):
142     command = ["perl", "./Tools/Scripts/build-webkit", WithProperties("--%(configuration)s")]
143     env = {'MFLAGS':''}
144     name = "compile-webkit"
145     description = ["compiling"]
146     descriptionDone = ["compiled"]
147     warningPattern = ".*arning: .*"
148
149     def start(self):
150         platform = self.getProperty('platform')
151         buildOnly = self.getProperty('buildOnly')
152         if platform == 'mac' and buildOnly:
153             self.setCommand(self.command + ['DEBUG_INFORMATION_FORMAT=dwarf-with-dsym'])
154
155         appendCustomBuildFlags(self, platform, self.getProperty('fullPlatform'))
156
157         features = self.getProperty('features')
158         for feature in features:
159             self.setCommand(self.command + ['--' + feature])
160
161         return shell.Compile.start(self)
162
163
164 class ArchiveBuiltProduct(shell.ShellCommand):
165     command = ["python", "./Tools/BuildSlaveSupport/built-product-archive",
166                WithProperties("--platform=%(platform)s"), WithProperties("--%(configuration)s"), "archive"]
167     name = "archive-built-product"
168     description = ["archiving built product"]
169     descriptionDone = ["archived built product"]
170     haltOnFailure = True
171
172
173 class ExtractBuiltProduct(shell.ShellCommand):
174     command = ["python", "./Tools/BuildSlaveSupport/built-product-archive",
175                WithProperties("--platform=%(platform)s"), WithProperties("--%(configuration)s"), "extract"]
176     name = "extract-built-product"
177     description = ["extracting built product"]
178     descriptionDone = ["extracted built product"]
179     haltOnFailure = True
180
181
182 class UploadBuiltProduct(transfer.FileUpload):
183     slavesrc = WithProperties("WebKitBuild/%(configuration)s.zip")
184     masterdest = WithProperties("archives/%(fullPlatform)s-%(architecture)s-%(configuration)s%(extraFeatures)s/%(got_revision)s.zip",
185                                 extraFeatures=lambda properties: UploadBuiltProduct.determineExtraFeatures(properties))
186     haltOnFailure = True
187
188     def __init__(self):
189         transfer.FileUpload.__init__(self, self.slavesrc, self.masterdest, mode=0644)
190
191     @staticmethod
192     def determineExtraFeatures(properties):
193         if not properties.has_key("features"):
194             return ''
195         features = properties.getProperty("features")
196         if not len(features):
197             return ''
198         result = '-with-' + '-and-'.join(features)
199         # Just to make sure no feature will add invalid chars, we allow only letters here.
200         result = re.sub('[^a-zA-Z]', '-', result)
201         return result
202
203
204 class DownloadBuiltProduct(shell.ShellCommand):
205     command = ["curl", "--output", WithProperties("WebKitBuild/%(configuration)s.zip"), WithProperties(c["buildbotURL"] + "archives/%(fullPlatform)s-%(architecture)s-%(configuration)s/%(got_revision)s.zip")]
206     name = "download-built-product"
207     description = ["downloading built product"]
208     descriptionDone = ["downloaded built product"]
209     haltOnFailure = True
210     flunkOnFailure = True
211
212
213 class CreateWebKitBuildDirectory(shell.ShellCommand):
214     command = ["mkdir", "-p", "WebKitBuild"]
215     name = "create-webkitbuild-dir"
216     description = ["creating WebKitBuild directory"]
217     descriptionDone = ["created WebKitBuild directory"]
218     haltOnFailure = True
219     flunkOnFailure = True
220
221
222 class RunJavaScriptCoreTests(shell.Test):
223     name = "jscore-test"
224     description = ["jscore-tests running"]
225     descriptionDone = ["jscore-tests"]
226     command = ["perl", "./Tools/Scripts/run-javascriptcore-tests", WithProperties("--%(configuration)s")]
227     logfiles = {'actual.html (source)': 'Source/JavaScriptCore/tests/mozilla/actual.html'}
228
229     def __init__(self, buildJSCTool=True, *args, **kwargs):
230         self.buildJSCTool = buildJSCTool
231         shell.Test.__init__(self, *args, **kwargs)
232         self.addFactoryArguments(buildJSCTool=buildJSCTool)
233
234     def start(self):
235         appendCustomBuildFlags(self, self.getProperty('platform'))
236         if not self.buildJSCTool:
237             self.setCommand(self.command + ['--no-build'])
238         return shell.Test.start(self)
239
240     def commandComplete(self, cmd):
241         shell.Test.commandComplete(self, cmd)
242
243         logText = cmd.logs['stdio'].getText()
244         statusLines = [line for line in logText.splitlines() if line.find('regression') >= 0 and line.find(' found.') >= 0]
245         if statusLines and statusLines[0].split()[0] != '0':
246             self.regressionLine = statusLines[0]
247         else:
248             self.regressionLine = None
249
250         if 'actual.html (source)' in cmd.logs:
251             self.addHTMLLog('actual.html', cmd.logs['actual.html (source)'].getText())
252
253     def evaluateCommand(self, cmd):
254         if self.regressionLine:
255             return FAILURE
256
257         if cmd.rc != 0:
258             return FAILURE
259
260         return SUCCESS
261
262     def getText(self, cmd, results):
263         return self.getText2(cmd, results)
264
265     def getText2(self, cmd, results):
266         if results != SUCCESS and self.regressionLine:
267             return [self.name, self.regressionLine]
268
269         return [self.name]
270
271
272 class RunWebKitTests(shell.Test):
273     name = "layout-test"
274     description = ["layout-tests running"]
275     descriptionDone = ["layout-tests"]
276     command = ["perl", "./Tools/Scripts/run-webkit-tests",
277                "--no-launch-safari",
278                "--no-new-test-results",
279                "--no-sample-on-timeout",
280                "--results-directory", "layout-test-results",
281                "--use-remote-links-to-tests",
282                "--builder-name", WithProperties("%(buildername)s"),
283                "--build-number", WithProperties("%(buildnumber)s"),
284                "--master-name", "webkit.org",
285                "--test-results-server", "test-results.appspot.com",
286                WithProperties("--%(configuration)s"),
287                "--exit-after-n-crashes-or-timeouts", "20",
288                "--exit-after-n-failures", "500"]
289
290     def __init__(self, buildJSCTool=True, *args, **kwargs):
291         self.buildJSCTool = buildJSCTool
292         shell.Test.__init__(self, *args, **kwargs)
293         self.addFactoryArguments(buildJSCTool=buildJSCTool)
294
295     def start(self):
296         platform = self.getProperty('platform')
297         appendCustomBuildFlags(self, platform, self.getProperty('fullPlatform'), True)
298         if platform == "win":
299             rootArgument = ['--root=' + os.path.join("WebKitBuild", self.getProperty('configuration'), "bin")]
300         else:
301             rootArgument = ['--root=WebKitBuild/bin']
302         if not self.buildJSCTool:
303             self.setCommand(self.command + rootArgument)
304         return shell.Test.start(self)
305
306     def _parseOldRunWebKitTestsOutput(self, logText):
307         incorrectLayoutLines = []
308         for line in logText.splitlines():
309             if line.find('had incorrect layout') >= 0 or line.find('were new') >= 0 or line.find('was new') >= 0:
310                 incorrectLayoutLines.append(line)
311             elif line.find('test case') >= 0 and (line.find(' crashed') >= 0 or line.find(' timed out') >= 0):
312                 incorrectLayoutLines.append(line)
313             elif line.startswith("WARNING:") and line.find(' leak') >= 0:
314                 incorrectLayoutLines.append(line.replace('WARNING: ', ''))
315             elif line.find('Exiting early') >= 0:
316                 incorrectLayoutLines.append(line)
317
318             # FIXME: Detect and summarize leaks of RefCounted objects
319
320         self.incorrectLayoutLines = incorrectLayoutLines
321
322     # FIXME: This will break if new-run-webkit-tests changes its default log formatter.
323     nrwt_log_message_regexp = re.compile(r'(?P<log_prefix>.*) (?P<log_level>DEBUG|INFO) (?P<message>.*)')
324
325     def _strip_python_logging_prefix(self, line):
326         match_object = self.nrwt_log_message_regexp.match(line)
327         if match_object:
328             return match_object.group('message')
329         return line
330
331     def _parseNewRunWebKitTestsOutput(self, logText):
332         incorrectLayoutLines = []
333         expressions = [
334             ('flakes', re.compile(r'Unexpected flakiness.+:?\s*\((\d+)\)')),
335             ('new passes', re.compile(r'Expected to .+, but passed:\s+\((\d+)\)')),
336             ('missing results', re.compile(r'no expected results found\s*:\s+\((\d+)\)')),
337             ('failures', re.compile(r'Regressions: Unexpected.+:?\s*\((\d+)\)')),
338         ]
339         testFailures = {}
340
341         for line in logText.splitlines():
342             if line.find('Exiting early') >= 0 or line.find('leaks found') >= 0:
343                 incorrectLayoutLines.append(self._strip_python_logging_prefix(line))
344                 continue
345             for name, expression in expressions:
346                 match = expression.search(line)
347
348                 if match:
349                     testFailures[name] = testFailures.get(name, 0) + int(match.group(1))
350                     break
351
352                 # FIXME: Parse file names and put them in results
353
354         for name in testFailures:
355             incorrectLayoutLines.append(str(testFailures[name]) + ' ' + name)
356
357         self.incorrectLayoutLines = incorrectLayoutLines
358
359     def commandComplete(self, cmd):
360         shell.Test.commandComplete(self, cmd)
361
362         logText = cmd.logs['stdio'].getText()
363         if logText.find("Collecting tests ...") >= 0:
364             self._parseNewRunWebKitTestsOutput(logText)
365         else:
366             self._parseOldRunWebKitTestsOutput(logText)
367
368     def evaluateCommand(self, cmd):
369         result = SUCCESS
370
371         if self.incorrectLayoutLines:
372             if len(self.incorrectLayoutLines) == 1:
373                 line = self.incorrectLayoutLines[0]
374                 if line.find('were new') >= 0 or line.find('was new') >= 0 or line.find(' leak') >= 0:
375                     return WARNINGS
376
377             for line in self.incorrectLayoutLines:
378                 if line.find('flakes') >= 0 or line.find('new passes') >= 0 or line.find('missing results') >= 0:
379                     result = WARNINGS
380                 else:
381                     return FAILURE
382
383         if cmd.rc != 0:
384             return FAILURE
385
386         return result
387
388     def getText(self, cmd, results):
389         return self.getText2(cmd, results)
390
391     def getText2(self, cmd, results):
392         if results != SUCCESS and self.incorrectLayoutLines:
393             return self.incorrectLayoutLines
394
395         return [self.name]
396
397
398 class NewRunWebKitTests(RunWebKitTests):
399     command = ["python", "./Tools/Scripts/new-run-webkit-tests",
400                "--no-show-results", "--no-new-test-results",
401                "--verbose", "--results-directory", "layout-test-results",
402                "--builder-name", WithProperties("%(buildername)s"),
403                "--build-number", WithProperties("%(buildnumber)s"),
404                "--master-name", "webkit.org",
405                "--test-results-server", "test-results.appspot.com",
406                WithProperties("--%(configuration)s")]
407
408     def commandComplete(self, cmd):
409         shell.Test.commandComplete(self, cmd)
410
411         logText = cmd.logs['stdio'].getText()
412         self._parseNewRunWebKitTestsOutput(logText)
413
414 class RunUnitTests(shell.Test):
415     name = "run-api-tests"
416     description = ["unit tests running"]
417     descriptionDone = ["unit-tests"]
418     command = ["perl", "./Tools/Scripts/run-api-tests", WithProperties("--%(configuration)s"), "--verbose"]
419
420     def start(self):
421         platform = self.getProperty('platform')
422         if platform == 'win':
423             self.setCommand(self.command + ['--no-build'])
424         if platform.startswith('chromium'):
425             self.setCommand(self.command + ['--chromium'])
426         return shell.Test.start(self)
427
428
429 class TestWithFailureCount(shell.Test):
430     failedTestsFormatString = "%d tests failed"
431
432     def countFailures(self, cmd):
433         return 0
434
435     def commandComplete(self, cmd):
436         shell.Test.commandComplete(self, cmd)
437         self.failedTestCount = self.countFailures(cmd)
438
439     def evaluateCommand(self, cmd):
440         if self.failedTestCount:
441             return FAILURE
442
443         if cmd.rc != 0:
444             return FAILURE
445
446         return SUCCESS
447
448     def getText(self, cmd, results):
449         return self.getText2(cmd, results)
450
451     def getText2(self, cmd, results):
452         if results != SUCCESS and self.failedTestCount:
453             return [self.failedTestsFormatString % self.failedTestCount]
454
455         return [self.name]
456
457
458 class RunPythonTests(TestWithFailureCount):
459     name = "webkitpy-test"
460     description = ["python-tests running"]
461     descriptionDone = ["python-tests"]
462     command = ["python", "./Tools/Scripts/test-webkitpy"]
463     failedTestsFormatString = "%d python tests failed"
464
465     def countFailures(self, cmd):
466         logText = cmd.logs['stdio'].getText()
467         # We're looking for the line that looks like this: FAILED (failures=2, errors=1)
468         regex = re.compile(r'^FAILED \((?P<counts>[^)]+)\)')
469         for line in logText.splitlines():
470             match = regex.match(line)
471             if not match:
472                 continue
473             return sum(int(component.split('=')[1]) for component in match.group('counts').split(', '))
474         return 0
475
476
477 class RunPerlTests(TestWithFailureCount):
478     name = "webkitperl-test"
479     description = ["perl-tests running"]
480     descriptionDone = ["perl-tests"]
481     command = ["perl", "./Tools/Scripts/test-webkitperl"]
482     failedTestsFormatString = "%d perl tests failed"
483
484     def countFailures(self, cmd):
485         logText = cmd.logs['stdio'].getText()
486         # We're looking for the line that looks like this: Failed 2/19 test programs. 5/363 subtests failed.
487         regex = re.compile(r'^Failed \d+/\d+ test programs\. (?P<count>\d+)/\d+ subtests failed\.')
488         for line in logText.splitlines():
489             match = regex.match(line)
490             if not match:
491                 continue
492             return int(match.group('count'))
493         return 0
494
495
496 class RunBindingsTests(shell.Test):
497     name = "bindings-generation-tests"
498     description = ["bindings-tests running"]
499     descriptionDone = ["bindings-tests"]
500     command = ["python", "./Tools/Scripts/run-bindings-tests"]
501
502
503 class RunGtkAPITests(shell.Test):
504     name = "API tests"
505     description = ["API tests running"]
506     descriptionDone = ["API tests"]
507     command = ["perl", "./Tools/Scripts/run-gtk-tests", WithProperties("--%(configuration)s")]
508
509     def commandComplete(self, cmd):
510         shell.Test.commandComplete(self, cmd)
511
512         logText = cmd.logs['stdio'].getText()
513         incorrectLines = []
514         for line in logText.splitlines():
515             if line.startswith('ERROR'):
516                 incorrectLines.append(line)
517
518         self.incorrectLines = incorrectLines
519
520     def evaluateCommand(self, cmd):
521         if self.incorrectLines:
522             return FAILURE
523
524         if cmd.rc != 0:
525             return FAILURE
526
527         return SUCCESS
528
529     def getText(self, cmd, results):
530         return self.getText2(cmd, results)
531
532     def getText2(self, cmd, results):
533         if results != SUCCESS and self.incorrectLines:
534             return ["%d API tests failed" % len(self.incorrectLines)]
535
536         return [self.name]
537
538 class RunQtAPITests(shell.Test):
539     name = "API tests"
540     description = ["API tests running"]
541     descriptionDone = ["API tests"]
542     command = ["python", "./Tools/Scripts/run-qtwebkit-tests",
543                "--output-file=qt-unit-tests.html", "--do-not-open-results", "--timeout=120",
544                WithProperties("WebKitBuild/%(configuration_pretty)s/Source/WebKit/qt/tests/")]
545
546     def start(self):
547         self.setProperty("configuration_pretty", self.getProperty("configuration").title())
548         return shell.Test.start(self)
549
550     def commandComplete(self, cmd):
551         shell.Test.commandComplete(self, cmd)
552
553         logText = cmd.logs['stdio'].getText()
554         foundItems = re.findall("TOTALS: (?P<passed>\d+) passed, (?P<failed>\d+) failed, (?P<skipped>\d+) skipped", logText)
555
556         self.incorrectTests = 0
557         self.statusLine = []
558
559         if foundItems:
560             self.incorrectTests = int(foundItems[0][1])
561             if self.incorrectTests > 0:
562                 self.statusLine = [
563                     "%s passed, %s failed, %s skipped" % (foundItems[0][0], foundItems[0][1], foundItems[0][2])
564                 ]
565
566     def evaluateCommand(self, cmd):
567         if self.incorrectTests:
568             return WARNINGS
569
570         if cmd.rc != 0:
571             return FAILURE
572
573         return SUCCESS
574
575     def getText(self, cmd, results):
576         return self.getText2(cmd, results)
577
578     def getText2(self, cmd, results):
579         if results != SUCCESS and self.incorrectTests:
580             return self.statusLine
581
582         return [self.name]
583
584 class RunWebKitLeakTests(RunWebKitTests):
585     warnOnWarnings = True
586     def start(self):
587         self.setCommand(self.command + ["--leaks"])
588         return RunWebKitTests.start(self)
589
590
591 class RunWebKit2Tests(RunWebKitTests):
592     def start(self):
593         self.setCommand(self.command + ["--webkit-test-runner"])
594         return RunWebKitTests.start(self)
595
596
597 class RunChromiumWebKitUnitTests(shell.Test):
598     name = "webkit-unit-tests"
599     description = ["webkit-unit-tests running"]
600     descriptionDone = ["webkit-unit-tests"]
601     command = ["perl", "./Tools/Scripts/run-chromium-webkit-unit-tests",
602                WithProperties("--%(configuration)s")]
603
604
605 class ArchiveTestResults(shell.ShellCommand):
606     command = ["python", "./Tools/BuildSlaveSupport/test-result-archive",
607                WithProperties("--platform=%(platform)s"), WithProperties("--%(configuration)s"), "archive"]
608     name = "archive-test-results"
609     description = ["archiving test results"]
610     descriptionDone = ["archived test results"]
611     haltOnFailure = True
612
613
614 class UploadTestResults(transfer.FileUpload):
615     slavesrc = "layout-test-results.zip"
616     masterdest = WithProperties("public_html/results/%(buildername)s/r%(got_revision)s (%(buildnumber)s).zip")
617
618     def __init__(self):
619         transfer.FileUpload.__init__(self, self.slavesrc, self.masterdest, mode=0644)
620
621
622 class ExtractTestResults(master.MasterShellCommand):
623     zipFile = WithProperties("public_html/results/%(buildername)s/r%(got_revision)s (%(buildnumber)s).zip")
624     resultDirectory = WithProperties("public_html/results/%(buildername)s/r%(got_revision)s (%(buildnumber)s)")
625     descriptionDone = ["uploaded results"]
626
627     def __init__(self):
628         master.MasterShellCommand.__init__(self, "")
629
630     def resultDirectoryURL(self):
631         return self.build.getProperties().render(self.resultDirectory).replace("public_html/", "/") + "/"
632
633     def start(self):
634         self.command = ["ditto", "-k", "-x", "-V", self.build.getProperties().render(self.zipFile), self.build.getProperties().render(self.resultDirectory)]
635         return master.MasterShellCommand.start(self)
636
637     def addCustomURLs(self):
638         url = self.resultDirectoryURL() + "results.html"
639         self.addURL("view results", url)
640
641     def finished(self, result):
642         self.addCustomURLs()
643         return master.MasterShellCommand.finished(self, result)
644
645
646 class ExtractTestResultsAndLeaks(ExtractTestResults):
647     def addCustomURLs(self):
648         ExtractTestResults.addCustomURLs(self)
649         url = "/LeaksViewer/?url=" + urllib.quote(self.resultDirectoryURL(), safe="")
650         self.addURL("view leaks", url)
651
652
653 class Factory(factory.BuildFactory):
654     def __init__(self, platform, configuration, architectures, buildOnly, features=None, **kwargs):
655         factory.BuildFactory.__init__(self)
656         self.addStep(ConfigureBuild, platform=platform, configuration=configuration, architecture=" ".join(architectures), buildOnly=buildOnly, features=features)
657         self.addStep(CheckOutSource)
658         self.addStep(KillOldProcesses)
659         if platform == "win":
660             self.addStep(InstallWin32Dependencies)
661         if platform.startswith("chromium"):
662             self.addStep(InstallChromiumDependencies)
663         if platform == "gtk":
664             self.addStep(InstallGtkDependencies)
665
666
667 class BuildFactory(Factory):
668     def __init__(self, platform, configuration, architectures, triggers=None, upload=False, **kwargs):
669         Factory.__init__(self, platform, configuration, architectures, True, **kwargs)
670         self.addStep(CompileWebKit)
671         if triggers or upload:
672             self.addStep(ArchiveBuiltProduct)
673             self.addStep(UploadBuiltProduct)
674         if triggers:
675             self.addStep(trigger.Trigger, schedulerNames=triggers)
676
677 def unitTestsSupported(configuration, platform):
678     return (platform == 'win' or platform.startswith('mac')
679             or (platform.startswith('chromium') and platform != 'chromium-android'))
680
681 class TestFactory(Factory):
682     TestClass = RunWebKitTests
683     ExtractTestResultsClass = ExtractTestResults
684     def __init__(self, platform, configuration, architectures, **kwargs):
685         Factory.__init__(self, platform, configuration, architectures, False, **kwargs)
686         self.addStep(CreateWebKitBuildDirectory)
687         self.addStep(DownloadBuiltProduct)
688         self.addStep(ExtractBuiltProduct)
689         self.addStep(RunJavaScriptCoreTests, buildJSCTool=False)
690         self.addStep(self.TestClass, buildJSCTool=(platform != 'win'))
691
692         if unitTestsSupported(configuration, platform): 
693             self.addStep(RunUnitTests)
694         self.addStep(RunPythonTests)
695         self.addStep(RunPerlTests)
696         self.addStep(RunBindingsTests)
697         self.addStep(ArchiveTestResults)
698         self.addStep(UploadTestResults)
699         self.addStep(self.ExtractTestResultsClass)
700
701 class BuildAndTestFactory(Factory):
702     TestClass = RunWebKitTests
703     ExtractTestResultsClass = ExtractTestResults
704     def __init__(self, platform, configuration, architectures, **kwargs):
705         Factory.__init__(self, platform, configuration, architectures, False, **kwargs)
706         if platform.startswith("chromium"):
707             self.addStep(CleanupChromiumCrashLogs)
708         self.addStep(CompileWebKit)
709         if not platform.startswith("chromium"):
710             self.addStep(RunJavaScriptCoreTests)
711         if platform.startswith("chromium"):
712             self.addStep(RunChromiumWebKitUnitTests)
713         self.addStep(self.TestClass)
714         if unitTestsSupported(configuration, platform): 
715             self.addStep(RunUnitTests)
716         self.addStep(RunPythonTests)
717         # Chromium Win runs in non-Cygwin environment, which is not yet fit
718         # for running tests. This can be removed once bug 48166 is fixed.
719         if platform != "chromium-win":
720             self.addStep(RunPerlTests)
721             self.addStep(RunBindingsTests)
722         self.addStep(ArchiveTestResults)
723         self.addStep(UploadTestResults)
724         self.addStep(self.ExtractTestResultsClass)
725         if platform == "gtk":
726             self.addStep(RunGtkAPITests)
727         if platform == "qt":
728             self.addStep(RunQtAPITests)
729
730 class BuildAndTestLeaksFactory(BuildAndTestFactory):
731     TestClass = RunWebKitLeakTests
732     ExtractTestResultsClass = ExtractTestResultsAndLeaks
733
734 class NewBuildAndTestFactory(BuildAndTestFactory):
735     TestClass = NewRunWebKitTests
736
737 class TestWebKit2Factory(TestFactory):
738     TestClass = RunWebKit2Tests
739
740 class PlatformSpecificScheduler(AnyBranchScheduler):
741     def __init__(self, platform, branch, **kwargs):
742         self.platform = platform
743         filter = ChangeFilter(branch=[branch, None], filter_fn=self.filter)
744         AnyBranchScheduler.__init__(self, name=platform, change_filter=filter, **kwargs)
745
746     def filter(self, change):
747         return wkbuild.should_build(self.platform, change.files)
748
749 trunk_filter = ChangeFilter(branch=["trunk", None])
750
751 def loadBuilderConfig(c):
752     # FIXME: These file handles are leaked.
753     passwords = simplejson.load(open('passwords.json'))
754     config = simplejson.load(open('config.json'))
755
756     # use webkitpy's buildbot module to test for core builders
757     wkbb = wkbuildbot()
758
759     c['slaves'] = [BuildSlave(slave['name'], passwords[slave['name']], max_builds=1) for slave in config['slaves']]
760
761     c['schedulers'] = []
762     for scheduler in config['schedulers']:
763         if "change_filter" in scheduler:
764             scheduler["change_filter"] = globals()[scheduler["change_filter"]]
765         kls = globals()[scheduler.pop('type')]
766         # Python 2.6 can't handle unicode keys as keyword arguments:
767         # http://bugs.python.org/issue2646.  Modern versions of simplejson return
768         # unicode strings from simplejson.load, so we map all keys to str objects.
769         scheduler = dict(map(lambda key_value_pair: (str(key_value_pair[0]), key_value_pair[1]), scheduler.items()))
770
771         # BaseScheduler asserts if given unicode objects instead of strs.
772         # http://trac.buildbot.net/ticket/2075
773         scheduler['builderNames'] = map(str, scheduler['builderNames'])
774         c['schedulers'].append(kls(**scheduler))
775
776     c['builders'] = []
777     for builder in config['builders']:
778         for slaveName in builder['slavenames']:
779             for slave in config['slaves']:
780                 if slave['name'] != slaveName or slave['platform'] == '*':
781                     continue
782
783                 if slave['platform'] != builder['platform']:
784                     raise Exception, "Builder %r is for platform %r but has slave %r for platform %r!" % (builder['name'], builder['platform'], slave['name'], slave['platform'])
785
786                 break
787
788         factory = globals()["%sFactory" % builder.pop('type')]
789         factoryArgs = []
790         for key in "platform", "configuration", "architectures", "triggers":
791             value = builder.pop(key, None)
792             if value:
793                 factoryArgs.append(value)
794
795         factoryKwArgs = {
796             "features": builder.pop("features", []),
797             "upload": builder.pop("upload", False)
798         }
799
800         builder["factory"] = factory(*factoryArgs, **factoryKwArgs)
801
802         builder["category"] = "noncore"
803         if wkbb._is_core_builder(builder['name']):
804             builder["category"] = "core"
805
806         c['builders'].append(builder)
807
808 loadBuilderConfig(c)