Categorize bots by ports instead of core/non-core separation
[WebKit-https.git] / Tools / BuildSlaveSupport / build.webkit.org-config / master.cfg
1 # -*- python -*-
2 # ex: set syntax=python:
3
4 from buildbot.buildslave import BuildSlave
5 from buildbot.changes.pb import PBChangeSource
6 from buildbot.scheduler import AnyBranchScheduler, Triggerable
7 from buildbot.schedulers.filter import ChangeFilter
8 from buildbot.status import html
9 from buildbot.status.web.authz import Authz
10 from buildbot.process import buildstep, factory, properties
11 from buildbot.steps import master, shell, source, transfer, trigger
12 from buildbot.status.builder import SUCCESS, FAILURE, WARNINGS, SKIPPED
13
14 from twisted.internet import defer
15
16 import os
17 import re
18 import simplejson
19 import urllib
20
21 from committer_auth import CommitterAuth
22 from webkitpy.common.config import build as wkbuild
23
24
25 c = BuildmasterConfig = {}
26
27 c['change_source'] = PBChangeSource()
28
29 # permissions for WebStatus
30 authz = Authz(
31     auth=CommitterAuth('auth.json'),
32     forceBuild='auth',
33     forceAllBuilds='auth',
34     pingBuilder=True,
35     gracefulShutdown=False,
36     stopBuild=True,
37     stopAllBuilds=True,
38     cancelPendingBuild=True,
39     stopChange=True,
40     cleanShutdown=False)
41
42 c['status'] = []
43 c['status'].append(html.WebStatus(http_port=8710, 
44                                   revlink="http://trac.webkit.org/changeset/%s", 
45                                   authz=authz))
46
47 c['slavePortnum'] = 17000
48 c['projectName'] = "WebKit"
49 c['projectURL'] = "http://webkit.org"
50 c['buildbotURL'] = "http://build.webkit.org/"
51
52 c['buildHorizon'] = 1000
53 c['logHorizon'] = 500
54 c['eventHorizon'] = 200
55 c['buildCacheSize'] = 60
56
57 WithProperties = properties.WithProperties
58
59
60 class TestWithFailureCount(shell.Test):
61     failedTestsFormatString = "%d tests failed"
62
63     def countFailures(self, cmd):
64         return 0
65
66     def commandComplete(self, cmd):
67         shell.Test.commandComplete(self, cmd)
68         self.failedTestCount = self.countFailures(cmd)
69
70     def evaluateCommand(self, cmd):
71         if self.failedTestCount:
72             return FAILURE
73
74         if cmd.rc != 0:
75             return FAILURE
76
77         return SUCCESS
78
79     def getText(self, cmd, results):
80         return self.getText2(cmd, results)
81
82     def getText2(self, cmd, results):
83         if results != SUCCESS and self.failedTestCount:
84             return [self.failedTestsFormatString % self.failedTestCount]
85
86         return [self.name]
87
88
89 class ConfigureBuild(buildstep.BuildStep):
90     name = "configure build"
91     description = ["configuring build"]
92     descriptionDone = ["configured build"]
93     def __init__(self, platform, configuration, architecture, buildOnly, features, *args, **kwargs):
94         buildstep.BuildStep.__init__(self, *args, **kwargs)
95         self.platform = platform.split('-', 1)[0]
96         self.fullPlatform = platform
97         self.configuration = configuration
98         self.architecture = architecture
99         self.buildOnly = buildOnly
100         self.features = features;
101         self.addFactoryArguments(platform=platform, configuration=configuration, architecture=architecture, buildOnly=buildOnly, features=features)
102
103     def start(self):
104         self.setProperty("platform", self.platform)
105         self.setProperty("fullPlatform", self.fullPlatform)
106         self.setProperty("configuration", self.configuration)
107         self.setProperty("architecture", self.architecture)
108         self.setProperty("buildOnly", self.buildOnly)
109         self.setProperty("features", self.features)
110         self.finished(SUCCESS)
111         return defer.succeed(None)
112
113
114 class CheckOutSource(source.SVN):
115     baseURL = "http://svn.webkit.org/repository/webkit/"
116     mode = "update"
117     def __init__(self, *args, **kwargs):
118         source.SVN.__init__(self, baseURL=self.baseURL, defaultBranch="trunk", mode=self.mode, *args, **kwargs)
119
120
121 class InstallWin32Dependencies(shell.Compile):
122     description = ["installing dependencies"]
123     descriptionDone = ["installed dependencies"]
124     command = ["perl", "./Tools/Scripts/update-webkit-auxiliary-libs"]
125
126 class KillOldProcesses(shell.Compile):
127     name = "kill old processes"
128     description = ["killing old processes"]
129     descriptionDone = ["killed old processes"]
130     command = ["python", "./Tools/BuildSlaveSupport/kill-old-processes"]
131
132 class InstallGtkDependencies(shell.ShellCommand):
133     name = "jhbuild"
134     description = ["updating gtk dependencies"]
135     descriptionDone = ["updated gtk dependencies"]
136     command = ["perl", "./Tools/Scripts/update-webkitgtk-libs"]
137     haltOnFailure = True
138
139 class InstallChromiumDependencies(shell.ShellCommand):
140     name = "gclient"
141     description = ["updating chromium dependencies"]
142     descriptionDone = ["updated chromium dependencies"]
143     command = ["perl", "./Tools/Scripts/update-webkit-chromium", "--force"]
144     haltOnFailure = True
145     def start(self):
146         if self.getProperty('fullPlatform') == "chromium-android":
147             self.setCommand(self.command + ['--chromium-android'])
148
149         return shell.ShellCommand.start(self)
150
151 class CleanupChromiumCrashLogs(shell.ShellCommand):
152     name = "cleanup crash logs"
153     description = ["removing crash logs"]
154     descriptionDone = ["removed crash logs"]
155     command = ["python", "./Tools/BuildSlaveSupport/chromium/remove-crash-logs"]
156     haltOnFailure = False
157
158
159 def appendCustomBuildFlags(step, platform, fullPlatform=""):
160     if fullPlatform == "chromium-android":
161         step.setCommand(step.command + ['--chromium-android'])
162     elif platform in ('chromium', 'efl', 'gtk', 'qt', 'wincairo', 'wince', 'wx'):
163         step.setCommand(step.command + ['--' + platform])
164
165
166 class CompileWebKit(shell.Compile):
167     command = ["perl", "./Tools/Scripts/build-webkit", WithProperties("--%(configuration)s")]
168     env = {'MFLAGS':''}
169     name = "compile-webkit"
170     description = ["compiling"]
171     descriptionDone = ["compiled"]
172     warningPattern = ".*arning: .*"
173
174     def start(self):
175         platform = self.getProperty('platform')
176         buildOnly = self.getProperty('buildOnly')
177         if platform == 'mac' and buildOnly:
178             self.setCommand(self.command + ['DEBUG_INFORMATION_FORMAT=dwarf-with-dsym'])
179
180         appendCustomBuildFlags(self, platform, self.getProperty('fullPlatform'))
181
182         features = self.getProperty('features')
183         for feature in features:
184             self.setCommand(self.command + ['--' + feature])
185
186         return shell.Compile.start(self)
187
188
189 class ArchiveBuiltProduct(shell.ShellCommand):
190     command = ["python", "./Tools/BuildSlaveSupport/built-product-archive",
191                WithProperties("--platform=%(platform)s"), WithProperties("--%(configuration)s"), "archive"]
192     name = "archive-built-product"
193     description = ["archiving built product"]
194     descriptionDone = ["archived built product"]
195     haltOnFailure = True
196
197
198 class ExtractBuiltProduct(shell.ShellCommand):
199     command = ["python", "./Tools/BuildSlaveSupport/built-product-archive",
200                WithProperties("--platform=%(platform)s"), WithProperties("--%(configuration)s"), "extract"]
201     name = "extract-built-product"
202     description = ["extracting built product"]
203     descriptionDone = ["extracted built product"]
204     haltOnFailure = True
205
206
207 class UploadBuiltProduct(transfer.FileUpload):
208     slavesrc = WithProperties("WebKitBuild/%(configuration)s.zip")
209     masterdest = WithProperties("archives/%(fullPlatform)s-%(architecture)s-%(configuration)s%(extraFeatures)s/%(got_revision)s.zip",
210                                 extraFeatures=lambda properties: UploadBuiltProduct.determineExtraFeatures(properties))
211     haltOnFailure = True
212
213     def __init__(self):
214         transfer.FileUpload.__init__(self, self.slavesrc, self.masterdest, mode=0644)
215
216     @staticmethod
217     def determineExtraFeatures(properties):
218         if not properties.has_key("features"):
219             return ''
220         features = properties.getProperty("features")
221         if not len(features):
222             return ''
223         result = '-with-' + '-and-'.join(features)
224         # Just to make sure no feature will add invalid chars, we allow only letters here.
225         result = re.sub('[^a-zA-Z]', '-', result)
226         return result
227
228
229 class DownloadBuiltProduct(shell.ShellCommand):
230     command = ["curl", "--output", WithProperties("WebKitBuild/%(configuration)s.zip"), WithProperties(c["buildbotURL"] + "archives/%(fullPlatform)s-%(architecture)s-%(configuration)s/%(got_revision)s.zip")]
231     name = "download-built-product"
232     description = ["downloading built product"]
233     descriptionDone = ["downloaded built product"]
234     haltOnFailure = True
235     flunkOnFailure = True
236
237
238 class CreateWebKitBuildDirectory(shell.ShellCommand):
239     command = ["mkdir", "-p", "WebKitBuild"]
240     name = "create-webkitbuild-dir"
241     description = ["creating WebKitBuild directory"]
242     descriptionDone = ["created WebKitBuild directory"]
243     haltOnFailure = True
244     flunkOnFailure = True
245
246
247 class RunJavaScriptCoreTests(shell.Test):
248     name = "jscore-test"
249     description = ["jscore-tests running"]
250     descriptionDone = ["jscore-tests"]
251     command = ["perl", "./Tools/Scripts/run-javascriptcore-tests", WithProperties("--%(configuration)s")]
252     logfiles = {'actual.html (source)': 'Source/JavaScriptCore/tests/mozilla/actual.html'}
253
254     def __init__(self, buildJSCTool=True, *args, **kwargs):
255         self.buildJSCTool = buildJSCTool
256         shell.Test.__init__(self, *args, **kwargs)
257         self.addFactoryArguments(buildJSCTool=buildJSCTool)
258
259     def start(self):
260         appendCustomBuildFlags(self, self.getProperty('platform'))
261         if not self.buildJSCTool:
262             self.setCommand(self.command + ['--no-build'])
263         return shell.Test.start(self)
264
265     def commandComplete(self, cmd):
266         shell.Test.commandComplete(self, cmd)
267
268         logText = cmd.logs['stdio'].getText()
269         statusLines = [line for line in logText.splitlines() if line.find('regression') >= 0 and line.find(' found.') >= 0]
270         if statusLines and statusLines[0].split()[0] != '0':
271             self.regressionLine = statusLines[0]
272         else:
273             self.regressionLine = None
274
275         if 'actual.html (source)' in cmd.logs:
276             self.addHTMLLog('actual.html', cmd.logs['actual.html (source)'].getText())
277
278     def evaluateCommand(self, cmd):
279         if self.regressionLine:
280             return FAILURE
281
282         if cmd.rc != 0:
283             return FAILURE
284
285         return SUCCESS
286
287     def getText(self, cmd, results):
288         return self.getText2(cmd, results)
289
290     def getText2(self, cmd, results):
291         if results != SUCCESS and self.regressionLine:
292             return [self.name, self.regressionLine]
293
294         return [self.name]
295
296
297 class RunWebKitTests(shell.Test):
298     name = "layout-test"
299     description = ["layout-tests running"]
300     descriptionDone = ["layout-tests"]
301     command = ["perl", "./Tools/Scripts/run-webkit-tests",
302                "--no-launch-safari",
303                "--no-new-test-results",
304                "--no-sample-on-timeout",
305                "--results-directory", "layout-test-results",
306                "--use-remote-links-to-tests",
307                "--builder-name", WithProperties("%(buildername)s"),
308                "--build-number", WithProperties("%(buildnumber)s"),
309                "--master-name", "webkit.org",
310                "--test-results-server", "test-results.appspot.com",
311                WithProperties("--%(configuration)s"),
312                "--exit-after-n-crashes-or-timeouts", "20",
313                "--exit-after-n-failures", "500"]
314
315     def __init__(self, buildJSCTool=True, *args, **kwargs):
316         self.buildJSCTool = buildJSCTool
317         shell.Test.__init__(self, *args, **kwargs)
318         self.addFactoryArguments(buildJSCTool=buildJSCTool)
319
320     def start(self):
321         platform = self.getProperty('platform')
322         appendCustomBuildFlags(self, platform, self.getProperty('fullPlatform'))
323         if platform == "win":
324             rootArgument = ['--root=' + os.path.join("WebKitBuild", self.getProperty('configuration'), "bin")]
325         else:
326             rootArgument = ['--root=WebKitBuild/bin']
327         if not self.buildJSCTool:
328             self.setCommand(self.command + rootArgument)
329         return shell.Test.start(self)
330
331     def _parseOldRunWebKitTestsOutput(self, logText):
332         incorrectLayoutLines = []
333         for line in logText.splitlines():
334             if line.find('had incorrect layout') >= 0 or line.find('were new') >= 0 or line.find('was new') >= 0:
335                 incorrectLayoutLines.append(line)
336             elif line.find('test case') >= 0 and (line.find(' crashed') >= 0 or line.find(' timed out') >= 0):
337                 incorrectLayoutLines.append(line)
338             elif line.startswith("WARNING:") and line.find(' leak') >= 0:
339                 incorrectLayoutLines.append(line.replace('WARNING: ', ''))
340             elif line.find('Exiting early') >= 0:
341                 incorrectLayoutLines.append(line)
342
343             # FIXME: Detect and summarize leaks of RefCounted objects
344
345         self.incorrectLayoutLines = incorrectLayoutLines
346
347     # FIXME: This will break if new-run-webkit-tests changes its default log formatter.
348     nrwt_log_message_regexp = re.compile(r'(?P<log_prefix>.*) (?P<log_level>DEBUG|INFO) (?P<message>.*)')
349
350     def _strip_python_logging_prefix(self, line):
351         match_object = self.nrwt_log_message_regexp.match(line)
352         if match_object:
353             return match_object.group('message')
354         return line
355
356     def _parseNewRunWebKitTestsOutput(self, logText):
357         incorrectLayoutLines = []
358         expressions = [
359             ('flakes', re.compile(r'Unexpected flakiness.+:?\s*\((\d+)\)')),
360             ('new passes', re.compile(r'Expected to .+, but passed:\s+\((\d+)\)')),
361             ('missing results', re.compile(r'no expected results found\s*:\s+\((\d+)\)')),
362             ('failures', re.compile(r'Regressions: Unexpected.+:?\s*\((\d+)\)')),
363         ]
364         testFailures = {}
365
366         for line in logText.splitlines():
367             if line.find('Exiting early') >= 0 or line.find('leaks found') >= 0:
368                 incorrectLayoutLines.append(self._strip_python_logging_prefix(line))
369                 continue
370             for name, expression in expressions:
371                 match = expression.search(line)
372
373                 if match:
374                     testFailures[name] = testFailures.get(name, 0) + int(match.group(1))
375                     break
376
377                 # FIXME: Parse file names and put them in results
378
379         for name in testFailures:
380             incorrectLayoutLines.append(str(testFailures[name]) + ' ' + name)
381
382         self.incorrectLayoutLines = incorrectLayoutLines
383
384     def commandComplete(self, cmd):
385         shell.Test.commandComplete(self, cmd)
386
387         logText = cmd.logs['stdio'].getText()
388         if logText.find("Collecting tests ...") >= 0:
389             self._parseNewRunWebKitTestsOutput(logText)
390         else:
391             self._parseOldRunWebKitTestsOutput(logText)
392
393     def evaluateCommand(self, cmd):
394         result = SUCCESS
395
396         if self.incorrectLayoutLines:
397             if len(self.incorrectLayoutLines) == 1:
398                 line = self.incorrectLayoutLines[0]
399                 if line.find('were new') >= 0 or line.find('was new') >= 0 or line.find(' leak') >= 0:
400                     return WARNINGS
401
402             for line in self.incorrectLayoutLines:
403                 if line.find('flakes') >= 0 or line.find('new passes') >= 0 or line.find('missing results') >= 0:
404                     result = WARNINGS
405                 else:
406                     return FAILURE
407
408         if cmd.rc != 0:
409             return FAILURE
410
411         return result
412
413     def getText(self, cmd, results):
414         return self.getText2(cmd, results)
415
416     def getText2(self, cmd, results):
417         if results != SUCCESS and self.incorrectLayoutLines:
418             return self.incorrectLayoutLines
419
420         return [self.name]
421
422
423 class NewRunWebKitTests(RunWebKitTests):
424     command = ["python", "./Tools/Scripts/new-run-webkit-tests",
425                "--no-show-results", "--no-new-test-results",
426                "--verbose", "--results-directory", "layout-test-results",
427                "--builder-name", WithProperties("%(buildername)s"),
428                "--build-number", WithProperties("%(buildnumber)s"),
429                "--master-name", "webkit.org",
430                "--test-results-server", "test-results.appspot.com",
431                WithProperties("--%(configuration)s")]
432
433     def commandComplete(self, cmd):
434         shell.Test.commandComplete(self, cmd)
435
436         logText = cmd.logs['stdio'].getText()
437         self._parseNewRunWebKitTestsOutput(logText)
438
439
440 class RunUnitTests(TestWithFailureCount):
441     name = "run-api-tests"
442     description = ["unit tests running"]
443     descriptionDone = ["unit-tests"]
444     command = ["perl", "./Tools/Scripts/run-api-tests", WithProperties("--%(configuration)s"), "--verbose"]
445     failedTestsFormatString = "%d unit tests failed or timed out"
446
447     def start(self):
448         platform = self.getProperty('platform')
449         if platform == 'win':
450             self.setCommand(self.command + ['--no-build'])
451         if platform.startswith('chromium'):
452             self.setCommand(self.command + ['--chromium'])
453         return shell.Test.start(self)
454
455     def countFailures(self, cmd):
456         log_text = cmd.logs['stdio'].getText()
457         count = 0
458
459         split = re.split(r'\sTests that timed out:\s', log_text)
460         if len(split) > 1:
461             count += len(re.findall(r'^\s+\S+$', split[1], flags=re.MULTILINE))
462
463         split = re.split(r'\sTests that failed:\s', split[0])
464         if len(split) > 1:
465             count += len(re.findall(r'^\s+\S+$', split[1], flags=re.MULTILINE))
466
467         return count
468
469
470 class RunPythonTests(TestWithFailureCount):
471     name = "webkitpy-test"
472     description = ["python-tests running"]
473     descriptionDone = ["python-tests"]
474     command = ["python", "./Tools/Scripts/test-webkitpy"]
475     failedTestsFormatString = "%d python tests failed"
476
477     def countFailures(self, cmd):
478         logText = cmd.logs['stdio'].getText()
479         # We're looking for the line that looks like this: FAILED (failures=2, errors=1)
480         regex = re.compile(r'^FAILED \((?P<counts>[^)]+)\)')
481         for line in logText.splitlines():
482             match = regex.match(line)
483             if not match:
484                 continue
485             return sum(int(component.split('=')[1]) for component in match.group('counts').split(', '))
486         return 0
487
488
489 class RunPerlTests(TestWithFailureCount):
490     name = "webkitperl-test"
491     description = ["perl-tests running"]
492     descriptionDone = ["perl-tests"]
493     command = ["perl", "./Tools/Scripts/test-webkitperl"]
494     failedTestsFormatString = "%d perl tests failed"
495
496     def countFailures(self, cmd):
497         logText = cmd.logs['stdio'].getText()
498         # We're looking for the line that looks like this: Failed 2/19 test programs. 5/363 subtests failed.
499         regex = re.compile(r'^Failed \d+/\d+ test programs\. (?P<count>\d+)/\d+ subtests failed\.')
500         for line in logText.splitlines():
501             match = regex.match(line)
502             if not match:
503                 continue
504             return int(match.group('count'))
505         return 0
506
507
508 class RunBindingsTests(shell.Test):
509     name = "bindings-generation-tests"
510     description = ["bindings-tests running"]
511     descriptionDone = ["bindings-tests"]
512     command = ["python", "./Tools/Scripts/run-bindings-tests"]
513
514
515 class RunGtkAPITests(shell.Test):
516     name = "API tests"
517     description = ["API tests running"]
518     descriptionDone = ["API tests"]
519     command = ["perl", "./Tools/Scripts/run-gtk-tests", WithProperties("--%(configuration)s")]
520
521     def commandComplete(self, cmd):
522         shell.Test.commandComplete(self, cmd)
523
524         logText = cmd.logs['stdio'].getText()
525         incorrectLines = []
526         for line in logText.splitlines():
527             if line.startswith('ERROR'):
528                 incorrectLines.append(line)
529
530         self.incorrectLines = incorrectLines
531
532     def evaluateCommand(self, cmd):
533         if self.incorrectLines:
534             return FAILURE
535
536         if cmd.rc != 0:
537             return FAILURE
538
539         return SUCCESS
540
541     def getText(self, cmd, results):
542         return self.getText2(cmd, results)
543
544     def getText2(self, cmd, results):
545         if results != SUCCESS and self.incorrectLines:
546             return ["%d API tests failed" % len(self.incorrectLines)]
547
548         return [self.name]
549
550 class RunQtAPITests(shell.Test):
551     name = "API tests"
552     description = ["API tests running"]
553     descriptionDone = ["API tests"]
554     command = ["python", "./Tools/Scripts/run-qtwebkit-tests",
555                "--output-file=qt-unit-tests.html", "--do-not-open-results", "--timeout=120",
556                WithProperties("WebKitBuild/%(configuration_pretty)s/Source/WebKit/qt/tests/")]
557
558     def start(self):
559         self.setProperty("configuration_pretty", self.getProperty("configuration").title())
560         return shell.Test.start(self)
561
562     def commandComplete(self, cmd):
563         shell.Test.commandComplete(self, cmd)
564
565         logText = cmd.logs['stdio'].getText()
566         foundItems = re.findall("TOTALS: (?P<passed>\d+) passed, (?P<failed>\d+) failed, (?P<skipped>\d+) skipped", logText)
567
568         self.incorrectTests = 0
569         self.statusLine = []
570
571         if foundItems:
572             self.incorrectTests = int(foundItems[0][1])
573             if self.incorrectTests > 0:
574                 self.statusLine = [
575                     "%s passed, %s failed, %s skipped" % (foundItems[0][0], foundItems[0][1], foundItems[0][2])
576                 ]
577
578     def evaluateCommand(self, cmd):
579         if self.incorrectTests:
580             return WARNINGS
581
582         if cmd.rc != 0:
583             return FAILURE
584
585         return SUCCESS
586
587     def getText(self, cmd, results):
588         return self.getText2(cmd, results)
589
590     def getText2(self, cmd, results):
591         if results != SUCCESS and self.incorrectTests:
592             return self.statusLine
593
594         return [self.name]
595
596 class RunWebKitLeakTests(RunWebKitTests):
597     warnOnWarnings = True
598     def start(self):
599         self.setCommand(self.command + ["--leaks"])
600         return RunWebKitTests.start(self)
601
602
603 class RunWebKit2Tests(RunWebKitTests):
604     def start(self):
605         self.setCommand(self.command + ["--webkit-test-runner"])
606         return RunWebKitTests.start(self)
607
608
609 class RunChromiumWebKitUnitTests(shell.Test):
610     name = "webkit-unit-tests"
611     description = ["webkit-unit-tests running"]
612     descriptionDone = ["webkit-unit-tests"]
613     command = ["perl", "./Tools/Scripts/run-chromium-webkit-unit-tests",
614                WithProperties("--%(configuration)s")]
615
616
617 class ArchiveTestResults(shell.ShellCommand):
618     command = ["python", "./Tools/BuildSlaveSupport/test-result-archive",
619                WithProperties("--platform=%(platform)s"), WithProperties("--%(configuration)s"), "archive"]
620     name = "archive-test-results"
621     description = ["archiving test results"]
622     descriptionDone = ["archived test results"]
623     haltOnFailure = True
624
625
626 class UploadTestResults(transfer.FileUpload):
627     slavesrc = "layout-test-results.zip"
628     masterdest = WithProperties("public_html/results/%(buildername)s/r%(got_revision)s (%(buildnumber)s).zip")
629
630     def __init__(self):
631         transfer.FileUpload.__init__(self, self.slavesrc, self.masterdest, mode=0644)
632
633
634 class ExtractTestResults(master.MasterShellCommand):
635     zipFile = WithProperties("public_html/results/%(buildername)s/r%(got_revision)s (%(buildnumber)s).zip")
636     resultDirectory = WithProperties("public_html/results/%(buildername)s/r%(got_revision)s (%(buildnumber)s)")
637     descriptionDone = ["uploaded results"]
638
639     def __init__(self):
640         master.MasterShellCommand.__init__(self, "")
641
642     def resultDirectoryURL(self):
643         return self.build.getProperties().render(self.resultDirectory).replace("public_html/", "/") + "/"
644
645     def start(self):
646         self.command = ["ditto", "-k", "-x", "-V", self.build.getProperties().render(self.zipFile), self.build.getProperties().render(self.resultDirectory)]
647         return master.MasterShellCommand.start(self)
648
649     def addCustomURLs(self):
650         url = self.resultDirectoryURL() + "results.html"
651         self.addURL("view results", url)
652
653     def finished(self, result):
654         self.addCustomURLs()
655         return master.MasterShellCommand.finished(self, result)
656
657
658 class ExtractTestResultsAndLeaks(ExtractTestResults):
659     def addCustomURLs(self):
660         ExtractTestResults.addCustomURLs(self)
661         url = "/LeaksViewer/?url=" + urllib.quote(self.resultDirectoryURL(), safe="")
662         self.addURL("view leaks", url)
663
664
665 class Factory(factory.BuildFactory):
666     def __init__(self, platform, configuration, architectures, buildOnly, features=None, **kwargs):
667         factory.BuildFactory.__init__(self)
668         self.addStep(ConfigureBuild, platform=platform, configuration=configuration, architecture=" ".join(architectures), buildOnly=buildOnly, features=features)
669         self.addStep(CheckOutSource)
670         self.addStep(KillOldProcesses)
671         if platform == "win":
672             self.addStep(InstallWin32Dependencies)
673         if platform.startswith("chromium"):
674             self.addStep(InstallChromiumDependencies)
675         if platform == "gtk":
676             self.addStep(InstallGtkDependencies)
677
678
679 class BuildFactory(Factory):
680     def __init__(self, platform, configuration, architectures, triggers=None, upload=False, **kwargs):
681         Factory.__init__(self, platform, configuration, architectures, True, **kwargs)
682         self.addStep(CompileWebKit)
683         if triggers or upload:
684             self.addStep(ArchiveBuiltProduct)
685             self.addStep(UploadBuiltProduct)
686         if triggers:
687             self.addStep(trigger.Trigger, schedulerNames=triggers)
688
689 def unitTestsSupported(configuration, platform):
690     return (platform == 'win' or platform.startswith('mac')
691             or (platform.startswith('chromium') and platform != 'chromium-android'))
692
693 class TestFactory(Factory):
694     TestClass = RunWebKitTests
695     ExtractTestResultsClass = ExtractTestResults
696     def __init__(self, platform, configuration, architectures, **kwargs):
697         Factory.__init__(self, platform, configuration, architectures, False, **kwargs)
698         self.addStep(CreateWebKitBuildDirectory)
699         self.addStep(DownloadBuiltProduct)
700         self.addStep(ExtractBuiltProduct)
701         self.addStep(RunJavaScriptCoreTests, buildJSCTool=False)
702         self.addStep(self.TestClass, buildJSCTool=(platform != 'win'))
703
704         if unitTestsSupported(configuration, platform): 
705             self.addStep(RunUnitTests)
706         self.addStep(RunPythonTests)
707         self.addStep(RunPerlTests)
708         self.addStep(RunBindingsTests)
709         self.addStep(ArchiveTestResults)
710         self.addStep(UploadTestResults)
711         self.addStep(self.ExtractTestResultsClass)
712
713 class BuildAndTestFactory(Factory):
714     TestClass = RunWebKitTests
715     ExtractTestResultsClass = ExtractTestResults
716     def __init__(self, platform, configuration, architectures, **kwargs):
717         Factory.__init__(self, platform, configuration, architectures, False, **kwargs)
718         if platform.startswith("chromium"):
719             self.addStep(CleanupChromiumCrashLogs)
720         self.addStep(CompileWebKit)
721         if not platform.startswith("chromium"):
722             self.addStep(RunJavaScriptCoreTests)
723         if platform.startswith("chromium"):
724             self.addStep(RunChromiumWebKitUnitTests)
725         self.addStep(self.TestClass)
726         if unitTestsSupported(configuration, platform): 
727             self.addStep(RunUnitTests)
728         self.addStep(RunPythonTests)
729         # Chromium Win runs in non-Cygwin environment, which is not yet fit
730         # for running tests. This can be removed once bug 48166 is fixed.
731         if platform != "chromium-win":
732             self.addStep(RunPerlTests)
733             self.addStep(RunBindingsTests)
734         self.addStep(ArchiveTestResults)
735         self.addStep(UploadTestResults)
736         self.addStep(self.ExtractTestResultsClass)
737         if platform == "gtk":
738             self.addStep(RunGtkAPITests)
739         if platform == "qt":
740             self.addStep(RunQtAPITests)
741
742 class BuildAndTestLeaksFactory(BuildAndTestFactory):
743     TestClass = RunWebKitLeakTests
744     ExtractTestResultsClass = ExtractTestResultsAndLeaks
745
746 class NewBuildAndTestFactory(BuildAndTestFactory):
747     TestClass = NewRunWebKitTests
748
749 class TestWebKit2Factory(TestFactory):
750     TestClass = RunWebKit2Tests
751
752 class PlatformSpecificScheduler(AnyBranchScheduler):
753     def __init__(self, platform, branch, **kwargs):
754         self.platform = platform
755         filter = ChangeFilter(branch=[branch, None], filter_fn=self.filter)
756         AnyBranchScheduler.__init__(self, name=platform, change_filter=filter, **kwargs)
757
758     def filter(self, change):
759         return wkbuild.should_build(self.platform, change.files)
760
761 trunk_filter = ChangeFilter(branch=["trunk", None])
762
763 def loadBuilderConfig(c):
764     # FIXME: These file handles are leaked.
765     passwords = simplejson.load(open('passwords.json'))
766     config = simplejson.load(open('config.json'))
767
768     c['slaves'] = [BuildSlave(slave['name'], passwords[slave['name']], max_builds=1) for slave in config['slaves']]
769
770     c['schedulers'] = []
771     for scheduler in config['schedulers']:
772         if "change_filter" in scheduler:
773             scheduler["change_filter"] = globals()[scheduler["change_filter"]]
774         kls = globals()[scheduler.pop('type')]
775         # Python 2.6 can't handle unicode keys as keyword arguments:
776         # http://bugs.python.org/issue2646.  Modern versions of simplejson return
777         # unicode strings from simplejson.load, so we map all keys to str objects.
778         scheduler = dict(map(lambda key_value_pair: (str(key_value_pair[0]), key_value_pair[1]), scheduler.items()))
779
780         # BaseScheduler asserts if given unicode objects instead of strs.
781         # http://trac.buildbot.net/ticket/2075
782         scheduler['builderNames'] = map(str, scheduler['builderNames'])
783         c['schedulers'].append(kls(**scheduler))
784
785     c['builders'] = []
786     for builder in config['builders']:
787         for slaveName in builder['slavenames']:
788             for slave in config['slaves']:
789                 if slave['name'] != slaveName or slave['platform'] == '*':
790                     continue
791
792                 if slave['platform'] != builder['platform']:
793                     raise Exception, "Builder %r is for platform %r but has slave %r for platform %r!" % (builder['name'], builder['platform'], slave['name'], slave['platform'])
794
795                 break
796
797         platform = builder['platform']
798
799         factory = globals()["%sFactory" % builder.pop('type')]
800         factoryArgs = []
801         for key in "platform", "configuration", "architectures", "triggers":
802             value = builder.pop(key, None)
803             if value:
804                 factoryArgs.append(value)
805
806         factoryKwArgs = {
807             "features": builder.pop("features", []),
808             "upload": builder.pop("upload", False)
809         }
810
811         builder["factory"] = factory(*factoryArgs, **factoryKwArgs)
812
813         if platform.startswith('chromium'):
814             builder["category"] = 'Chromium'
815         elif platform == 'win':
816             builder["category"] = 'Windows'
817         elif platform.startswith('gtk'):
818             builder["category"] = 'GTK'
819         elif platform.startswith('qt'):
820             builder["category"] = 'Qt'
821         elif platform.startswith('mac'):
822             builder["category"] = 'Mac'
823         else:
824             builder["category"] = 'Others'
825
826         c['builders'].append(builder)
827
828 loadBuilderConfig(c)