f2ce836c8bb8710f063fb228befa891ef950a41f
[WebKit-https.git] / Tools / BuildSlaveSupport / build.webkit.org-config / master.cfg
1 # -*- python -*-
2 # ex: set syntax=python:
3
4 from buildbot.buildslave import BuildSlave
5 from buildbot.changes.pb import PBChangeSource
6 from buildbot.scheduler import AnyBranchScheduler, Triggerable
7 from buildbot.schedulers.filter import ChangeFilter
8 from buildbot.status import html
9 from buildbot.status.web.authz import Authz
10 from buildbot.process import buildstep, factory, properties
11 from buildbot.steps import master, shell, source, transfer, trigger
12 from buildbot.status.builder import SUCCESS, FAILURE, WARNINGS, SKIPPED
13
14 from twisted.internet import defer
15
16 import os
17 import re
18 import simplejson
19 import urllib
20
21 from committer_auth import CommitterAuth
22 from webkitpy.common.config import build as wkbuild
23 from webkitpy.common.net.buildbot import BuildBot as wkbuildbot
24
25
26 c = BuildmasterConfig = {}
27
28 c['change_source'] = PBChangeSource()
29
30 # permissions for WebStatus
31 authz = Authz(
32     auth=CommitterAuth('auth.json'),
33     forceBuild='auth',
34     forceAllBuilds='auth',
35     pingBuilder=True,
36     gracefulShutdown=False,
37     stopBuild=True,
38     stopAllBuilds=True,
39     cancelPendingBuild=True,
40     stopChange=True,
41     cleanShutdown=False)
42
43 c['status'] = []
44 c['status'].append(html.WebStatus(http_port=8710, 
45                                   revlink="http://trac.webkit.org/changeset/%s", 
46                                   authz=authz))
47
48 c['slavePortnum'] = 17000
49 c['projectName'] = "WebKit"
50 c['projectURL'] = "http://webkit.org"
51 c['buildbotURL'] = "http://build.webkit.org/"
52
53 c['buildHorizon'] = 1000
54 c['logHorizon'] = 500
55 c['eventHorizon'] = 200
56 c['buildCacheSize'] = 60
57
58 WithProperties = properties.WithProperties
59
60
61 class TestWithFailureCount(shell.Test):
62     failedTestsFormatString = "%d tests failed"
63
64     def countFailures(self, cmd):
65         return 0
66
67     def commandComplete(self, cmd):
68         shell.Test.commandComplete(self, cmd)
69         self.failedTestCount = self.countFailures(cmd)
70
71     def evaluateCommand(self, cmd):
72         if self.failedTestCount:
73             return FAILURE
74
75         if cmd.rc != 0:
76             return FAILURE
77
78         return SUCCESS
79
80     def getText(self, cmd, results):
81         return self.getText2(cmd, results)
82
83     def getText2(self, cmd, results):
84         if results != SUCCESS and self.failedTestCount:
85             return [self.failedTestsFormatString % self.failedTestCount]
86
87         return [self.name]
88
89
90 class ConfigureBuild(buildstep.BuildStep):
91     name = "configure build"
92     description = ["configuring build"]
93     descriptionDone = ["configured build"]
94     def __init__(self, platform, configuration, architecture, buildOnly, features, *args, **kwargs):
95         buildstep.BuildStep.__init__(self, *args, **kwargs)
96         self.platform = platform.split('-', 1)[0]
97         self.fullPlatform = platform
98         self.configuration = configuration
99         self.architecture = architecture
100         self.buildOnly = buildOnly
101         self.features = features;
102         self.addFactoryArguments(platform=platform, configuration=configuration, architecture=architecture, buildOnly=buildOnly, features=features)
103
104     def start(self):
105         self.setProperty("platform", self.platform)
106         self.setProperty("fullPlatform", self.fullPlatform)
107         self.setProperty("configuration", self.configuration)
108         self.setProperty("architecture", self.architecture)
109         self.setProperty("buildOnly", self.buildOnly)
110         self.setProperty("features", self.features)
111         self.finished(SUCCESS)
112         return defer.succeed(None)
113
114
115 class CheckOutSource(source.SVN):
116     baseURL = "http://svn.webkit.org/repository/webkit/"
117     mode = "update"
118     def __init__(self, *args, **kwargs):
119         source.SVN.__init__(self, baseURL=self.baseURL, defaultBranch="trunk", mode=self.mode, *args, **kwargs)
120
121
122 class InstallWin32Dependencies(shell.Compile):
123     description = ["installing dependencies"]
124     descriptionDone = ["installed dependencies"]
125     command = ["perl", "./Tools/Scripts/update-webkit-auxiliary-libs"]
126
127 class KillOldProcesses(shell.Compile):
128     name = "kill old processes"
129     description = ["killing old processes"]
130     descriptionDone = ["killed old processes"]
131     command = ["python", "./Tools/BuildSlaveSupport/kill-old-processes"]
132
133 class InstallGtkDependencies(shell.ShellCommand):
134     name = "jhbuild"
135     description = ["updating gtk dependencies"]
136     descriptionDone = ["updated gtk dependencies"]
137     command = ["perl", "./Tools/Scripts/update-webkitgtk-libs"]
138     haltOnFailure = True
139
140 class InstallChromiumDependencies(shell.ShellCommand):
141     name = "gclient"
142     description = ["updating chromium dependencies"]
143     descriptionDone = ["updated chromium dependencies"]
144     command = ["perl", "./Tools/Scripts/update-webkit-chromium", "--force"]
145     haltOnFailure = True
146     def start(self):
147         if self.getProperty('fullPlatform') == "chromium-android":
148             self.setCommand(self.command + ['--chromium-android'])
149
150         return shell.ShellCommand.start(self)
151
152 class CleanupChromiumCrashLogs(shell.ShellCommand):
153     name = "cleanup crash logs"
154     description = ["removing crash logs"]
155     descriptionDone = ["removed crash logs"]
156     command = ["python", "./Tools/BuildSlaveSupport/chromium/remove-crash-logs"]
157     haltOnFailure = False
158
159
160 def appendCustomBuildFlags(step, platform, fullPlatform=""):
161     if fullPlatform == "chromium-android":
162         step.setCommand(step.command + ['--chromium-android'])
163     elif platform in ('chromium', 'efl', 'gtk', 'qt', 'wincairo', 'wince', 'wx'):
164         step.setCommand(step.command + ['--' + platform])
165
166
167 class CompileWebKit(shell.Compile):
168     command = ["perl", "./Tools/Scripts/build-webkit", WithProperties("--%(configuration)s")]
169     env = {'MFLAGS':''}
170     name = "compile-webkit"
171     description = ["compiling"]
172     descriptionDone = ["compiled"]
173     warningPattern = ".*arning: .*"
174
175     def start(self):
176         platform = self.getProperty('platform')
177         buildOnly = self.getProperty('buildOnly')
178         if platform == 'mac' and buildOnly:
179             self.setCommand(self.command + ['DEBUG_INFORMATION_FORMAT=dwarf-with-dsym'])
180
181         appendCustomBuildFlags(self, platform, self.getProperty('fullPlatform'))
182
183         features = self.getProperty('features')
184         for feature in features:
185             self.setCommand(self.command + ['--' + feature])
186
187         return shell.Compile.start(self)
188
189
190 class ArchiveBuiltProduct(shell.ShellCommand):
191     command = ["python", "./Tools/BuildSlaveSupport/built-product-archive",
192                WithProperties("--platform=%(platform)s"), WithProperties("--%(configuration)s"), "archive"]
193     name = "archive-built-product"
194     description = ["archiving built product"]
195     descriptionDone = ["archived built product"]
196     haltOnFailure = True
197
198
199 class ExtractBuiltProduct(shell.ShellCommand):
200     command = ["python", "./Tools/BuildSlaveSupport/built-product-archive",
201                WithProperties("--platform=%(platform)s"), WithProperties("--%(configuration)s"), "extract"]
202     name = "extract-built-product"
203     description = ["extracting built product"]
204     descriptionDone = ["extracted built product"]
205     haltOnFailure = True
206
207
208 class UploadBuiltProduct(transfer.FileUpload):
209     slavesrc = WithProperties("WebKitBuild/%(configuration)s.zip")
210     masterdest = WithProperties("archives/%(fullPlatform)s-%(architecture)s-%(configuration)s%(extraFeatures)s/%(got_revision)s.zip",
211                                 extraFeatures=lambda properties: UploadBuiltProduct.determineExtraFeatures(properties))
212     haltOnFailure = True
213
214     def __init__(self):
215         transfer.FileUpload.__init__(self, self.slavesrc, self.masterdest, mode=0644)
216
217     @staticmethod
218     def determineExtraFeatures(properties):
219         if not properties.has_key("features"):
220             return ''
221         features = properties.getProperty("features")
222         if not len(features):
223             return ''
224         result = '-with-' + '-and-'.join(features)
225         # Just to make sure no feature will add invalid chars, we allow only letters here.
226         result = re.sub('[^a-zA-Z]', '-', result)
227         return result
228
229
230 class DownloadBuiltProduct(shell.ShellCommand):
231     command = ["curl", "--output", WithProperties("WebKitBuild/%(configuration)s.zip"), WithProperties(c["buildbotURL"] + "archives/%(fullPlatform)s-%(architecture)s-%(configuration)s/%(got_revision)s.zip")]
232     name = "download-built-product"
233     description = ["downloading built product"]
234     descriptionDone = ["downloaded built product"]
235     haltOnFailure = True
236     flunkOnFailure = True
237
238
239 class CreateWebKitBuildDirectory(shell.ShellCommand):
240     command = ["mkdir", "-p", "WebKitBuild"]
241     name = "create-webkitbuild-dir"
242     description = ["creating WebKitBuild directory"]
243     descriptionDone = ["created WebKitBuild directory"]
244     haltOnFailure = True
245     flunkOnFailure = True
246
247
248 class RunJavaScriptCoreTests(shell.Test):
249     name = "jscore-test"
250     description = ["jscore-tests running"]
251     descriptionDone = ["jscore-tests"]
252     command = ["perl", "./Tools/Scripts/run-javascriptcore-tests", WithProperties("--%(configuration)s")]
253     logfiles = {'actual.html (source)': 'Source/JavaScriptCore/tests/mozilla/actual.html'}
254
255     def __init__(self, buildJSCTool=True, *args, **kwargs):
256         self.buildJSCTool = buildJSCTool
257         shell.Test.__init__(self, *args, **kwargs)
258         self.addFactoryArguments(buildJSCTool=buildJSCTool)
259
260     def start(self):
261         appendCustomBuildFlags(self, self.getProperty('platform'))
262         if not self.buildJSCTool:
263             self.setCommand(self.command + ['--no-build'])
264         return shell.Test.start(self)
265
266     def commandComplete(self, cmd):
267         shell.Test.commandComplete(self, cmd)
268
269         logText = cmd.logs['stdio'].getText()
270         statusLines = [line for line in logText.splitlines() if line.find('regression') >= 0 and line.find(' found.') >= 0]
271         if statusLines and statusLines[0].split()[0] != '0':
272             self.regressionLine = statusLines[0]
273         else:
274             self.regressionLine = None
275
276         if 'actual.html (source)' in cmd.logs:
277             self.addHTMLLog('actual.html', cmd.logs['actual.html (source)'].getText())
278
279     def evaluateCommand(self, cmd):
280         if self.regressionLine:
281             return FAILURE
282
283         if cmd.rc != 0:
284             return FAILURE
285
286         return SUCCESS
287
288     def getText(self, cmd, results):
289         return self.getText2(cmd, results)
290
291     def getText2(self, cmd, results):
292         if results != SUCCESS and self.regressionLine:
293             return [self.name, self.regressionLine]
294
295         return [self.name]
296
297
298 class RunWebKitTests(shell.Test):
299     name = "layout-test"
300     description = ["layout-tests running"]
301     descriptionDone = ["layout-tests"]
302     command = ["perl", "./Tools/Scripts/run-webkit-tests",
303                "--no-launch-safari",
304                "--no-new-test-results",
305                "--no-sample-on-timeout",
306                "--results-directory", "layout-test-results",
307                "--use-remote-links-to-tests",
308                "--builder-name", WithProperties("%(buildername)s"),
309                "--build-number", WithProperties("%(buildnumber)s"),
310                "--master-name", "webkit.org",
311                "--test-results-server", "test-results.appspot.com",
312                WithProperties("--%(configuration)s"),
313                "--exit-after-n-crashes-or-timeouts", "20",
314                "--exit-after-n-failures", "500"]
315
316     def __init__(self, buildJSCTool=True, *args, **kwargs):
317         self.buildJSCTool = buildJSCTool
318         shell.Test.__init__(self, *args, **kwargs)
319         self.addFactoryArguments(buildJSCTool=buildJSCTool)
320
321     def start(self):
322         platform = self.getProperty('platform')
323         appendCustomBuildFlags(self, platform, self.getProperty('fullPlatform'))
324         if platform == "win":
325             rootArgument = ['--root=' + os.path.join("WebKitBuild", self.getProperty('configuration'), "bin")]
326         else:
327             rootArgument = ['--root=WebKitBuild/bin']
328         if not self.buildJSCTool:
329             self.setCommand(self.command + rootArgument)
330         return shell.Test.start(self)
331
332     def _parseOldRunWebKitTestsOutput(self, logText):
333         incorrectLayoutLines = []
334         for line in logText.splitlines():
335             if line.find('had incorrect layout') >= 0 or line.find('were new') >= 0 or line.find('was new') >= 0:
336                 incorrectLayoutLines.append(line)
337             elif line.find('test case') >= 0 and (line.find(' crashed') >= 0 or line.find(' timed out') >= 0):
338                 incorrectLayoutLines.append(line)
339             elif line.startswith("WARNING:") and line.find(' leak') >= 0:
340                 incorrectLayoutLines.append(line.replace('WARNING: ', ''))
341             elif line.find('Exiting early') >= 0:
342                 incorrectLayoutLines.append(line)
343
344             # FIXME: Detect and summarize leaks of RefCounted objects
345
346         self.incorrectLayoutLines = incorrectLayoutLines
347
348     # FIXME: This will break if new-run-webkit-tests changes its default log formatter.
349     nrwt_log_message_regexp = re.compile(r'(?P<log_prefix>.*) (?P<log_level>DEBUG|INFO) (?P<message>.*)')
350
351     def _strip_python_logging_prefix(self, line):
352         match_object = self.nrwt_log_message_regexp.match(line)
353         if match_object:
354             return match_object.group('message')
355         return line
356
357     def _parseNewRunWebKitTestsOutput(self, logText):
358         incorrectLayoutLines = []
359         expressions = [
360             ('flakes', re.compile(r'Unexpected flakiness.+:?\s*\((\d+)\)')),
361             ('new passes', re.compile(r'Expected to .+, but passed:\s+\((\d+)\)')),
362             ('missing results', re.compile(r'no expected results found\s*:\s+\((\d+)\)')),
363             ('failures', re.compile(r'Regressions: Unexpected.+:?\s*\((\d+)\)')),
364         ]
365         testFailures = {}
366
367         for line in logText.splitlines():
368             if line.find('Exiting early') >= 0 or line.find('leaks found') >= 0:
369                 incorrectLayoutLines.append(self._strip_python_logging_prefix(line))
370                 continue
371             for name, expression in expressions:
372                 match = expression.search(line)
373
374                 if match:
375                     testFailures[name] = testFailures.get(name, 0) + int(match.group(1))
376                     break
377
378                 # FIXME: Parse file names and put them in results
379
380         for name in testFailures:
381             incorrectLayoutLines.append(str(testFailures[name]) + ' ' + name)
382
383         self.incorrectLayoutLines = incorrectLayoutLines
384
385     def commandComplete(self, cmd):
386         shell.Test.commandComplete(self, cmd)
387
388         logText = cmd.logs['stdio'].getText()
389         if logText.find("Collecting tests ...") >= 0:
390             self._parseNewRunWebKitTestsOutput(logText)
391         else:
392             self._parseOldRunWebKitTestsOutput(logText)
393
394     def evaluateCommand(self, cmd):
395         result = SUCCESS
396
397         if self.incorrectLayoutLines:
398             if len(self.incorrectLayoutLines) == 1:
399                 line = self.incorrectLayoutLines[0]
400                 if line.find('were new') >= 0 or line.find('was new') >= 0 or line.find(' leak') >= 0:
401                     return WARNINGS
402
403             for line in self.incorrectLayoutLines:
404                 if line.find('flakes') >= 0 or line.find('new passes') >= 0 or line.find('missing results') >= 0:
405                     result = WARNINGS
406                 else:
407                     return FAILURE
408
409         if cmd.rc != 0:
410             return FAILURE
411
412         return result
413
414     def getText(self, cmd, results):
415         return self.getText2(cmd, results)
416
417     def getText2(self, cmd, results):
418         if results != SUCCESS and self.incorrectLayoutLines:
419             return self.incorrectLayoutLines
420
421         return [self.name]
422
423
424 class NewRunWebKitTests(RunWebKitTests):
425     command = ["python", "./Tools/Scripts/new-run-webkit-tests",
426                "--no-show-results", "--no-new-test-results",
427                "--verbose", "--results-directory", "layout-test-results",
428                "--builder-name", WithProperties("%(buildername)s"),
429                "--build-number", WithProperties("%(buildnumber)s"),
430                "--master-name", "webkit.org",
431                "--test-results-server", "test-results.appspot.com",
432                WithProperties("--%(configuration)s")]
433
434     def commandComplete(self, cmd):
435         shell.Test.commandComplete(self, cmd)
436
437         logText = cmd.logs['stdio'].getText()
438         self._parseNewRunWebKitTestsOutput(logText)
439
440
441 class RunUnitTests(TestWithFailureCount):
442     name = "run-api-tests"
443     description = ["unit tests running"]
444     descriptionDone = ["unit-tests"]
445     command = ["perl", "./Tools/Scripts/run-api-tests", WithProperties("--%(configuration)s"), "--verbose"]
446     failedTestsFormatString = "%d unit tests failed or timed out"
447
448     def start(self):
449         platform = self.getProperty('platform')
450         if platform == 'win':
451             self.setCommand(self.command + ['--no-build'])
452         if platform.startswith('chromium'):
453             self.setCommand(self.command + ['--chromium'])
454         return shell.Test.start(self)
455
456     def countFailures(self, cmd):
457         log_text = cmd.logs['stdio'].getText()
458         count = 0
459
460         split = re.split(r'\sTests that timed out:\s', log_text)
461         if len(split) > 1:
462             count += len(re.findall(r'^\s+\S+$', split[1], flags=re.MULTILINE))
463
464         split = re.split(r'\sTests that failed:\s', split[0])
465         if len(split) > 1:
466             count += len(re.findall(r'^\s+\S+$', split[1], flags=re.MULTILINE))
467
468         return count
469
470
471 class RunPythonTests(TestWithFailureCount):
472     name = "webkitpy-test"
473     description = ["python-tests running"]
474     descriptionDone = ["python-tests"]
475     command = ["python", "./Tools/Scripts/test-webkitpy"]
476     failedTestsFormatString = "%d python tests failed"
477
478     def countFailures(self, cmd):
479         logText = cmd.logs['stdio'].getText()
480         # We're looking for the line that looks like this: FAILED (failures=2, errors=1)
481         regex = re.compile(r'^FAILED \((?P<counts>[^)]+)\)')
482         for line in logText.splitlines():
483             match = regex.match(line)
484             if not match:
485                 continue
486             return sum(int(component.split('=')[1]) for component in match.group('counts').split(', '))
487         return 0
488
489
490 class RunPerlTests(TestWithFailureCount):
491     name = "webkitperl-test"
492     description = ["perl-tests running"]
493     descriptionDone = ["perl-tests"]
494     command = ["perl", "./Tools/Scripts/test-webkitperl"]
495     failedTestsFormatString = "%d perl tests failed"
496
497     def countFailures(self, cmd):
498         logText = cmd.logs['stdio'].getText()
499         # We're looking for the line that looks like this: Failed 2/19 test programs. 5/363 subtests failed.
500         regex = re.compile(r'^Failed \d+/\d+ test programs\. (?P<count>\d+)/\d+ subtests failed\.')
501         for line in logText.splitlines():
502             match = regex.match(line)
503             if not match:
504                 continue
505             return int(match.group('count'))
506         return 0
507
508
509 class RunBindingsTests(shell.Test):
510     name = "bindings-generation-tests"
511     description = ["bindings-tests running"]
512     descriptionDone = ["bindings-tests"]
513     command = ["python", "./Tools/Scripts/run-bindings-tests"]
514
515
516 class RunGtkAPITests(shell.Test):
517     name = "API tests"
518     description = ["API tests running"]
519     descriptionDone = ["API tests"]
520     command = ["perl", "./Tools/Scripts/run-gtk-tests", WithProperties("--%(configuration)s")]
521
522     def commandComplete(self, cmd):
523         shell.Test.commandComplete(self, cmd)
524
525         logText = cmd.logs['stdio'].getText()
526         incorrectLines = []
527         for line in logText.splitlines():
528             if line.startswith('ERROR'):
529                 incorrectLines.append(line)
530
531         self.incorrectLines = incorrectLines
532
533     def evaluateCommand(self, cmd):
534         if self.incorrectLines:
535             return FAILURE
536
537         if cmd.rc != 0:
538             return FAILURE
539
540         return SUCCESS
541
542     def getText(self, cmd, results):
543         return self.getText2(cmd, results)
544
545     def getText2(self, cmd, results):
546         if results != SUCCESS and self.incorrectLines:
547             return ["%d API tests failed" % len(self.incorrectLines)]
548
549         return [self.name]
550
551 class RunQtAPITests(shell.Test):
552     name = "API tests"
553     description = ["API tests running"]
554     descriptionDone = ["API tests"]
555     command = ["python", "./Tools/Scripts/run-qtwebkit-tests",
556                "--output-file=qt-unit-tests.html", "--do-not-open-results", "--timeout=120",
557                WithProperties("WebKitBuild/%(configuration_pretty)s/Source/WebKit/qt/tests/")]
558
559     def start(self):
560         self.setProperty("configuration_pretty", self.getProperty("configuration").title())
561         return shell.Test.start(self)
562
563     def commandComplete(self, cmd):
564         shell.Test.commandComplete(self, cmd)
565
566         logText = cmd.logs['stdio'].getText()
567         foundItems = re.findall("TOTALS: (?P<passed>\d+) passed, (?P<failed>\d+) failed, (?P<skipped>\d+) skipped", logText)
568
569         self.incorrectTests = 0
570         self.statusLine = []
571
572         if foundItems:
573             self.incorrectTests = int(foundItems[0][1])
574             if self.incorrectTests > 0:
575                 self.statusLine = [
576                     "%s passed, %s failed, %s skipped" % (foundItems[0][0], foundItems[0][1], foundItems[0][2])
577                 ]
578
579     def evaluateCommand(self, cmd):
580         if self.incorrectTests:
581             return WARNINGS
582
583         if cmd.rc != 0:
584             return FAILURE
585
586         return SUCCESS
587
588     def getText(self, cmd, results):
589         return self.getText2(cmd, results)
590
591     def getText2(self, cmd, results):
592         if results != SUCCESS and self.incorrectTests:
593             return self.statusLine
594
595         return [self.name]
596
597 class RunWebKitLeakTests(RunWebKitTests):
598     warnOnWarnings = True
599     def start(self):
600         self.setCommand(self.command + ["--leaks"])
601         return RunWebKitTests.start(self)
602
603
604 class RunWebKit2Tests(RunWebKitTests):
605     def start(self):
606         self.setCommand(self.command + ["--webkit-test-runner"])
607         return RunWebKitTests.start(self)
608
609
610 class RunChromiumWebKitUnitTests(shell.Test):
611     name = "webkit-unit-tests"
612     description = ["webkit-unit-tests running"]
613     descriptionDone = ["webkit-unit-tests"]
614     command = ["perl", "./Tools/Scripts/run-chromium-webkit-unit-tests",
615                WithProperties("--%(configuration)s")]
616
617
618 class ArchiveTestResults(shell.ShellCommand):
619     command = ["python", "./Tools/BuildSlaveSupport/test-result-archive",
620                WithProperties("--platform=%(platform)s"), WithProperties("--%(configuration)s"), "archive"]
621     name = "archive-test-results"
622     description = ["archiving test results"]
623     descriptionDone = ["archived test results"]
624     haltOnFailure = True
625
626
627 class UploadTestResults(transfer.FileUpload):
628     slavesrc = "layout-test-results.zip"
629     masterdest = WithProperties("public_html/results/%(buildername)s/r%(got_revision)s (%(buildnumber)s).zip")
630
631     def __init__(self):
632         transfer.FileUpload.__init__(self, self.slavesrc, self.masterdest, mode=0644)
633
634
635 class ExtractTestResults(master.MasterShellCommand):
636     zipFile = WithProperties("public_html/results/%(buildername)s/r%(got_revision)s (%(buildnumber)s).zip")
637     resultDirectory = WithProperties("public_html/results/%(buildername)s/r%(got_revision)s (%(buildnumber)s)")
638     descriptionDone = ["uploaded results"]
639
640     def __init__(self):
641         master.MasterShellCommand.__init__(self, "")
642
643     def resultDirectoryURL(self):
644         return self.build.getProperties().render(self.resultDirectory).replace("public_html/", "/") + "/"
645
646     def start(self):
647         self.command = ["ditto", "-k", "-x", "-V", self.build.getProperties().render(self.zipFile), self.build.getProperties().render(self.resultDirectory)]
648         return master.MasterShellCommand.start(self)
649
650     def addCustomURLs(self):
651         url = self.resultDirectoryURL() + "results.html"
652         self.addURL("view results", url)
653
654     def finished(self, result):
655         self.addCustomURLs()
656         return master.MasterShellCommand.finished(self, result)
657
658
659 class ExtractTestResultsAndLeaks(ExtractTestResults):
660     def addCustomURLs(self):
661         ExtractTestResults.addCustomURLs(self)
662         url = "/LeaksViewer/?url=" + urllib.quote(self.resultDirectoryURL(), safe="")
663         self.addURL("view leaks", url)
664
665
666 class Factory(factory.BuildFactory):
667     def __init__(self, platform, configuration, architectures, buildOnly, features=None, **kwargs):
668         factory.BuildFactory.__init__(self)
669         self.addStep(ConfigureBuild, platform=platform, configuration=configuration, architecture=" ".join(architectures), buildOnly=buildOnly, features=features)
670         self.addStep(CheckOutSource)
671         self.addStep(KillOldProcesses)
672         if platform == "win":
673             self.addStep(InstallWin32Dependencies)
674         if platform.startswith("chromium"):
675             self.addStep(InstallChromiumDependencies)
676         if platform == "gtk":
677             self.addStep(InstallGtkDependencies)
678
679
680 class BuildFactory(Factory):
681     def __init__(self, platform, configuration, architectures, triggers=None, upload=False, **kwargs):
682         Factory.__init__(self, platform, configuration, architectures, True, **kwargs)
683         self.addStep(CompileWebKit)
684         if triggers or upload:
685             self.addStep(ArchiveBuiltProduct)
686             self.addStep(UploadBuiltProduct)
687         if triggers:
688             self.addStep(trigger.Trigger, schedulerNames=triggers)
689
690 def unitTestsSupported(configuration, platform):
691     return (platform == 'win' or platform.startswith('mac')
692             or (platform.startswith('chromium') and platform != 'chromium-android'))
693
694 class TestFactory(Factory):
695     TestClass = RunWebKitTests
696     ExtractTestResultsClass = ExtractTestResults
697     def __init__(self, platform, configuration, architectures, **kwargs):
698         Factory.__init__(self, platform, configuration, architectures, False, **kwargs)
699         self.addStep(CreateWebKitBuildDirectory)
700         self.addStep(DownloadBuiltProduct)
701         self.addStep(ExtractBuiltProduct)
702         self.addStep(RunJavaScriptCoreTests, buildJSCTool=False)
703         self.addStep(self.TestClass, buildJSCTool=(platform != 'win'))
704
705         if unitTestsSupported(configuration, platform): 
706             self.addStep(RunUnitTests)
707         self.addStep(RunPythonTests)
708         self.addStep(RunPerlTests)
709         self.addStep(RunBindingsTests)
710         self.addStep(ArchiveTestResults)
711         self.addStep(UploadTestResults)
712         self.addStep(self.ExtractTestResultsClass)
713
714 class BuildAndTestFactory(Factory):
715     TestClass = RunWebKitTests
716     ExtractTestResultsClass = ExtractTestResults
717     def __init__(self, platform, configuration, architectures, **kwargs):
718         Factory.__init__(self, platform, configuration, architectures, False, **kwargs)
719         if platform.startswith("chromium"):
720             self.addStep(CleanupChromiumCrashLogs)
721         self.addStep(CompileWebKit)
722         if not platform.startswith("chromium"):
723             self.addStep(RunJavaScriptCoreTests)
724         if platform.startswith("chromium"):
725             self.addStep(RunChromiumWebKitUnitTests)
726         self.addStep(self.TestClass)
727         if unitTestsSupported(configuration, platform): 
728             self.addStep(RunUnitTests)
729         self.addStep(RunPythonTests)
730         # Chromium Win runs in non-Cygwin environment, which is not yet fit
731         # for running tests. This can be removed once bug 48166 is fixed.
732         if platform != "chromium-win":
733             self.addStep(RunPerlTests)
734             self.addStep(RunBindingsTests)
735         self.addStep(ArchiveTestResults)
736         self.addStep(UploadTestResults)
737         self.addStep(self.ExtractTestResultsClass)
738         if platform == "gtk":
739             self.addStep(RunGtkAPITests)
740         if platform == "qt":
741             self.addStep(RunQtAPITests)
742
743 class BuildAndTestLeaksFactory(BuildAndTestFactory):
744     TestClass = RunWebKitLeakTests
745     ExtractTestResultsClass = ExtractTestResultsAndLeaks
746
747 class NewBuildAndTestFactory(BuildAndTestFactory):
748     TestClass = NewRunWebKitTests
749
750 class TestWebKit2Factory(TestFactory):
751     TestClass = RunWebKit2Tests
752
753 class PlatformSpecificScheduler(AnyBranchScheduler):
754     def __init__(self, platform, branch, **kwargs):
755         self.platform = platform
756         filter = ChangeFilter(branch=[branch, None], filter_fn=self.filter)
757         AnyBranchScheduler.__init__(self, name=platform, change_filter=filter, **kwargs)
758
759     def filter(self, change):
760         return wkbuild.should_build(self.platform, change.files)
761
762 trunk_filter = ChangeFilter(branch=["trunk", None])
763
764 def loadBuilderConfig(c):
765     # FIXME: These file handles are leaked.
766     passwords = simplejson.load(open('passwords.json'))
767     config = simplejson.load(open('config.json'))
768
769     # use webkitpy's buildbot module to test for core builders
770     wkbb = wkbuildbot()
771
772     c['slaves'] = [BuildSlave(slave['name'], passwords[slave['name']], max_builds=1) for slave in config['slaves']]
773
774     c['schedulers'] = []
775     for scheduler in config['schedulers']:
776         if "change_filter" in scheduler:
777             scheduler["change_filter"] = globals()[scheduler["change_filter"]]
778         kls = globals()[scheduler.pop('type')]
779         # Python 2.6 can't handle unicode keys as keyword arguments:
780         # http://bugs.python.org/issue2646.  Modern versions of simplejson return
781         # unicode strings from simplejson.load, so we map all keys to str objects.
782         scheduler = dict(map(lambda key_value_pair: (str(key_value_pair[0]), key_value_pair[1]), scheduler.items()))
783
784         # BaseScheduler asserts if given unicode objects instead of strs.
785         # http://trac.buildbot.net/ticket/2075
786         scheduler['builderNames'] = map(str, scheduler['builderNames'])
787         c['schedulers'].append(kls(**scheduler))
788
789     c['builders'] = []
790     for builder in config['builders']:
791         for slaveName in builder['slavenames']:
792             for slave in config['slaves']:
793                 if slave['name'] != slaveName or slave['platform'] == '*':
794                     continue
795
796                 if slave['platform'] != builder['platform']:
797                     raise Exception, "Builder %r is for platform %r but has slave %r for platform %r!" % (builder['name'], builder['platform'], slave['name'], slave['platform'])
798
799                 break
800
801         factory = globals()["%sFactory" % builder.pop('type')]
802         factoryArgs = []
803         for key in "platform", "configuration", "architectures", "triggers":
804             value = builder.pop(key, None)
805             if value:
806                 factoryArgs.append(value)
807
808         factoryKwArgs = {
809             "features": builder.pop("features", []),
810             "upload": builder.pop("upload", False)
811         }
812
813         builder["factory"] = factory(*factoryArgs, **factoryKwArgs)
814
815         builder["category"] = "noncore"
816         if wkbb._is_core_builder(builder['name']):
817             builder["category"] = "core"
818
819         c['builders'].append(builder)
820
821 loadBuilderConfig(c)