2 # ex: set syntax=python:
4 from buildbot.buildslave import BuildSlave
5 from buildbot.changes.pb import PBChangeSource
6 from buildbot.scheduler import AnyBranchScheduler, Triggerable
7 from buildbot.schedulers.forcesched import FixedParameter, ForceScheduler, StringParameter
8 from buildbot.schedulers.filter import ChangeFilter
9 from buildbot.status import html
10 from buildbot.status.web.authz import Authz
11 from buildbot.process import buildstep, factory, properties
12 from buildbot.steps import master, shell, source, transfer, trigger
13 from buildbot.status.builder import SUCCESS, FAILURE, WARNINGS, SKIPPED
15 from twisted.internet import defer
23 from committer_auth import CommitterAuth
27 c = BuildmasterConfig = {}
29 c['change_source'] = PBChangeSource()
31 # permissions for WebStatus
33 auth=CommitterAuth('auth.json'),
35 forceAllBuilds='auth',
37 gracefulShutdown=False,
40 cancelPendingBuild='auth',
45 c['status'].append(html.WebStatus(http_port=8710,
46 revlink="http://trac.webkit.org/changeset/%s",
47 changecommentlink=(r"(https://bugs\.webkit\.org/show_bug\.cgi\?id=|webkit\.org/b/)(\d+)", r"https://bugs.webkit.org/show_bug.cgi?id=\2"),
50 c['slavePortnum'] = 17000
51 c['projectName'] = "WebKit"
52 c['projectURL'] = "http://webkit.org"
53 c['buildbotURL'] = "http://build.webkit.org/"
55 c['buildHorizon'] = 1000
57 c['eventHorizon'] = 200
58 c['buildCacheSize'] = 60
60 WithProperties = properties.WithProperties
63 class TestWithFailureCount(shell.Test):
64 failedTestsFormatString = "%d tests failed"
66 def countFailures(self, cmd):
69 def commandComplete(self, cmd):
70 shell.Test.commandComplete(self, cmd)
71 self.failedTestCount = self.countFailures(cmd)
73 def evaluateCommand(self, cmd):
74 if self.failedTestCount:
82 def getText(self, cmd, results):
83 return self.getText2(cmd, results)
85 def getText2(self, cmd, results):
86 if results != SUCCESS and self.failedTestCount:
87 return [self.failedTestsFormatString % self.failedTestCount]
92 class ConfigureBuild(buildstep.BuildStep):
93 name = "configure build"
94 description = ["configuring build"]
95 descriptionDone = ["configured build"]
96 def __init__(self, platform, configuration, architecture, buildOnly, SVNMirror, *args, **kwargs):
97 buildstep.BuildStep.__init__(self, *args, **kwargs)
98 self.platform = platform.split('-', 1)[0]
99 self.fullPlatform = platform
100 self.configuration = configuration
101 self.architecture = architecture
102 self.buildOnly = buildOnly
103 self.SVNMirror = SVNMirror
104 self.addFactoryArguments(platform=platform, configuration=configuration, architecture=architecture, buildOnly=buildOnly, SVNMirror=SVNMirror)
107 self.setProperty("platform", self.platform)
108 self.setProperty("fullPlatform", self.fullPlatform)
109 self.setProperty("configuration", self.configuration)
110 self.setProperty("architecture", self.architecture)
111 self.setProperty("buildOnly", self.buildOnly)
112 self.setProperty("shouldAbortEarly", True)
113 self.setProperty("SVNMirror", self.SVNMirror)
114 self.finished(SUCCESS)
115 return defer.succeed(None)
118 class CheckOutSource(source.SVN):
120 def __init__(self, SVNMirror, **kwargs):
121 kwargs['baseURL'] = SVNMirror or "http://svn.webkit.org/repository/webkit/"
122 kwargs['defaultBranch'] = "trunk"
123 kwargs['mode'] = self.mode
124 source.SVN.__init__(self, **kwargs)
125 self.addFactoryArguments(SVNMirror=SVNMirror)
127 class WaitForSVNServer(shell.ShellCommand):
128 name = "wait-for-svn-server"
129 command = ["python", "./Tools/BuildSlaveSupport/wait-for-SVN-server.py", "-r", WithProperties("%(revision)s"), "-s", WithProperties("%(SVNMirror)s")]
130 description = ["waiting for SVN server"]
131 descriptionDone = ["SVN server is ready"]
134 class InstallWin32Dependencies(shell.Compile):
135 description = ["installing dependencies"]
136 descriptionDone = ["installed dependencies"]
137 command = ["perl", "./Tools/Scripts/update-webkit-auxiliary-libs"]
139 class KillOldProcesses(shell.Compile):
140 name = "kill old processes"
141 description = ["killing old processes"]
142 descriptionDone = ["killed old processes"]
143 command = ["python", "./Tools/BuildSlaveSupport/kill-old-processes"]
145 class DeleteStaleBuildFiles(shell.Compile):
146 name = "delete stale build files"
147 description = ["deleting stale build files"]
148 descriptionDone = ["delete stale build files"]
149 command = ["python", "./Tools/BuildSlaveSupport/delete-stale-build-files", WithProperties("--platform=%(fullPlatform)s"), WithProperties("--%(configuration)s")]
151 class InstallEflDependencies(shell.ShellCommand):
153 description = ["updating efl dependencies"]
154 descriptionDone = ["updated efl dependencies"]
155 command = ["perl", "./Tools/Scripts/update-webkitefl-libs"]
158 class InstallGtkDependencies(shell.ShellCommand):
160 description = ["updating gtk dependencies"]
161 descriptionDone = ["updated gtk dependencies"]
162 command = ["perl", "./Tools/Scripts/update-webkitgtk-libs"]
165 def appendCustomBuildFlags(step, platform, fullPlatform=""):
166 if platform in ('efl', 'gtk', 'qt', 'wincairo', 'wince', 'wx'):
167 step.setCommand(step.command + ['--' + platform])
169 class CompileWebKit(shell.Compile):
170 command = ["perl", "./Tools/Scripts/build-webkit", WithProperties("--%(configuration)s")]
172 name = "compile-webkit"
173 description = ["compiling"]
174 descriptionDone = ["compiled"]
175 warningPattern = ".*arning: .*"
178 platform = self.getProperty('platform')
179 buildOnly = self.getProperty('buildOnly')
180 architecture = self.getProperty('architecture')
181 if platform == 'mac' and architecture == 'i386':
182 self.setCommand(self.command + ['--32-bit'])
183 if platform == 'mac' and buildOnly:
184 self.setCommand(self.command + ['DEBUG_INFORMATION_FORMAT=dwarf-with-dsym'])
186 appendCustomBuildFlags(self, platform, self.getProperty('fullPlatform'))
188 return shell.Compile.start(self)
191 class CompileWebKit1Only(CompileWebKit):
192 command = ["perl", "./Tools/Scripts/build-webkit", "--no-webkit2", WithProperties("--%(configuration)s")]
195 class CompileWebKit2Only(CompileWebKit):
196 command = ["perl", "./Tools/Scripts/build-webkit", "--no-webkit1", WithProperties("--%(configuration)s")]
199 class ArchiveBuiltProduct(shell.ShellCommand):
200 command = ["python", "./Tools/BuildSlaveSupport/built-product-archive",
201 WithProperties("--platform=%(fullPlatform)s"), WithProperties("--%(configuration)s"), "archive"]
202 name = "archive-built-product"
203 description = ["archiving built product"]
204 descriptionDone = ["archived built product"]
208 class ExtractBuiltProduct(shell.ShellCommand):
209 command = ["python", "./Tools/BuildSlaveSupport/built-product-archive",
210 WithProperties("--platform=%(fullPlatform)s"), WithProperties("--%(configuration)s"), "extract"]
211 name = "extract-built-product"
212 description = ["extracting built product"]
213 descriptionDone = ["extracted built product"]
217 class UploadBuiltProduct(transfer.FileUpload):
218 slavesrc = WithProperties("WebKitBuild/%(configuration)s.zip")
219 masterdest = WithProperties("archives/%(fullPlatform)s-%(architecture)s-%(configuration)s/%(got_revision)s.zip")
222 def __init__(self, **kwargs):
223 kwargs['slavesrc'] = self.slavesrc
224 kwargs['masterdest'] = self.masterdest
225 kwargs['mode'] = 0644
226 transfer.FileUpload.__init__(self, **kwargs)
229 class DownloadBuiltProduct(shell.ShellCommand):
230 command = ["python", "./Tools/BuildSlaveSupport/download-built-product",
231 WithProperties("--platform=%(platform)s"), WithProperties("--%(configuration)s"),
232 WithProperties(c["buildbotURL"] + "archives/%(fullPlatform)s-%(architecture)s-%(configuration)s/%(got_revision)s.zip")]
233 name = "download-built-product"
234 description = ["downloading built product"]
235 descriptionDone = ["downloaded built product"]
237 flunkOnFailure = True
240 class RunJavaScriptCoreTests(shell.Test):
242 description = ["jscore-tests running"]
243 descriptionDone = ["jscore-tests"]
244 command = ["perl", "./Tools/Scripts/run-javascriptcore-tests", WithProperties("--%(configuration)s")]
245 logfiles = {'actual.html (source)': 'Source/JavaScriptCore/tests/mozilla/actual.html'}
247 def __init__(self, buildJSCTool=True, *args, **kwargs):
248 self.buildJSCTool = buildJSCTool
249 shell.Test.__init__(self, *args, **kwargs)
250 self.addFactoryArguments(buildJSCTool=buildJSCTool)
253 appendCustomBuildFlags(self, self.getProperty('platform'))
254 if not self.buildJSCTool:
255 self.setCommand(self.command + ['--no-build'])
256 return shell.Test.start(self)
258 def commandComplete(self, cmd):
259 shell.Test.commandComplete(self, cmd)
260 logText = cmd.logs['stdio'].getText()
263 ('failing Mozilla test', re.compile(r'^Results for Mozilla tests:\r?\n\s+(\d+) regression', re.MULTILINE)),
264 ('failing js test', re.compile(r'^Results for LayoutTests/js tests:\r?\n\s+(\d+) failure', re.MULTILINE)),
265 ('crashing js test', re.compile(r'^Results for LayoutTests/js tests:\r?\n.*\n\s+(\d+) crash', re.MULTILINE)),
266 ('failing JSC stress test', re.compile(r'^Results for JSC stress tests:\r?\n\s+(\d+) failure', re.MULTILINE)),
270 for name, expression in expressions:
271 match = expression.search(logText)
272 resultLines[name] = int(match.group(1)) if match else 0
274 self.regressionLine = []
275 for name in resultLines:
276 failures = resultLines[name]
284 self.regressionLine.append("%d %s%s " % (failures, name, pluralSuffix))
286 if 'actual.html (source)' in cmd.logs:
287 self.addHTMLLog('actual.html', cmd.logs['actual.html (source)'].getText())
289 def evaluateCommand(self, cmd):
290 if self.regressionLine:
298 def getText(self, cmd, results):
299 return self.getText2(cmd, results)
301 def getText2(self, cmd, results):
303 result.extend(self.regressionLine)
306 class RunWebKitTests(shell.Test):
308 description = ["layout-tests running"]
309 descriptionDone = ["layout-tests"]
310 command = ["perl", "./Tools/Scripts/run-webkit-tests",
312 "--no-new-test-results",
313 "--no-sample-on-timeout",
314 "--results-directory", "layout-test-results",
315 "--builder-name", WithProperties("%(buildername)s"),
316 "--build-number", WithProperties("%(buildnumber)s"),
317 "--master-name", "webkit.org",
318 "--test-results-server", "webkit-test-results.appspot.com",
319 WithProperties("--%(configuration)s")]
321 def __init__(self, buildJSCTool=True, *args, **kwargs):
322 self.buildJSCTool = buildJSCTool
323 shell.Test.__init__(self, *args, **kwargs)
324 self.addFactoryArguments(buildJSCTool=buildJSCTool)
327 platform = self.getProperty('platform')
328 shouldAbortEarly = self.getProperty('shouldAbortEarly')
329 appendCustomBuildFlags(self, platform, self.getProperty('fullPlatform'))
330 if platform.startswith('mac'):
331 self.setCommand(self.command + ['--no-build'])
333 self.setCommand(self.command + ["--exit-after-n-crashes-or-timeouts", "50", "--exit-after-n-failures", "500"])
334 if platform == "win":
335 rootArgument = ['--root=' + os.path.join("WebKitBuild", self.getProperty('configuration'), "bin32")]
336 self.setCommand(self.command + ['--no-build'])
338 rootArgument = ['--root=WebKitBuild/bin']
339 if not self.buildJSCTool:
340 self.setCommand(self.command + rootArgument)
341 return shell.Test.start(self)
343 def _parseOldRunWebKitTestsOutput(self, logText):
344 incorrectLayoutLines = []
345 for line in logText.splitlines():
346 if line.find('had incorrect layout') >= 0 or line.find('were new') >= 0 or line.find('was new') >= 0:
347 incorrectLayoutLines.append(line)
348 elif line.find('test case') >= 0 and (line.find(' crashed') >= 0 or line.find(' timed out') >= 0):
349 incorrectLayoutLines.append(line)
350 elif line.startswith("WARNING:") and line.find(' leak') >= 0:
351 incorrectLayoutLines.append(line.replace('WARNING: ', ''))
352 elif line.find('Exiting early') >= 0:
353 incorrectLayoutLines.append(line)
355 # FIXME: Detect and summarize leaks of RefCounted objects
357 self.incorrectLayoutLines = incorrectLayoutLines
359 # FIXME: This will break if new-run-webkit-tests changes its default log formatter.
360 nrwt_log_message_regexp = re.compile(r'\d{2}:\d{2}:\d{2}(\.\d+)?\s+\d+\s+(?P<message>.*)')
362 def _strip_python_logging_prefix(self, line):
363 match_object = self.nrwt_log_message_regexp.match(line)
365 return match_object.group('message')
368 def _parseNewRunWebKitTestsOutput(self, logText):
369 incorrectLayoutLines = []
371 ('flakes', re.compile(r'Unexpected flakiness.+\((\d+)\)')),
372 ('new passes', re.compile(r'Expected to .+, but passed:\s+\((\d+)\)')),
373 ('missing results', re.compile(r'Regressions: Unexpected missing results\s+\((\d+)\)')),
374 ('failures', re.compile(r'Regressions: Unexpected.+\((\d+)\)')),
378 for line in logText.splitlines():
379 if line.find('Exiting early') >= 0 or line.find('leaks found') >= 0:
380 incorrectLayoutLines.append(self._strip_python_logging_prefix(line))
382 for name, expression in expressions:
383 match = expression.search(line)
386 testFailures[name] = testFailures.get(name, 0) + int(match.group(1))
389 # FIXME: Parse file names and put them in results
391 for name in testFailures:
392 incorrectLayoutLines.append(str(testFailures[name]) + ' ' + name)
394 self.incorrectLayoutLines = incorrectLayoutLines
396 def commandComplete(self, cmd):
397 shell.Test.commandComplete(self, cmd)
399 logText = cmd.logs['stdio'].getText()
400 if logText.find("Collecting tests ...") >= 0:
401 self._parseNewRunWebKitTestsOutput(logText)
403 self._parseOldRunWebKitTestsOutput(logText)
405 def evaluateCommand(self, cmd):
408 if self.incorrectLayoutLines:
409 if len(self.incorrectLayoutLines) == 1:
410 line = self.incorrectLayoutLines[0]
411 if line.find('were new') >= 0 or line.find('was new') >= 0 or line.find(' leak') >= 0:
414 for line in self.incorrectLayoutLines:
415 if line.find('flakes') >= 0 or line.find('new passes') >= 0 or line.find('missing results') >= 0:
425 def getText(self, cmd, results):
426 return self.getText2(cmd, results)
428 def getText2(self, cmd, results):
429 if results != SUCCESS and self.incorrectLayoutLines:
430 return self.incorrectLayoutLines
435 class RunUnitTests(TestWithFailureCount):
436 name = "run-api-tests"
437 description = ["unit tests running"]
438 descriptionDone = ["unit-tests"]
439 command = ["perl", "./Tools/Scripts/run-api-tests", WithProperties("--%(configuration)s"), "--verbose"]
440 failedTestsFormatString = "%d unit tests failed or timed out"
443 platform = self.getProperty('fullPlatform')
444 if platform.startswith('win'):
445 self.setCommand(self.command + ['--no-build'])
446 if platform.startswith('mac'):
447 self.setCommand(self.command + ['--no-build'])
448 return shell.Test.start(self)
450 def countFailures(self, cmd):
451 log_text = cmd.logs['stdio'].getText()
454 split = re.split(r'\sTests that timed out:\s', log_text)
456 count += len(re.findall(r'^\s+\S+$', split[1], flags=re.MULTILINE))
458 split = re.split(r'\sTests that failed:\s', split[0])
460 count += len(re.findall(r'^\s+\S+$', split[1], flags=re.MULTILINE))
465 class RunPythonTests(TestWithFailureCount):
466 name = "webkitpy-test"
467 description = ["python-tests running"]
468 descriptionDone = ["python-tests"]
469 command = ["python", "./Tools/Scripts/test-webkitpy"]
470 failedTestsFormatString = "%d python tests failed"
473 platform = self.getProperty('platform')
474 # Python tests are flaky on the GTK builders, running them serially
475 # helps and does not significantly prolong the cycle time.
476 if platform == 'gtk':
477 self.setCommand(self.command + ['--child-processes', '1'])
478 # Python tests fail on windows bots when running more than one child process
479 # https://bugs.webkit.org/show_bug.cgi?id=97465
480 if platform == 'win':
481 self.setCommand(self.command + ['--child-processes', '1'])
482 return shell.Test.start(self)
484 def countFailures(self, cmd):
485 logText = cmd.logs['stdio'].getText()
486 # We're looking for the line that looks like this: FAILED (failures=2, errors=1)
487 regex = re.compile(r'^FAILED \((?P<counts>[^)]+)\)')
488 for line in logText.splitlines():
489 match = regex.match(line)
492 return sum(int(component.split('=')[1]) for component in match.group('counts').split(', '))
496 class RunPerlTests(TestWithFailureCount):
497 name = "webkitperl-test"
498 description = ["perl-tests running"]
499 descriptionDone = ["perl-tests"]
500 command = ["perl", "./Tools/Scripts/test-webkitperl"]
501 failedTestsFormatString = "%d perl tests failed"
503 def countFailures(self, cmd):
504 logText = cmd.logs['stdio'].getText()
505 # We're looking for the line that looks like this: Failed 2/19 test programs. 5/363 subtests failed.
506 regex = re.compile(r'^Failed \d+/\d+ test programs\. (?P<count>\d+)/\d+ subtests failed\.')
507 for line in logText.splitlines():
508 match = regex.match(line)
511 return int(match.group('count'))
515 class RunBindingsTests(shell.Test):
516 name = "bindings-generation-tests"
517 description = ["bindings-tests running"]
518 descriptionDone = ["bindings-tests"]
519 command = ["python", "./Tools/Scripts/run-bindings-tests"]
522 class RunEflAPITests(shell.Test):
524 description = ["API tests running"]
525 descriptionDone = ["API tests"]
526 command = ["perl", "./Tools/Scripts/run-efl-tests", WithProperties("--%(configuration)s")]
529 class RunGtkAPITests(shell.Test):
531 description = ["API tests running"]
532 descriptionDone = ["API tests"]
533 command = ["python", "./Tools/Scripts/run-gtk-tests", "--verbose", WithProperties("--%(configuration)s")]
535 def commandComplete(self, cmd):
536 shell.Test.commandComplete(self, cmd)
538 logText = cmd.logs['stdio'].getText()
540 self.incorrectTests = 0
541 self.timedOutTests = 0
542 self.skippedTests = 0
545 foundItems = re.findall("Tests failed \((\d+)\):", logText)
547 self.incorrectTests = int(foundItems[0])
549 foundItems = re.findall("Tests that timed out \((\d+)\):", logText)
551 self.timedOutTests = int(foundItems[0])
553 foundItems = re.findall("Tests skipped \((\d+)\):", logText)
555 self.skippedTests = int(foundItems[0])
557 self.totalFailedTests = self.incorrectTests + self.timedOutTests
559 if self.totalFailedTests > 0:
561 "%d API tests failed, %d timed out, %d skipped" %
562 (self.incorrectTests, self.timedOutTests, self.skippedTests)
565 def evaluateCommand(self, cmd):
566 if self.totalFailedTests > 0:
574 def getText(self, cmd, results):
575 return self.getText2(cmd, results)
577 def getText2(self, cmd, results):
578 if results != SUCCESS and self.totalFailedTests > 0:
579 return self.statusLine
583 class RunQtAPITests(shell.Test):
585 description = ["API tests running"]
586 descriptionDone = ["API tests"]
589 self.setCommand( ["python", "./Tools/Scripts/run-qtwebkit-tests", "--output-file=qt-unit-tests.html", "--do-not-open-results", "--timeout=120", "--%s" % self.getProperty('configuration')] )
590 if self.getProperty('fullPlatform').endswith("-wk2"):
591 self.setCommand(self.command + ['--webkit2'])
592 return shell.Test.start(self)
594 def commandComplete(self, cmd):
595 shell.Test.commandComplete(self, cmd)
597 logText = cmd.logs['stdio'].getText()
598 foundItems = re.findall("TOTALS: (?P<passed>\d+) passed, (?P<failed>\d+) failed, (?P<skipped>\d+) skipped, (?P<crashed>\d+) crashed", logText)
600 self.incorrectTests = 0
601 self.crashedTests = 0
605 self.incorrectTests = int(foundItems[0][1])
606 self.crashedTests = int(foundItems[0][3])
608 if self.incorrectTests > 0 or self.crashedTests > 0:
610 "%s passed, %s failed, %s skipped, %s crashed" % (foundItems[0][0], foundItems[0][1], foundItems[0][2], foundItems[0][3])
613 def evaluateCommand(self, cmd):
614 if self.crashedTests:
617 if re.findall("Timeout, process", cmd.logs['stdio'].getText()):
618 self.statusLine = ["Failure: timeout occured during testing"]
621 if self.incorrectTests:
629 def getText(self, cmd, results):
630 return self.getText2(cmd, results)
632 def getText2(self, cmd, results):
633 if results != SUCCESS and self.incorrectTests:
634 return self.statusLine
638 class RunWebKitLeakTests(RunWebKitTests):
639 warnOnWarnings = True
641 self.setCommand(self.command + ["--leaks"])
642 return RunWebKitTests.start(self)
645 class RunWebKit2Tests(RunWebKitTests):
647 self.setProperty("shouldAbortEarly", False)
648 self.setCommand(self.command + ["--webkit-test-runner"])
649 if self.getProperty('buildername') == "Qt Mountain Lion Release":
650 self.setCommand(self.command + ["-p"])
652 return RunWebKitTests.start(self)
655 class RunAndUploadPerfTests(shell.Test):
657 description = ["perf-tests running"]
658 descriptionDone = ["perf-tests"]
659 command = ["python", "./Tools/Scripts/run-perf-tests",
660 "--output-json-path", "perf-test-results.json",
661 "--slave-config-json-path", "../../perf-test-config.json",
664 "--test-results-server", "perf.webkit.org",
665 "--builder-name", WithProperties("%(buildername)s"),
666 "--build-number", WithProperties("%(buildnumber)s"),
667 "--platform", WithProperties("%(fullPlatform)s"),
668 WithProperties("--%(configuration)s")]
671 self.setCommand(self.command)
672 return shell.Test.start(self)
674 def getText(self, cmd, results):
675 return self.getText2(cmd, results)
677 def getText2(self, cmd, results):
678 if results != SUCCESS
679 return ["%d perf tests failed" % cmd.rc]
684 class RunAndUploadPerfTestsWebKit2(RunAndUploadPerfTests):
686 self.setCommand(self.command + ["--webkit-test-runner"])
687 return RunAndUploadPerfTests.start(self)
690 class ArchiveTestResults(shell.ShellCommand):
691 command = ["python", "./Tools/BuildSlaveSupport/test-result-archive",
692 WithProperties("--platform=%(platform)s"), WithProperties("--%(configuration)s"), "archive"]
693 name = "archive-test-results"
694 description = ["archiving test results"]
695 descriptionDone = ["archived test results"]
699 class UploadTestResults(transfer.FileUpload):
700 slavesrc = "layout-test-results.zip"
701 masterdest = WithProperties("public_html/results/%(buildername)s/r%(got_revision)s (%(buildnumber)s).zip")
703 def __init__(self, **kwargs):
704 kwargs['slavesrc'] = self.slavesrc
705 kwargs['masterdest'] = self.masterdest
706 kwargs['mode'] = 0644
707 transfer.FileUpload.__init__(self, **kwargs)
710 class ExtractTestResults(master.MasterShellCommand):
711 zipFile = WithProperties("public_html/results/%(buildername)s/r%(got_revision)s (%(buildnumber)s).zip")
712 resultDirectory = WithProperties("public_html/results/%(buildername)s/r%(got_revision)s (%(buildnumber)s)")
713 descriptionDone = ["uploaded results"]
715 def __init__(self, **kwargs):
716 kwargs['command'] = ""
717 master.MasterShellCommand.__init__(self, **kwargs)
719 def resultDirectoryURL(self):
720 return self.build.getProperties().render(self.resultDirectory).replace("public_html/", "/") + "/"
723 self.command = ["unzip", self.build.getProperties().render(self.zipFile), "-d", self.build.getProperties().render(self.resultDirectory)]
724 return master.MasterShellCommand.start(self)
726 def addCustomURLs(self):
727 url = self.resultDirectoryURL() + "results.html"
728 self.addURL("view results", url)
730 def finished(self, result):
732 return master.MasterShellCommand.finished(self, result)
735 class ExtractTestResultsAndLeaks(ExtractTestResults):
736 def addCustomURLs(self):
737 ExtractTestResults.addCustomURLs(self)
738 url = "/LeaksViewer/?url=" + urllib.quote(self.resultDirectoryURL(), safe="")
739 self.addURL("view leaks", url)
742 class Factory(factory.BuildFactory):
743 def __init__(self, platform, configuration, architectures, buildOnly, SVNMirror):
744 factory.BuildFactory.__init__(self)
745 self.addStep(ConfigureBuild(platform=platform, configuration=configuration, architecture=" ".join(architectures), buildOnly=buildOnly, SVNMirror=SVNMirror))
747 self.addStep(WaitForSVNServer())
748 self.addStep(CheckOutSource(SVNMirror=SVNMirror))
749 # There are multiple Qt slaves running on same machines, so buildslaves shouldn't kill the processes of other slaves.
750 if not platform.startswith("qt"):
751 self.addStep(KillOldProcesses())
752 self.addStep(DeleteStaleBuildFiles())
753 if platform == "win":
754 self.addStep(InstallWin32Dependencies())
755 if platform == "gtk":
756 self.addStep(InstallGtkDependencies())
757 if platform == "efl":
758 self.addStep(InstallEflDependencies())
761 class BuildFactory(Factory):
762 def __init__(self, platform, configuration, architectures, triggers=None, SVNMirror=None):
763 Factory.__init__(self, platform, configuration, architectures, True, SVNMirror)
764 self.addStep(CompileWebKit())
766 self.addStep(ArchiveBuiltProduct())
767 self.addStep(UploadBuiltProduct())
768 self.addStep(trigger.Trigger(schedulerNames=triggers))
770 def unitTestsSupported(configuration, platform):
771 if platform.startswith('mac') and configuration == "release":
772 return False; # https://bugs.webkit.org/show_bug.cgi?id=82652
773 return platform == 'win' or platform.startswith('mac')
775 def pickLatestBuild(builder, requests):
776 return max(requests, key=operator.attrgetter("submittedAt"))
778 class TestFactory(Factory):
779 LayoutTestClass = RunWebKitTests
780 ExtractTestResultsClass = ExtractTestResults
781 def __init__(self, platform, configuration, architectures, SVNMirror=None):
782 Factory.__init__(self, platform, configuration, architectures, False, SVNMirror)
783 self.addStep(DownloadBuiltProduct())
784 self.addStep(ExtractBuiltProduct())
785 self.addStep(RunJavaScriptCoreTests(buildJSCTool=False))
786 if self.LayoutTestClass:
787 self.addStep(self.LayoutTestClass(buildJSCTool=(platform != 'win')))
789 if unitTestsSupported(configuration, platform):
790 self.addStep(RunUnitTests())
791 self.addStep(RunPythonTests())
792 self.addStep(RunPerlTests())
793 self.addStep(RunBindingsTests())
794 if self.LayoutTestClass:
795 self.addStep(ArchiveTestResults())
796 self.addStep(UploadTestResults())
797 self.addStep(self.ExtractTestResultsClass())
798 if platform == "efl":
799 self.addStep(RunEflAPITests)
800 if platform == "gtk":
801 self.addStep(RunGtkAPITests())
802 if platform.startswith("qt"):
803 self.addStep(RunQtAPITests)
805 class BuildAndTestFactory(Factory):
806 CompileClass = CompileWebKit
807 LayoutTestClass = RunWebKitTests
808 ExtractTestResultsClass = ExtractTestResults
809 def __init__(self, platform, configuration, architectures, triggers=None, SVNMirror=None, **kwargs):
810 Factory.__init__(self, platform, configuration, architectures, False, SVNMirror, **kwargs)
811 self.addStep(self.CompileClass())
812 self.addStep(RunJavaScriptCoreTests())
813 if self.LayoutTestClass:
814 self.addStep(self.LayoutTestClass())
815 if unitTestsSupported(configuration, platform):
816 self.addStep(RunUnitTests())
817 self.addStep(RunPythonTests())
818 self.addStep(RunPerlTests())
819 self.addStep(RunBindingsTests())
820 if self.LayoutTestClass:
821 self.addStep(ArchiveTestResults())
822 self.addStep(UploadTestResults())
823 self.addStep(self.ExtractTestResultsClass())
824 if platform == "efl":
825 self.addStep(RunEflAPITests())
826 if platform == "gtk":
827 self.addStep(RunGtkAPITests())
828 if platform.startswith("qt"):
829 self.addStep(RunQtAPITests())
831 self.addStep(ArchiveBuiltProduct())
832 self.addStep(UploadBuiltProduct())
833 self.addStep(trigger.Trigger(schedulerNames=triggers))
835 class BuildAndTestWebKit2Factory(BuildAndTestFactory):
836 CompileClass = CompileWebKit
837 LayoutTestClass = RunWebKit2Tests
839 class BuildAndTestWebKit1OnlyFactory(BuildAndTestFactory):
840 CompileClass = CompileWebKit1Only
842 class BuildAndTestWebKit2OnlyFactory(BuildAndTestFactory):
843 CompileClass = CompileWebKit2Only
844 LayoutTestClass = RunWebKit2Tests
846 class BuildAndNonLayoutTestFactory(BuildAndTestFactory):
847 LayoutTestClass = None
849 class TestLeaksFactory(TestFactory):
850 LayoutTestClass = RunWebKitLeakTests
851 ExtractTestResultsClass = ExtractTestResultsAndLeaks
854 class TestWebKit2Factory(TestFactory):
855 LayoutTestClass = RunWebKit2Tests
857 class BuildAndPerfTestFactory(Factory):
858 def __init__(self, platform, configuration, architectures, SVNMirror=None, **kwargs):
859 Factory.__init__(self, platform, configuration, architectures, False, SVNMirror, **kwargs)
860 self.addStep(CompileWebKit())
861 self.addStep(RunAndUploadPerfTests())
863 class BuildAndPerfTestWebKit2Factory(Factory):
864 def __init__(self, platform, configuration, architectures, SVNMirror=None, **kwargs):
865 Factory.__init__(self, platform, configuration, architectures, False, SVNMirror, **kwargs)
866 self.addStep(CompileWebKit())
867 self.addStep(RunAndUploadPerfTestsWebKit2())
869 class DownloadAndPerfTestFactory(Factory):
870 def __init__(self, platform, configuration, architectures, SVNMirror=None, **kwargs):
871 Factory.__init__(self, platform, configuration, architectures, False, SVNMirror, **kwargs)
872 self.addStep(DownloadBuiltProduct())
873 self.addStep(ExtractBuiltProduct())
874 self.addStep(RunAndUploadPerfTests())
876 class DownloadAndPerfTestWebKit2Factory(Factory):
877 def __init__(self, platform, configuration, architectures, SVNMirror=None, **kwargs):
878 Factory.__init__(self, platform, configuration, architectures, False, SVNMirror, **kwargs)
879 self.addStep(DownloadBuiltProduct())
880 self.addStep(ExtractBuiltProduct())
881 self.addStep(RunAndUploadPerfTestsWebKit2())
883 class PlatformSpecificScheduler(AnyBranchScheduler):
884 def __init__(self, platform, branch, **kwargs):
885 self.platform = platform
886 filter = ChangeFilter(branch=[branch, None], filter_fn=self.filter)
887 AnyBranchScheduler.__init__(self, name=platform, change_filter=filter, **kwargs)
889 def filter(self, change):
890 return wkbuild.should_build(self.platform, change.files)
892 trunk_filter = ChangeFilter(branch=["trunk", None])
894 def loadBuilderConfig(c):
895 # FIXME: These file handles are leaked.
896 passwords = json.load(open('passwords.json'))
897 config = json.load(open('config.json'))
899 c['slaves'] = [BuildSlave(slave['name'], passwords[slave['name']], max_builds=1) for slave in config['slaves']]
902 for scheduler in config['schedulers']:
903 if "change_filter" in scheduler:
904 scheduler["change_filter"] = globals()[scheduler["change_filter"]]
905 kls = globals()[scheduler.pop('type')]
906 # Python 2.6 can't handle unicode keys as keyword arguments:
907 # http://bugs.python.org/issue2646. Modern versions of json return
908 # unicode strings from json.load, so we map all keys to str objects.
909 scheduler = dict(map(lambda key_value_pair: (str(key_value_pair[0]), key_value_pair[1]), scheduler.items()))
911 c['schedulers'].append(kls(**scheduler))
913 forceScheduler = ForceScheduler(
915 builderNames=[builder['name'] for builder in config['builders']],
916 reason=StringParameter(name="reason", default="", size=40),
918 # Validate SVN revision: number or emtpy string
919 revision=StringParameter(name="revision", default="", regex=re.compile(r'^(\d*)$')),
921 # Disable default enabled input fields: branch, repository, project, additional properties
922 branch=FixedParameter(name="branch"),
923 repository=FixedParameter(name="repository"),
924 project=FixedParameter(name="project"),
927 c['schedulers'].append(forceScheduler)
930 for builder in config['builders']:
931 for slaveName in builder['slavenames']:
932 for slave in config['slaves']:
933 if slave['name'] != slaveName or slave['platform'] == '*':
936 if slave['platform'] != builder['platform']:
937 raise Exception, "Builder %r is for platform %r but has slave %r for platform %r!" % (builder['name'], builder['platform'], slave['name'], slave['platform'])
941 platform = builder['platform']
943 builderType = builder.pop('type')
944 factory = globals()["%sFactory" % builderType]
946 for key in "platform", "configuration", "architectures", "triggers", "SVNMirror":
947 value = builder.pop(key, None)
949 factorykwargs[key] = value
951 builder["factory"] = factory(**factorykwargs)
953 if platform.startswith('mac'):
954 builder["category"] = 'AppleMac'
955 elif platform == 'win':
956 builder["category"] = 'AppleWin'
957 elif platform.startswith('gtk'):
958 builder["category"] = 'GTK'
959 elif platform.startswith('qt'):
960 builder["category"] = 'Qt'
961 elif platform.startswith('efl'):
962 builder["category"] = "EFL"
964 builder["category"] = 'misc'
966 if (builder['category'] == 'AppleMac' or builder['category'] == 'AppleWin') and builderType != 'Build':
967 builder['nextBuild'] = pickLatestBuild
969 c['builders'].append(builder)