88e037aa5baeab712a1b2884462535cb936d3009
[WebKit-https.git] / Tools / BuildSlaveSupport / build.webkit.org-config / master.cfg
1 # -*- python -*-
2 # ex: set syntax=python:
3
4 from buildbot.buildslave import BuildSlave
5 from buildbot.changes.pb import PBChangeSource
6 from buildbot.scheduler import AnyBranchScheduler, Triggerable
7 from buildbot.schedulers.forcesched import FixedParameter, ForceScheduler, StringParameter, BooleanParameter
8 from buildbot.schedulers.filter import ChangeFilter
9 from buildbot.status import html
10 from buildbot.status.web.authz import Authz
11 from buildbot.process import buildstep, factory, properties
12 from buildbot.steps import master, shell, source, transfer, trigger
13 from buildbot.status.builder import SUCCESS, FAILURE, WARNINGS, SKIPPED, EXCEPTION
14
15 from twisted.internet import defer
16
17 import os
18 import re
19 import json
20 import operator
21 import cStringIO
22 import urllib
23
24 from committer_auth import CommitterAuth
25 import wkbuild
26
27
28 c = BuildmasterConfig = {}
29
30 c['change_source'] = PBChangeSource(port=16000)
31
32 # permissions for WebStatus
33 authz = Authz(
34     auth=CommitterAuth('auth.json'),
35     forceBuild='auth',
36     forceAllBuilds='auth',
37     pingBuilder=True,
38     gracefulShutdown=False,
39     stopBuild='auth',
40     stopAllBuilds='auth',
41     cancelPendingBuild='auth',
42     stopChange=True,
43     cleanShutdown=False)
44
45 c['status'] = []
46 c['status'].append(html.WebStatus(http_port=8710,
47                                   revlink="https://trac.webkit.org/changeset/%s", 
48                                   changecommentlink=(r"(https://bugs\.webkit\.org/show_bug\.cgi\?id=|webkit\.org/b/)(\d+)", r"https://bugs.webkit.org/show_bug.cgi?id=\2"),
49                                   authz=authz))
50
51 c['slavePortnum'] = 17000
52 c['projectName'] = "WebKit"
53 c['projectURL'] = "https://webkit.org"
54 c['buildbotURL'] = "https://build.webkit.org/"
55
56 c['buildHorizon'] = 1000
57 c['logHorizon'] = 500
58 c['eventHorizon'] = 200
59 c['buildCacheSize'] = 60
60
61 WithProperties = properties.WithProperties
62
63
64 class TestWithFailureCount(shell.Test):
65     failedTestsFormatString = "%d test%s failed"
66
67     def countFailures(self, cmd):
68         return 0
69
70     def commandComplete(self, cmd):
71         shell.Test.commandComplete(self, cmd)
72         self.failedTestCount = self.countFailures(cmd)
73         self.failedTestPluralSuffix = "" if self.failedTestCount == 1 else "s"
74
75     def evaluateCommand(self, cmd):
76         if self.failedTestCount:
77             return FAILURE
78
79         if cmd.rc != 0:
80             return FAILURE
81
82         return SUCCESS
83
84     def getText(self, cmd, results):
85         return self.getText2(cmd, results)
86
87     def getText2(self, cmd, results):
88         if results != SUCCESS and self.failedTestCount:
89             return [self.failedTestsFormatString % (self.failedTestCount, self.failedTestPluralSuffix)]
90
91         return [self.name]
92
93
94 class ConfigureBuild(buildstep.BuildStep):
95     name = "configure build"
96     description = ["configuring build"]
97     descriptionDone = ["configured build"]
98     def __init__(self, platform, configuration, architecture, buildOnly, additionalArguments, SVNMirror, *args, **kwargs):
99         buildstep.BuildStep.__init__(self, *args, **kwargs)
100         self.platform = platform
101         if platform != 'jsc-only':
102             self.platform = platform.split('-', 1)[0]
103         self.fullPlatform = platform
104         self.configuration = configuration
105         self.architecture = architecture
106         self.buildOnly = buildOnly
107         self.additionalArguments = additionalArguments
108         self.SVNMirror = SVNMirror
109         self.addFactoryArguments(platform=platform, configuration=configuration, architecture=architecture, buildOnly=buildOnly, additionalArguments=additionalArguments, SVNMirror=SVNMirror)
110
111     def start(self):
112         self.setProperty("platform", self.platform)
113         self.setProperty("fullPlatform", self.fullPlatform)
114         self.setProperty("configuration", self.configuration)
115         self.setProperty("architecture", self.architecture)
116         self.setProperty("buildOnly", self.buildOnly)
117         self.setProperty("additionalArguments", self.additionalArguments)
118         self.setProperty("SVNMirror", self.SVNMirror)
119         self.finished(SUCCESS)
120         return defer.succeed(None)
121
122
123 class CheckOutSource(source.SVN):
124     mode = "update"
125     def __init__(self, SVNMirror, **kwargs):
126         kwargs['baseURL'] = SVNMirror or "https://svn.webkit.org/repository/webkit/"
127         kwargs['defaultBranch'] = "trunk"
128         kwargs['mode'] = self.mode
129         source.SVN.__init__(self, **kwargs)
130         self.addFactoryArguments(SVNMirror=SVNMirror)
131
132 class WaitForSVNServer(shell.ShellCommand):
133     name = "wait-for-svn-server"
134     command = ["python", "./Tools/BuildSlaveSupport/wait-for-SVN-server.py", "-r", WithProperties("%(revision)s"), "-s", WithProperties("%(SVNMirror)s")]
135     description = ["waiting for SVN server"]
136     descriptionDone = ["SVN server is ready"]
137     warnOnFailure = True
138
139     def evaluateCommand(self, cmd):
140         if cmd.rc != 0:
141             return WARNINGS
142         return SUCCESS
143
144 class InstallWin32Dependencies(shell.Compile):
145     description = ["installing dependencies"]
146     descriptionDone = ["installed dependencies"]
147     command = ["perl", "./Tools/Scripts/update-webkit-auxiliary-libs"]
148
149 class KillOldProcesses(shell.Compile):
150     name = "kill old processes"
151     description = ["killing old processes"]
152     descriptionDone = ["killed old processes"]
153     command = ["python", "./Tools/BuildSlaveSupport/kill-old-processes"]
154
155 class CleanBuildIfScheduled(shell.Compile):
156     name = "delete WebKitBuild directory"
157     description = ["deleting WebKitBuild directory"]
158     descriptionDone = ["deleted WebKitBuild directory"]
159     command = ["python", "./Tools/BuildSlaveSupport/clean-build", WithProperties("--platform=%(fullPlatform)s"), WithProperties("--%(configuration)s")]
160
161     def start(self):
162         if not self.getProperty('is_clean'):
163             self.hideStepIf = True
164             return SKIPPED
165         return shell.Compile.start(self)
166
167 class DeleteStaleBuildFiles(shell.Compile):
168     name = "delete stale build files"
169     description = ["deleting stale build files"]
170     descriptionDone = ["deleted stale build files"]
171     command = ["python", "./Tools/BuildSlaveSupport/delete-stale-build-files", WithProperties("--platform=%(fullPlatform)s"), WithProperties("--%(configuration)s")]
172
173     def start(self):
174         if self.getProperty('is_clean'): # Nothing to be done if WebKitBuild had been removed.
175             self.hideStepIf = True
176             return SKIPPED
177         return shell.Compile.start(self)
178
179 class InstallGtkDependencies(shell.ShellCommand):
180     name = "jhbuild"
181     description = ["updating gtk dependencies"]
182     descriptionDone = ["updated gtk dependencies"]
183     command = ["perl", "./Tools/Scripts/update-webkitgtk-libs"]
184     haltOnFailure = True
185
186 def appendCustomBuildFlags(step, platform, fullPlatform):
187     if platform not in ('gtk', 'wincairo', 'ios', 'jsc-only'):
188         return
189     if fullPlatform.startswith('ios-simulator'):
190         platform = 'ios-simulator'
191     elif platform == 'ios':
192         platform = 'device'
193     step.setCommand(step.command + ['--' + platform])
194
195 class CompileWebKit(shell.Compile):
196     command = ["perl", "./Tools/Scripts/build-webkit", WithProperties("--%(configuration)s")]
197     env = {'MFLAGS':''}
198     name = "compile-webkit"
199     description = ["compiling"]
200     descriptionDone = ["compiled"]
201     warningPattern = ".*arning: .*"
202
203     def start(self):
204         platform = self.getProperty('platform')
205         buildOnly = self.getProperty('buildOnly')
206         architecture = self.getProperty('architecture')
207         additionalArguments = self.getProperty('additionalArguments')
208
209         if additionalArguments:
210             self.setCommand(self.command + additionalArguments)
211         if platform in ('mac', 'ios') and architecture:
212             self.setCommand(self.command + ['ARCHS=' + architecture])
213             if platform == 'ios':
214                 self.setCommand(self.command + ['ONLY_ACTIVE_ARCH=NO'])
215         # Generating dSYM files is slow, but these are needed to have line numbers in crash reports on testers.
216         # Debug builds on Yosemite can't use dSYMs, because crash logs end up unsymbolicated.
217         if platform in ('mac', 'ios') and buildOnly and (self.getProperty('fullPlatform') != "mac-yosemite" or self.getProperty('configuration') != "debug"):
218             self.setCommand(self.command + ['DEBUG_INFORMATION_FORMAT=dwarf-with-dsym'])
219
220         appendCustomBuildFlags(self, platform, self.getProperty('fullPlatform'))
221
222         return shell.Compile.start(self)
223
224     def createSummary(self, log):
225         platform = self.getProperty('platform')
226         if platform.startswith('mac'):    
227             warnings = []
228             errors = []
229             sio = cStringIO.StringIO(log.getText())
230             for line in sio.readlines():
231                 if "arning:" in line:
232                     warnings.append(line)
233                 if "rror:" in line:
234                     errors.append(line)
235             if warnings:
236                 self.addCompleteLog('warnings', "".join(warnings))
237             if errors:
238                 self.addCompleteLog('errors', "".join(errors))
239
240
241 class CompileLLINTCLoop(CompileWebKit):
242     command = ["perl", "./Tools/Scripts/build-jsc", "--cloop", WithProperties("--%(configuration)s")]
243
244 class Compile32bitJSC(CompileWebKit):
245     command = ["perl", "./Tools/Scripts/build-jsc", "--32-bit", WithProperties("--%(configuration)s")]
246
247 class CompileJSCOnly(CompileWebKit):
248     command = ["perl", "./Tools/Scripts/build-jsc", WithProperties("--%(configuration)s")]
249
250 class ArchiveBuiltProduct(shell.ShellCommand):
251     command = ["python", "./Tools/BuildSlaveSupport/built-product-archive",
252                WithProperties("--platform=%(fullPlatform)s"), WithProperties("--%(configuration)s"), "archive"]
253     name = "archive-built-product"
254     description = ["archiving built product"]
255     descriptionDone = ["archived built product"]
256     haltOnFailure = True
257
258
259 class ExtractBuiltProduct(shell.ShellCommand):
260     command = ["python", "./Tools/BuildSlaveSupport/built-product-archive",
261                WithProperties("--platform=%(fullPlatform)s"), WithProperties("--%(configuration)s"), "extract"]
262     name = "extract-built-product"
263     description = ["extracting built product"]
264     descriptionDone = ["extracted built product"]
265     haltOnFailure = True
266
267
268 class UploadBuiltProduct(transfer.FileUpload):
269     slavesrc = WithProperties("WebKitBuild/%(configuration)s.zip")
270     masterdest = WithProperties("archives/%(fullPlatform)s-%(architecture)s-%(configuration)s/%(got_revision)s.zip")
271     haltOnFailure = True
272
273     def __init__(self, **kwargs):
274         kwargs['slavesrc'] = self.slavesrc
275         kwargs['masterdest'] = self.masterdest
276         kwargs['mode'] = 0644
277         kwargs['blocksize'] = 1024*256
278         transfer.FileUpload.__init__(self, **kwargs)
279
280
281 class DownloadBuiltProduct(shell.ShellCommand):
282     command = ["python", "./Tools/BuildSlaveSupport/download-built-product",
283         WithProperties("--platform=%(platform)s"), WithProperties("--%(configuration)s"),
284         WithProperties(c["buildbotURL"] + "archives/%(fullPlatform)s-%(architecture)s-%(configuration)s/%(got_revision)s.zip")]
285     name = "download-built-product"
286     description = ["downloading built product"]
287     descriptionDone = ["downloaded built product"]
288     haltOnFailure = True
289     flunkOnFailure = True
290
291
292 class RunJavaScriptCoreTests(TestWithFailureCount):
293     name = "jscore-test"
294     description = ["jscore-tests running"]
295     descriptionDone = ["jscore-tests"]
296     jsonFileName = "jsc_results.json"
297     command = ["perl", "./Tools/Scripts/run-javascriptcore-tests", "--no-build", "--no-fail-fast", "--json-output={0}".format(jsonFileName), WithProperties("--%(configuration)s")]
298     failedTestsFormatString = "%d JSC test%s failed"
299     logfiles = {"json": jsonFileName}
300
301     def start(self):
302         appendCustomBuildFlags(self, self.getProperty('platform'), self.getProperty('fullPlatform'))
303         return shell.Test.start(self)
304
305     def countFailures(self, cmd):
306         logText = cmd.logs['stdio'].getText()
307
308         match = re.search(r'^Results for JSC stress tests:\r?\n\s+(\d+) failure', logText, re.MULTILINE)
309         if match:
310             return int(match.group(1))
311
312         match = re.search(r'^Results for Mozilla tests:\r?\n\s+(\d+) regression', logText, re.MULTILINE)
313         if match:
314             return int(match.group(1))
315
316         return 0
317
318
319 class RunRemoteJavaScriptCoreTests(RunJavaScriptCoreTests):
320     def start(self):
321         self.setCommand(self.command + ["--memory-limited", "--remote-config-file", "../../remote-jsc-tests-config.json"])
322         return RunJavaScriptCoreTests.start(self)
323
324
325 class RunWebKitTests(shell.Test):
326     name = "layout-test"
327     description = ["layout-tests running"]
328     descriptionDone = ["layout-tests"]
329     resultDirectory = "layout-test-results"
330     command = ["python", "./Tools/Scripts/run-webkit-tests",
331                "--no-build",
332                "--no-show-results",
333                "--no-new-test-results",
334                "--builder-name", WithProperties("%(buildername)s"),
335                "--build-number", WithProperties("%(buildnumber)s"),
336                "--master-name", "webkit.org",
337                "--test-results-server", "webkit-test-results.webkit.org",
338                "--exit-after-n-crashes-or-timeouts", "50",
339                "--exit-after-n-failures", "500",
340                WithProperties("--%(configuration)s")]
341
342     def start(self):
343         platform = self.getProperty('platform')
344         appendCustomBuildFlags(self, platform, self.getProperty('fullPlatform'))
345         additionalArguments = self.getProperty('additionalArguments')
346
347         self.setCommand(self.command + ["--results-directory", self.resultDirectory])
348         self.setCommand(self.command + ['--debug-rwt-logging'])
349
350         if platform == "win":
351             self.setCommand(self.command + ['--batch-size', '100', '--root=' + os.path.join("WebKitBuild", self.getProperty('configuration'), "bin32")])
352
353         if additionalArguments:
354             self.setCommand(self.command + additionalArguments)
355         return shell.Test.start(self)
356
357     # FIXME: This will break if run-webkit-tests changes its default log formatter.
358     nrwt_log_message_regexp = re.compile(r'\d{2}:\d{2}:\d{2}(\.\d+)?\s+\d+\s+(?P<message>.*)')
359
360     def _strip_python_logging_prefix(self, line):
361         match_object = self.nrwt_log_message_regexp.match(line)
362         if match_object:
363             return match_object.group('message')
364         return line
365
366     def _parseRunWebKitTestsOutput(self, logText):
367         incorrectLayoutLines = []
368         expressions = [
369             ('flakes', re.compile(r'Unexpected flakiness.+\((\d+)\)')),
370             ('new passes', re.compile(r'Expected to .+, but passed:\s+\((\d+)\)')),
371             ('missing results', re.compile(r'Regressions: Unexpected missing results\s+\((\d+)\)')),
372             ('failures', re.compile(r'Regressions: Unexpected.+\((\d+)\)')),
373         ]
374         testFailures = {}
375
376         for line in logText.splitlines():
377             if line.find('Exiting early') >= 0 or line.find('leaks found') >= 0:
378                 incorrectLayoutLines.append(self._strip_python_logging_prefix(line))
379                 continue
380             for name, expression in expressions:
381                 match = expression.search(line)
382
383                 if match:
384                     testFailures[name] = testFailures.get(name, 0) + int(match.group(1))
385                     break
386
387                 # FIXME: Parse file names and put them in results
388
389         for name in testFailures:
390             incorrectLayoutLines.append(str(testFailures[name]) + ' ' + name)
391
392         self.incorrectLayoutLines = incorrectLayoutLines
393
394     def commandComplete(self, cmd):
395         shell.Test.commandComplete(self, cmd)
396
397         logText = cmd.logs['stdio'].getText()
398         self._parseRunWebKitTestsOutput(logText)
399
400     def evaluateCommand(self, cmd):
401         result = SUCCESS
402
403         if self.incorrectLayoutLines:
404             if len(self.incorrectLayoutLines) == 1:
405                 line = self.incorrectLayoutLines[0]
406                 if line.find('were new') >= 0 or line.find('was new') >= 0 or line.find(' leak') >= 0:
407                     return WARNINGS
408
409             for line in self.incorrectLayoutLines:
410                 if line.find('flakes') >= 0 or line.find('new passes') >= 0 or line.find('missing results') >= 0:
411                     result = WARNINGS
412                 else:
413                     return FAILURE
414
415         # Return code from Tools/Scripts/layout_tests/run_webkit_tests.py.
416         # This means that an exception was raised when running run-webkit-tests and
417         # was never handled.
418         if cmd.rc == 254:
419             return EXCEPTION
420         if cmd.rc != 0:
421             return FAILURE
422
423         return result
424
425     def getText(self, cmd, results):
426         return self.getText2(cmd, results)
427
428     def getText2(self, cmd, results):
429         if results != SUCCESS and self.incorrectLayoutLines:
430             return self.incorrectLayoutLines
431
432         return [self.name]
433
434
435 class RunDashboardTests(RunWebKitTests):
436     name = "dashboard-tests"
437     description = ["dashboard-tests running"]
438     descriptionDone = ["dashboard-tests"]
439     resultDirectory = os.path.join(RunWebKitTests.resultDirectory, "dashboard-layout-test-results")
440
441     def start(self):
442         self.setCommand(self.command + ["--layout-tests-directory", "./Tools/BuildSlaveSupport/build.webkit.org-config/public_html/dashboard/Scripts/tests"])
443         return RunWebKitTests.start(self)
444
445
446 class RunUnitTests(TestWithFailureCount):
447     name = "run-api-tests"
448     description = ["unit tests running"]
449     descriptionDone = ["unit-tests"]
450     command = ["perl", "./Tools/Scripts/run-api-tests", "--no-build", WithProperties("--%(configuration)s"), "--verbose"]
451     failedTestsFormatString = "%d unit test%s failed or timed out"
452
453     def start(self):
454         appendCustomBuildFlags(self, self.getProperty('platform'), self.getProperty('fullPlatform'))
455         return shell.Test.start(self)
456
457     def countFailures(self, cmd):
458         log_text = cmd.logs['stdio'].getText()
459         count = 0
460
461         split = re.split(r'\sTests that timed out:\s', log_text)
462         if len(split) > 1:
463             count += len(re.findall(r'^\s+\S+$', split[1], flags=re.MULTILINE))
464
465         split = re.split(r'\sTests that failed:\s', split[0])
466         if len(split) > 1:
467             count += len(re.findall(r'^\s+\S+$', split[1], flags=re.MULTILINE))
468
469         return count
470
471
472 class RunPythonTests(TestWithFailureCount):
473     name = "webkitpy-test"
474     description = ["python-tests running"]
475     descriptionDone = ["python-tests"]
476     command = ["python", "./Tools/Scripts/test-webkitpy", "--verbose"]
477     failedTestsFormatString = "%d python test%s failed"
478
479     def start(self):
480         platform = self.getProperty('platform')
481         # Python tests are flaky on the GTK builders, running them serially
482         # helps and does not significantly prolong the cycle time.
483         if platform == 'gtk':
484             self.setCommand(self.command + ['--child-processes', '1'])
485         # Python tests fail on windows bots when running more than one child process
486         # https://bugs.webkit.org/show_bug.cgi?id=97465
487         if platform == 'win':
488             self.setCommand(self.command + ['--child-processes', '1'])
489         return shell.Test.start(self)
490
491     def countFailures(self, cmd):
492         logText = cmd.logs['stdio'].getText()
493         # We're looking for the line that looks like this: FAILED (failures=2, errors=1)
494         regex = re.compile(r'^FAILED \((?P<counts>[^)]+)\)')
495         for line in logText.splitlines():
496             match = regex.match(line)
497             if not match:
498                 continue
499             return sum(int(component.split('=')[1]) for component in match.group('counts').split(', '))
500         return 0
501
502
503 class RunPerlTests(TestWithFailureCount):
504     name = "webkitperl-test"
505     description = ["perl-tests running"]
506     descriptionDone = ["perl-tests"]
507     command = ["perl", "./Tools/Scripts/test-webkitperl"]
508     failedTestsFormatString = "%d perl test%s failed"
509
510     def countFailures(self, cmd):
511         logText = cmd.logs['stdio'].getText()
512         # We're looking for the line that looks like this: Failed 2/19 test programs. 5/363 subtests failed.
513         regex = re.compile(r'^Failed \d+/\d+ test programs\. (?P<count>\d+)/\d+ subtests failed\.')
514         for line in logText.splitlines():
515             match = regex.match(line)
516             if not match:
517                 continue
518             return int(match.group('count'))
519         return 0
520
521
522 class RunLLINTCLoopTests(TestWithFailureCount):
523     name = "webkit-jsc-cloop-test"
524     description = ["cloop-tests running"]
525     descriptionDone = ["cloop-tests"]
526     jsonFileName = "jsc_cloop.json"
527     command = ["perl", "./Tools/Scripts/run-javascriptcore-tests", "--cloop", "--no-build", "--no-jsc-stress", "--no-fail-fast", "--json-output={0}".format(jsonFileName), WithProperties("--%(configuration)s")]
528     failedTestsFormatString = "%d regression%s found."
529     logfiles = {"json": jsonFileName}
530
531     def countFailures(self, cmd):
532         logText = cmd.logs['stdio'].getText()
533         # We're looking for the line that looks like this: 0 regressions found.
534         regex = re.compile(r'\s*(?P<count>\d+) regressions? found.')
535         for line in logText.splitlines():
536             match = regex.match(line)
537             if not match:
538                 continue
539             return int(match.group('count'))
540         return 0
541
542
543 class Run32bitJSCTests(TestWithFailureCount):
544     name = "webkit-32bit-jsc-test"
545     description = ["32bit-jsc-tests running"]
546     descriptionDone = ["32bit-jsc-tests"]
547     jsonFileName = "jsc_32bit.json"
548     command = ["perl", "./Tools/Scripts/run-javascriptcore-tests", "--32-bit", "--no-build", "--no-fail-fast", "--json-output={0}".format(jsonFileName), WithProperties("--%(configuration)s")]
549     failedTestsFormatString = "%d regression%s found."
550     logfiles = {"json": jsonFileName}
551
552     def countFailures(self, cmd):
553         logText = cmd.logs['stdio'].getText()
554         # We're looking for the line that looks like this: 0 failures found.
555         regex = re.compile(r'\s*(?P<count>\d+) failures? found.')
556         for line in logText.splitlines():
557             match = regex.match(line)
558             if not match:
559                 continue
560             return int(match.group('count'))
561         return 0
562
563
564 class RunBindingsTests(shell.Test):
565     name = "bindings-generation-tests"
566     description = ["bindings-tests running"]
567     descriptionDone = ["bindings-tests"]
568     command = ["python", "./Tools/Scripts/run-bindings-tests"]
569
570
571 class RunBuiltinsTests(shell.Test):
572     name = "builtins-generator-tests"
573     description = ["builtins-generator-tests running"]
574     descriptionDone = ["builtins-generator-tests"]
575     command = ["python", "./Tools/Scripts/run-builtins-generator-tests"]
576
577
578 class RunGtkAPITests(shell.Test):
579     name = "API tests"
580     description = ["API tests running"]
581     descriptionDone = ["API tests"]
582     command = ["python", "./Tools/Scripts/run-gtk-tests", "--verbose", WithProperties("--%(configuration)s")]
583
584     def start(self):
585         additionalArguments = self.getProperty("additionalArguments")
586         if additionalArguments:
587             self.command += additionalArguments
588         self.setCommand(self.command)
589         return shell.Test.start(self)
590
591     def commandComplete(self, cmd):
592         shell.Test.commandComplete(self, cmd)
593
594         logText = cmd.logs['stdio'].getText()
595
596         self.incorrectTests = 0
597         self.crashedTests = 0
598         self.timedOutTests = 0
599         self.skippedTests = 0
600         self.statusLine = []
601
602         foundItems = re.findall("Tests failed \((\d+)\):", logText)
603         if (foundItems):
604             self.incorrectTests = int(foundItems[0])
605
606         foundItems = re.findall("Tests that crashed \((\d+)\):", logText)
607         if (foundItems):
608             self.crashedTests = int(foundItems[0])
609
610         foundItems = re.findall("Tests that timed out \((\d+)\):", logText)
611         if (foundItems):
612             self.timedOutTests = int(foundItems[0])
613
614         foundItems = re.findall("Tests skipped \((\d+)\):", logText)
615         if (foundItems):
616             self.skippedTests = int(foundItems[0])
617
618         self.totalFailedTests = self.incorrectTests + self.crashedTests + self.timedOutTests
619
620         if self.totalFailedTests > 0:
621             self.statusLine = [
622                 "%d API tests failed, %d crashed, %d timed out, %d skipped" %
623                 (self.incorrectTests, self.crashedTests, self.timedOutTests, self.skippedTests)
624             ]
625
626     def evaluateCommand(self, cmd):
627         if self.totalFailedTests > 0:
628             return FAILURE
629
630         if cmd.rc != 0:
631             return FAILURE
632
633         return SUCCESS
634
635     def getText(self, cmd, results):
636         return self.getText2(cmd, results)
637
638     def getText2(self, cmd, results):
639         if results != SUCCESS and self.totalFailedTests > 0:
640             return self.statusLine
641
642         return [self.name]
643
644 class RunWebKit1Tests(RunWebKitTests):
645     def start(self):
646         self.setCommand(self.command + ["--dump-render-tree"])
647
648         return RunWebKitTests.start(self)
649
650 class RunWebKit1LeakTests(RunWebKit1Tests):
651     want_stdout = False
652     want_stderr = False
653     warnOnWarnings = True
654     def start(self):
655         self.setCommand(self.command + ["--leaks"])
656         return RunWebKit1Tests.start(self)
657
658 class RunAndUploadPerfTests(shell.Test):
659     name = "perf-test"
660     description = ["perf-tests running"]
661     descriptionDone = ["perf-tests"]
662     command = ["python", "./Tools/Scripts/run-perf-tests",
663                "--output-json-path", "perf-test-results.json",
664                "--slave-config-json-path", "../../perf-test-config.json",
665                "--no-show-results",
666                "--reset-results",
667                "--test-results-server", "perf.webkit.org",
668                "--builder-name", WithProperties("%(buildername)s"),
669                "--build-number", WithProperties("%(buildnumber)s"),
670                "--platform", WithProperties("%(fullPlatform)s"),
671                "--no-build",
672                WithProperties("--%(configuration)s")]
673
674     def start(self):
675         additionalArguments = self.getProperty("additionalArguments")
676         if additionalArguments:
677             self.command += additionalArguments
678         self.setCommand(self.command)
679         return shell.Test.start(self)
680
681     def getText(self, cmd, results):
682         return self.getText2(cmd, results)
683
684     def getText2(self, cmd, results):
685         if results != SUCCESS:
686             if cmd.rc == -1 & 0xff:
687                 return ["build not up to date"]
688             elif cmd.rc == -2 & 0xff:
689                 return ["slave config JSON error"]
690             elif cmd.rc == -3 & 0xff:
691                 return ["output JSON merge error"]
692             elif cmd.rc == -4 & 0xff:
693                 return ["upload error"]
694             elif cmd.rc == -5 & 0xff:
695                 return ["system dependency error"]
696             elif cmd.rc == -1:
697                 return ["timeout"]
698             else:
699                 return ["%d perf tests failed" % cmd.rc]
700
701         return [self.name]
702
703 class RunBenchmarkTests(shell.Test):
704     name = "benchmark-test"
705     description = ["benchmark tests running"]
706     descriptionDone = ["benchmark tests"]
707     # Buildbot default timeout without output for a step is 1200.
708     # The current maximum timeout for a benchmark plan is also 1200.
709     # So raise the buildbot timeout to avoid aborting this whole step when a test timeouts.
710     timeout = 1500
711     command = ["python", "./Tools/Scripts/run-benchmark", "--allplans"]
712
713     def start(self):
714         platform = self.getProperty("platform")
715         if platform == "gtk":
716             self.command += ["--browser", "minibrowser-gtk"]
717         self.setCommand(self.command)
718         return shell.Test.start(self)
719
720     def getText(self, cmd, results):
721         return self.getText2(cmd, results)
722
723     def getText2(self, cmd, results):
724         if results != SUCCESS:
725             return ["%d benchmark tests failed" % cmd.rc]
726         return [self.name]
727
728 class ArchiveTestResults(shell.ShellCommand):
729     command = ["python", "./Tools/BuildSlaveSupport/test-result-archive",
730                WithProperties("--platform=%(platform)s"), WithProperties("--%(configuration)s"), "archive"]
731     name = "archive-test-results"
732     description = ["archiving test results"]
733     descriptionDone = ["archived test results"]
734     haltOnFailure = True
735
736
737 class UploadTestResults(transfer.FileUpload):
738     slavesrc = "layout-test-results.zip"
739     masterdest = WithProperties("public_html/results/%(buildername)s/r%(got_revision)s (%(buildnumber)s).zip")
740
741     def __init__(self, **kwargs):
742         kwargs['slavesrc'] = self.slavesrc
743         kwargs['masterdest'] = self.masterdest
744         kwargs['mode'] = 0644
745         transfer.FileUpload.__init__(self, **kwargs)
746
747
748 class ExtractTestResults(master.MasterShellCommand):
749     zipFile = WithProperties("public_html/results/%(buildername)s/r%(got_revision)s (%(buildnumber)s).zip")
750     resultDirectory = WithProperties("public_html/results/%(buildername)s/r%(got_revision)s (%(buildnumber)s)")
751     descriptionDone = ["uploaded results"]
752
753     def __init__(self, **kwargs):
754         kwargs['command'] = ""
755         master.MasterShellCommand.__init__(self, **kwargs)
756
757     def resultDirectoryURL(self):
758         return self.build.getProperties().render(self.resultDirectory).replace("public_html/", "/") + "/"
759
760     def start(self):
761         self.command = ["unzip", self.build.getProperties().render(self.zipFile), "-d", self.build.getProperties().render(self.resultDirectory)]
762         return master.MasterShellCommand.start(self)
763
764     def addCustomURLs(self):
765         self.addURL("view layout test results", self.resultDirectoryURL() + "results.html")
766         self.addURL("view dashboard test results", self.resultDirectoryURL() + "dashboard-layout-test-results/results.html")
767
768     def finished(self, result):
769         self.addCustomURLs()
770         return master.MasterShellCommand.finished(self, result)
771
772
773 class ExtractTestResultsAndLeaks(ExtractTestResults):
774     def addCustomURLs(self):
775         ExtractTestResults.addCustomURLs(self)
776         url = "/LeaksViewer/?url=" + urllib.quote(self.resultDirectoryURL(), safe="")
777         self.addURL("view leaks", url)
778
779
780 class Factory(factory.BuildFactory):
781     def __init__(self, platform, configuration, architectures, buildOnly, additionalArguments, SVNMirror):
782         factory.BuildFactory.__init__(self)
783         self.addStep(ConfigureBuild(platform=platform, configuration=configuration, architecture=" ".join(architectures), buildOnly=buildOnly, additionalArguments=additionalArguments, SVNMirror=SVNMirror))
784         if SVNMirror:
785             self.addStep(WaitForSVNServer())
786         self.addStep(CheckOutSource(SVNMirror=SVNMirror))
787         if not (platform == "jsc-only"):
788             self.addStep(KillOldProcesses())
789         self.addStep(CleanBuildIfScheduled())
790         self.addStep(DeleteStaleBuildFiles())
791         if platform == "win":
792             self.addStep(InstallWin32Dependencies())
793         if platform == "gtk" and additionalArguments != ["--default-cmake-features"]:
794             self.addStep(InstallGtkDependencies())
795
796
797 class BuildFactory(Factory):
798     def __init__(self, platform, configuration, architectures, triggers=None, additionalArguments=None, SVNMirror=None):
799         Factory.__init__(self, platform, configuration, architectures, True, additionalArguments, SVNMirror)
800
801         if platform == "win":
802             self.addStep(CompileWebKit(timeout=2*60*60))
803         else:
804             self.addStep(CompileWebKit())
805
806         if triggers:
807             self.addStep(ArchiveBuiltProduct())
808             self.addStep(UploadBuiltProduct())
809             self.addStep(trigger.Trigger(schedulerNames=triggers))
810
811 def pickLatestBuild(builder, requests):
812     return max(requests, key=operator.attrgetter("submittedAt"))
813
814 class TestFactory(Factory):
815     JSCTestClass = RunJavaScriptCoreTests
816     LayoutTestClass = RunWebKitTests
817
818     def getProduct(self):
819         self.addStep(DownloadBuiltProduct())
820         self.addStep(ExtractBuiltProduct())
821
822     def __init__(self, platform, configuration, architectures, additionalArguments=None, SVNMirror=None, **kwargs):
823         Factory.__init__(self, platform, configuration, architectures, False, additionalArguments, SVNMirror, **kwargs)
824         self.getProduct()
825         if self.JSCTestClass:
826             self.addStep(self.JSCTestClass())
827         if self.LayoutTestClass:
828             self.addStep(self.LayoutTestClass())
829
830         if platform == 'win' or platform.startswith('mac') or platform.startswith('ios-simulator'):
831             self.addStep(RunUnitTests())
832         self.addStep(RunPythonTests())
833         self.addStep(RunPerlTests())
834         self.addStep(RunBindingsTests())
835         self.addStep(RunBuiltinsTests())
836         self.addStep(RunDashboardTests())
837         if self.LayoutTestClass:
838             self.addStep(ArchiveTestResults())
839             self.addStep(UploadTestResults())
840             self.addStep(ExtractTestResults())
841         if platform == "gtk":
842             self.addStep(RunGtkAPITests())
843
844 class BuildAndTestFactory(TestFactory):
845     def getProduct(self):
846         self.addStep(CompileWebKit())
847
848     def __init__(self, platform, configuration, architectures, triggers=None, additionalArguments=None, SVNMirror=None, **kwargs):
849         TestFactory.__init__(self, platform, configuration, architectures, additionalArguments, SVNMirror, **kwargs)
850         if triggers:
851             self.addStep(ArchiveBuiltProduct())
852             self.addStep(UploadBuiltProduct())
853             self.addStep(trigger.Trigger(schedulerNames=triggers))
854
855 class BuildAndTestLLINTCLoopFactory(Factory):
856     def __init__(self, platform, configuration, architectures, triggers=None, additionalArguments=None, SVNMirror=None, **kwargs):
857         Factory.__init__(self, platform, configuration, architectures, False, additionalArguments, SVNMirror, **kwargs)
858         self.addStep(CompileLLINTCLoop())
859         self.addStep(RunLLINTCLoopTests())
860
861 class BuildAndTest32bitJSCFactory(Factory):
862     def __init__(self, platform, configuration, architectures, triggers=None, additionalArguments=None, SVNMirror=None, **kwargs):
863         Factory.__init__(self, platform, configuration, architectures, False, additionalArguments, SVNMirror, **kwargs)
864         self.addStep(Compile32bitJSC())
865         self.addStep(Run32bitJSCTests())
866
867 class BuildAndNonLayoutTestFactory(BuildAndTestFactory):
868     LayoutTestClass = None
869
870 class BuildAndRemoteJSCTestsFactory(Factory):
871     def __init__(self, platform, configuration, architectures, triggers=None, additionalArguments=None, SVNMirror=None):
872         Factory.__init__(self, platform, configuration, architectures, False, additionalArguments, SVNMirror)
873         self.addStep(CompileJSCOnly())
874         self.addStep(RunRemoteJavaScriptCoreTests())
875
876 class TestWebKit1LeaksFactory(Factory):
877     def __init__(self, platform, configuration, architectures, additionalArguments=None, SVNMirror=None):
878         Factory.__init__(self, platform, configuration, architectures, False, additionalArguments, SVNMirror)
879         self.addStep(DownloadBuiltProduct())
880         self.addStep(ExtractBuiltProduct())
881         self.addStep(RunWebKit1LeakTests())
882         self.addStep(ArchiveTestResults())
883         self.addStep(UploadTestResults())
884         self.addStep(ExtractTestResultsAndLeaks())
885
886 class TestAllButJSCFactory(TestFactory):
887     JSCTestClass = None
888
889 class TestJSCFactory(Factory):
890     def __init__(self, platform, configuration, architectures, additionalArguments=None, SVNMirror=None):
891         Factory.__init__(self, platform, configuration, architectures, False, additionalArguments, SVNMirror)
892         self.addStep(DownloadBuiltProduct())
893         self.addStep(ExtractBuiltProduct())
894         self.addStep(RunJavaScriptCoreTests())
895
896 class TestWebKit1Factory(TestFactory):
897     LayoutTestClass = RunWebKit1Tests
898
899 class TestWebKit1AllButJSCFactory(TestWebKit1Factory):
900     JSCTestClass = None
901
902 class BuildAndPerfTestFactory(Factory):
903     def __init__(self, platform, configuration, architectures, additionalArguments=None, SVNMirror=None, **kwargs):
904         Factory.__init__(self, platform, configuration, architectures, False, additionalArguments, SVNMirror, **kwargs)
905         self.addStep(CompileWebKit())
906         self.addStep(RunAndUploadPerfTests())
907         if platform == "gtk":
908             self.addStep(RunBenchmarkTests())
909
910 class DownloadAndPerfTestFactory(Factory):
911     def __init__(self, platform, configuration, architectures, additionalArguments=None, SVNMirror=None, **kwargs):
912         Factory.__init__(self, platform, configuration, architectures, False, additionalArguments, SVNMirror, **kwargs)
913         self.addStep(DownloadBuiltProduct())
914         self.addStep(ExtractBuiltProduct())
915         self.addStep(RunAndUploadPerfTests())
916         if platform == "gtk":
917             self.addStep(RunBenchmarkTests())
918
919 class PlatformSpecificScheduler(AnyBranchScheduler):
920     def __init__(self, platform, branch, **kwargs):
921         self.platform = platform
922         filter = ChangeFilter(branch=[branch, None], filter_fn=self.filter)
923         AnyBranchScheduler.__init__(self, name=platform, change_filter=filter, **kwargs)
924
925     def filter(self, change):
926         return wkbuild.should_build(self.platform, change.files)
927
928 trunk_filter = ChangeFilter(branch=["trunk", None])
929
930 def loadBuilderConfig(c):
931     # FIXME: These file handles are leaked.
932     passwords = json.load(open('passwords.json'))
933     config = json.load(open('config.json'))
934
935     c['slaves'] = [BuildSlave(slave['name'], passwords[slave['name']], max_builds=1) for slave in config['slaves']]
936
937     c['schedulers'] = []
938     for scheduler in config['schedulers']:
939         if "change_filter" in scheduler:
940             scheduler["change_filter"] = globals()[scheduler["change_filter"]]
941         kls = globals()[scheduler.pop('type')]
942         # Python 2.6 can't handle unicode keys as keyword arguments:
943         # http://bugs.python.org/issue2646.  Modern versions of json return
944         # unicode strings from json.load, so we map all keys to str objects.
945         scheduler = dict(map(lambda key_value_pair: (str(key_value_pair[0]), key_value_pair[1]), scheduler.items()))
946
947         c['schedulers'].append(kls(**scheduler))
948
949     forceScheduler = ForceScheduler(
950         name="force",
951         builderNames=[str(builder['name']) for builder in config['builders']],
952         reason=StringParameter(name="reason", default="", size=40),
953
954         # Validate SVN revision: number or empty string
955         revision=StringParameter(name="revision", default="", regex=re.compile(r'^(\d*)$')),
956
957         # Disable default enabled input fields: branch, repository, project, additional properties
958         branch=FixedParameter(name="branch"),
959         repository=FixedParameter(name="repository"),
960         project=FixedParameter(name="project"),
961         properties=[BooleanParameter(name="is_clean", label="Force Clean build")]
962     )
963     c['schedulers'].append(forceScheduler)
964
965     c['builders'] = []
966     for builder in config['builders']:
967         for slaveName in builder['slavenames']:
968             for slave in config['slaves']:
969                 if slave['name'] != slaveName or slave['platform'] == '*':
970                     continue
971
972                 if slave['platform'] != builder['platform']:
973                     raise Exception, "Builder %r is for platform %r but has slave %r for platform %r!" % (builder['name'], builder['platform'], slave['name'], slave['platform'])
974
975                 break
976
977         platform = builder['platform']
978
979         builderType = builder.pop('type')
980         factory = globals()["%sFactory" % builderType]
981         factorykwargs = {}
982         for key in "platform", "configuration", "architectures", "triggers", "additionalArguments", "SVNMirror":
983             value = builder.pop(key, None)
984             if value:
985                 factorykwargs[key] = value
986
987         builder["factory"] = factory(**factorykwargs)
988
989         if platform.startswith('mac'):
990             builder["category"] = 'AppleMac'
991         elif platform.startswith('ios'):
992             builder['category'] = 'iOS'
993         elif platform == 'win':
994             builder["category"] = 'AppleWin'
995         elif platform.startswith('gtk'):
996             builder["category"] = 'GTK'
997         else:
998             builder["category"] = 'misc'
999
1000         if (builder['category'] in ('AppleMac', 'AppleWin', 'iOS')) and builderType != 'Build':
1001             builder['nextBuild'] = pickLatestBuild
1002
1003         c['builders'].append(builder)
1004
1005 loadBuilderConfig(c)