Performance tester bots should report the number of failing tests
[WebKit-https.git] / Tools / BuildSlaveSupport / build.webkit.org-config / master.cfg
1 # -*- python -*-
2 # ex: set syntax=python:
3
4 from buildbot.buildslave import BuildSlave
5 from buildbot.changes.pb import PBChangeSource
6 from buildbot.scheduler import AnyBranchScheduler, Triggerable
7 from buildbot.schedulers.forcesched import FixedParameter, ForceScheduler, StringParameter
8 from buildbot.schedulers.filter import ChangeFilter
9 from buildbot.status import html
10 from buildbot.status.web.authz import Authz
11 from buildbot.process import buildstep, factory, properties
12 from buildbot.steps import master, shell, source, transfer, trigger
13 from buildbot.status.builder import SUCCESS, FAILURE, WARNINGS, SKIPPED
14
15 from twisted.internet import defer
16
17 import os
18 import re
19 import json
20 import operator
21 import urllib
22
23 from committer_auth import CommitterAuth
24 import wkbuild
25
26
27 c = BuildmasterConfig = {}
28
29 c['change_source'] = PBChangeSource()
30
31 # permissions for WebStatus
32 authz = Authz(
33     auth=CommitterAuth('auth.json'),
34     forceBuild='auth',
35     forceAllBuilds='auth',
36     pingBuilder=True,
37     gracefulShutdown=False,
38     stopBuild='auth',
39     stopAllBuilds='auth',
40     cancelPendingBuild='auth',
41     stopChange=True,
42     cleanShutdown=False)
43
44 c['status'] = []
45 c['status'].append(html.WebStatus(http_port=8710,
46                                   revlink="http://trac.webkit.org/changeset/%s", 
47                                   changecommentlink=(r"(https://bugs\.webkit\.org/show_bug\.cgi\?id=|webkit\.org/b/)(\d+)", r"https://bugs.webkit.org/show_bug.cgi?id=\2"),
48                                   authz=authz))
49
50 c['slavePortnum'] = 17000
51 c['projectName'] = "WebKit"
52 c['projectURL'] = "http://webkit.org"
53 c['buildbotURL'] = "http://build.webkit.org/"
54
55 c['buildHorizon'] = 1000
56 c['logHorizon'] = 500
57 c['eventHorizon'] = 200
58 c['buildCacheSize'] = 60
59
60 WithProperties = properties.WithProperties
61
62
63 class TestWithFailureCount(shell.Test):
64     failedTestsFormatString = "%d tests failed"
65
66     def countFailures(self, cmd):
67         return 0
68
69     def commandComplete(self, cmd):
70         shell.Test.commandComplete(self, cmd)
71         self.failedTestCount = self.countFailures(cmd)
72
73     def evaluateCommand(self, cmd):
74         if self.failedTestCount:
75             return FAILURE
76
77         if cmd.rc != 0:
78             return FAILURE
79
80         return SUCCESS
81
82     def getText(self, cmd, results):
83         return self.getText2(cmd, results)
84
85     def getText2(self, cmd, results):
86         if results != SUCCESS and self.failedTestCount:
87             return [self.failedTestsFormatString % self.failedTestCount]
88
89         return [self.name]
90
91
92 class ConfigureBuild(buildstep.BuildStep):
93     name = "configure build"
94     description = ["configuring build"]
95     descriptionDone = ["configured build"]
96     def __init__(self, platform, configuration, architecture, buildOnly, SVNMirror, *args, **kwargs):
97         buildstep.BuildStep.__init__(self, *args, **kwargs)
98         self.platform = platform.split('-', 1)[0]
99         self.fullPlatform = platform
100         self.configuration = configuration
101         self.architecture = architecture
102         self.buildOnly = buildOnly
103         self.SVNMirror = SVNMirror
104         self.addFactoryArguments(platform=platform, configuration=configuration, architecture=architecture, buildOnly=buildOnly, SVNMirror=SVNMirror)
105
106     def start(self):
107         self.setProperty("platform", self.platform)
108         self.setProperty("fullPlatform", self.fullPlatform)
109         self.setProperty("configuration", self.configuration)
110         self.setProperty("architecture", self.architecture)
111         self.setProperty("buildOnly", self.buildOnly)
112         self.setProperty("shouldAbortEarly", True)
113         self.setProperty("SVNMirror", self.SVNMirror)
114         self.finished(SUCCESS)
115         return defer.succeed(None)
116
117
118 class CheckOutSource(source.SVN):
119     mode = "update"
120     def __init__(self, SVNMirror, **kwargs):
121         kwargs['baseURL'] = SVNMirror or "http://svn.webkit.org/repository/webkit/"
122         kwargs['defaultBranch'] = "trunk"
123         kwargs['mode'] = self.mode
124         source.SVN.__init__(self, **kwargs)
125         self.addFactoryArguments(SVNMirror=SVNMirror)
126
127 class WaitForSVNServer(shell.ShellCommand):
128     name = "wait-for-svn-server"
129     command = ["python", "./Tools/BuildSlaveSupport/wait-for-SVN-server.py", "-r", WithProperties("%(revision)s"), "-s", WithProperties("%(SVNMirror)s")]
130     description = ["waiting for SVN server"]
131     descriptionDone = ["SVN server is ready"]
132     haltOnFailure = True
133
134 class InstallWin32Dependencies(shell.Compile):
135     description = ["installing dependencies"]
136     descriptionDone = ["installed dependencies"]
137     command = ["perl", "./Tools/Scripts/update-webkit-auxiliary-libs"]
138
139 class KillOldProcesses(shell.Compile):
140     name = "kill old processes"
141     description = ["killing old processes"]
142     descriptionDone = ["killed old processes"]
143     command = ["python", "./Tools/BuildSlaveSupport/kill-old-processes"]
144
145 class DeleteStaleBuildFiles(shell.Compile):
146     name = "delete stale build files"
147     description = ["deleting stale build files"]
148     descriptionDone = ["delete stale build files"]
149     command = ["python", "./Tools/BuildSlaveSupport/delete-stale-build-files", WithProperties("--platform=%(fullPlatform)s"), WithProperties("--%(configuration)s")]
150
151 class InstallEflDependencies(shell.ShellCommand):
152     name = "jhbuild"
153     description = ["updating efl dependencies"]
154     descriptionDone = ["updated efl dependencies"]
155     command = ["perl", "./Tools/Scripts/update-webkitefl-libs"]
156     haltOnFailure = True
157
158 class InstallGtkDependencies(shell.ShellCommand):
159     name = "jhbuild"
160     description = ["updating gtk dependencies"]
161     descriptionDone = ["updated gtk dependencies"]
162     command = ["perl", "./Tools/Scripts/update-webkitgtk-libs"]
163     haltOnFailure = True
164
165 def appendCustomBuildFlags(step, platform, fullPlatform=""): 
166     if platform in ('efl', 'gtk', 'qt', 'wincairo', 'wince', 'wx'): 
167         step.setCommand(step.command + ['--' + platform])
168
169 class CompileWebKit(shell.Compile):
170     command = ["perl", "./Tools/Scripts/build-webkit", WithProperties("--%(configuration)s")]
171     env = {'MFLAGS':''}
172     name = "compile-webkit"
173     description = ["compiling"]
174     descriptionDone = ["compiled"]
175     warningPattern = ".*arning: .*"
176
177     def start(self):
178         platform = self.getProperty('platform')
179         buildOnly = self.getProperty('buildOnly')
180         architecture = self.getProperty('architecture')
181         if platform == 'mac' and architecture == 'i386':
182             self.setCommand(self.command + ['--32-bit'])
183         if platform == 'mac' and buildOnly:
184             self.setCommand(self.command + ['DEBUG_INFORMATION_FORMAT=dwarf-with-dsym'])
185
186         appendCustomBuildFlags(self, platform, self.getProperty('fullPlatform'))
187
188         return shell.Compile.start(self)
189
190
191 class CompileWebKit1Only(CompileWebKit):
192     command = ["perl", "./Tools/Scripts/build-webkit", "--no-webkit2", WithProperties("--%(configuration)s")]
193
194
195 class CompileWebKit2Only(CompileWebKit):
196     command = ["perl", "./Tools/Scripts/build-webkit", "--no-webkit1", WithProperties("--%(configuration)s")]
197
198
199 class ArchiveBuiltProduct(shell.ShellCommand):
200     command = ["python", "./Tools/BuildSlaveSupport/built-product-archive",
201                WithProperties("--platform=%(fullPlatform)s"), WithProperties("--%(configuration)s"), "archive"]
202     name = "archive-built-product"
203     description = ["archiving built product"]
204     descriptionDone = ["archived built product"]
205     haltOnFailure = True
206
207
208 class ExtractBuiltProduct(shell.ShellCommand):
209     command = ["python", "./Tools/BuildSlaveSupport/built-product-archive",
210                WithProperties("--platform=%(fullPlatform)s"), WithProperties("--%(configuration)s"), "extract"]
211     name = "extract-built-product"
212     description = ["extracting built product"]
213     descriptionDone = ["extracted built product"]
214     haltOnFailure = True
215
216
217 class UploadBuiltProduct(transfer.FileUpload):
218     slavesrc = WithProperties("WebKitBuild/%(configuration)s.zip")
219     masterdest = WithProperties("archives/%(fullPlatform)s-%(architecture)s-%(configuration)s/%(got_revision)s.zip")
220     haltOnFailure = True
221
222     def __init__(self, **kwargs):
223         kwargs['slavesrc'] = self.slavesrc
224         kwargs['masterdest'] = self.masterdest
225         kwargs['mode'] = 0644
226         transfer.FileUpload.__init__(self, **kwargs)
227
228
229 class DownloadBuiltProduct(shell.ShellCommand):
230     command = ["python", "./Tools/BuildSlaveSupport/download-built-product",
231         WithProperties("--platform=%(platform)s"), WithProperties("--%(configuration)s"),
232         WithProperties(c["buildbotURL"] + "archives/%(fullPlatform)s-%(architecture)s-%(configuration)s/%(got_revision)s.zip")]
233     name = "download-built-product"
234     description = ["downloading built product"]
235     descriptionDone = ["downloaded built product"]
236     haltOnFailure = True
237     flunkOnFailure = True
238
239
240 class RunJavaScriptCoreTests(shell.Test):
241     name = "jscore-test"
242     description = ["jscore-tests running"]
243     descriptionDone = ["jscore-tests"]
244     command = ["perl", "./Tools/Scripts/run-javascriptcore-tests", WithProperties("--%(configuration)s")]
245     logfiles = {'actual.html (source)': 'Source/JavaScriptCore/tests/mozilla/actual.html'}
246
247     def __init__(self, buildJSCTool=True, *args, **kwargs):
248         self.buildJSCTool = buildJSCTool
249         shell.Test.__init__(self, *args, **kwargs)
250         self.addFactoryArguments(buildJSCTool=buildJSCTool)
251
252     def start(self):
253         appendCustomBuildFlags(self, self.getProperty('platform'))
254         if not self.buildJSCTool:
255             self.setCommand(self.command + ['--no-build'])
256         return shell.Test.start(self)
257
258     def commandComplete(self, cmd):
259         shell.Test.commandComplete(self, cmd)
260         logText = cmd.logs['stdio'].getText()
261
262         expressions = [
263             ('failing Mozilla test', re.compile(r'^Results for Mozilla tests:\r?\n\s+(\d+) regression', re.MULTILINE)),
264             ('failing js test', re.compile(r'^Results for LayoutTests/js tests:\r?\n\s+(\d+) failure', re.MULTILINE)),
265             ('crashing js test', re.compile(r'^Results for LayoutTests/js tests:\r?\n.*\n\s+(\d+) crash', re.MULTILINE)),
266             ('failing JSC stress test', re.compile(r'^Results for JSC stress tests:\r?\n\s+(\d+) failure', re.MULTILINE)),
267         ]
268         resultLines = {}
269
270         for name, expression in expressions:
271             match = expression.search(logText)
272             resultLines[name] = int(match.group(1)) if match else 0
273
274         self.regressionLine = []
275         for name in resultLines:
276             failures = resultLines[name]
277             if not failures:
278                 continue
279
280             if failures == 1:
281                 pluralSuffix = ''
282             else:
283                 pluralSuffix = 's'
284             self.regressionLine.append("%d %s%s " % (failures, name, pluralSuffix))
285
286         if 'actual.html (source)' in cmd.logs:
287             self.addHTMLLog('actual.html', cmd.logs['actual.html (source)'].getText())
288
289     def evaluateCommand(self, cmd):
290         if self.regressionLine:
291             return FAILURE
292
293         if cmd.rc != 0:
294             return FAILURE
295
296         return SUCCESS
297
298     def getText(self, cmd, results):
299         return self.getText2(cmd, results)
300
301     def getText2(self, cmd, results):
302         result = [self.name]
303         result.extend(self.regressionLine)
304         return result
305
306 class RunWebKitTests(shell.Test):
307     name = "layout-test"
308     description = ["layout-tests running"]
309     descriptionDone = ["layout-tests"]
310     command = ["perl", "./Tools/Scripts/run-webkit-tests",
311                "--no-show-results",
312                "--no-new-test-results",
313                "--no-sample-on-timeout",
314                "--results-directory", "layout-test-results",
315                "--builder-name", WithProperties("%(buildername)s"),
316                "--build-number", WithProperties("%(buildnumber)s"),
317                "--master-name", "webkit.org",
318                "--test-results-server", "webkit-test-results.appspot.com",
319                WithProperties("--%(configuration)s")]
320
321     def __init__(self, buildJSCTool=True, *args, **kwargs):
322         self.buildJSCTool = buildJSCTool
323         shell.Test.__init__(self, *args, **kwargs)
324         self.addFactoryArguments(buildJSCTool=buildJSCTool)
325
326     def start(self):
327         platform = self.getProperty('platform')
328         shouldAbortEarly = self.getProperty('shouldAbortEarly')
329         appendCustomBuildFlags(self, platform, self.getProperty('fullPlatform'))
330         if platform.startswith('mac'):
331             self.setCommand(self.command + ['--no-build'])
332         if shouldAbortEarly:
333             self.setCommand(self.command + ["--exit-after-n-crashes-or-timeouts", "50", "--exit-after-n-failures", "500"])
334         if platform == "win":
335             rootArgument = ['--root=' + os.path.join("WebKitBuild", self.getProperty('configuration'), "bin32")]
336             self.setCommand(self.command + ['--no-build'])
337         else:
338             rootArgument = ['--root=WebKitBuild/bin']
339         if not self.buildJSCTool:
340             self.setCommand(self.command + rootArgument)
341         return shell.Test.start(self)
342
343     def _parseOldRunWebKitTestsOutput(self, logText):
344         incorrectLayoutLines = []
345         for line in logText.splitlines():
346             if line.find('had incorrect layout') >= 0 or line.find('were new') >= 0 or line.find('was new') >= 0:
347                 incorrectLayoutLines.append(line)
348             elif line.find('test case') >= 0 and (line.find(' crashed') >= 0 or line.find(' timed out') >= 0):
349                 incorrectLayoutLines.append(line)
350             elif line.startswith("WARNING:") and line.find(' leak') >= 0:
351                 incorrectLayoutLines.append(line.replace('WARNING: ', ''))
352             elif line.find('Exiting early') >= 0:
353                 incorrectLayoutLines.append(line)
354
355             # FIXME: Detect and summarize leaks of RefCounted objects
356
357         self.incorrectLayoutLines = incorrectLayoutLines
358
359     # FIXME: This will break if new-run-webkit-tests changes its default log formatter.
360     nrwt_log_message_regexp = re.compile(r'\d{2}:\d{2}:\d{2}(\.\d+)?\s+\d+\s+(?P<message>.*)')
361
362     def _strip_python_logging_prefix(self, line):
363         match_object = self.nrwt_log_message_regexp.match(line)
364         if match_object:
365             return match_object.group('message')
366         return line
367
368     def _parseNewRunWebKitTestsOutput(self, logText):
369         incorrectLayoutLines = []
370         expressions = [
371             ('flakes', re.compile(r'Unexpected flakiness.+\((\d+)\)')),
372             ('new passes', re.compile(r'Expected to .+, but passed:\s+\((\d+)\)')),
373             ('missing results', re.compile(r'Regressions: Unexpected missing results\s+\((\d+)\)')),
374             ('failures', re.compile(r'Regressions: Unexpected.+\((\d+)\)')),
375         ]
376         testFailures = {}
377
378         for line in logText.splitlines():
379             if line.find('Exiting early') >= 0 or line.find('leaks found') >= 0:
380                 incorrectLayoutLines.append(self._strip_python_logging_prefix(line))
381                 continue
382             for name, expression in expressions:
383                 match = expression.search(line)
384
385                 if match:
386                     testFailures[name] = testFailures.get(name, 0) + int(match.group(1))
387                     break
388
389                 # FIXME: Parse file names and put them in results
390
391         for name in testFailures:
392             incorrectLayoutLines.append(str(testFailures[name]) + ' ' + name)
393
394         self.incorrectLayoutLines = incorrectLayoutLines
395
396     def commandComplete(self, cmd):
397         shell.Test.commandComplete(self, cmd)
398
399         logText = cmd.logs['stdio'].getText()
400         if logText.find("Collecting tests ...") >= 0:
401             self._parseNewRunWebKitTestsOutput(logText)
402         else:
403             self._parseOldRunWebKitTestsOutput(logText)
404
405     def evaluateCommand(self, cmd):
406         result = SUCCESS
407
408         if self.incorrectLayoutLines:
409             if len(self.incorrectLayoutLines) == 1:
410                 line = self.incorrectLayoutLines[0]
411                 if line.find('were new') >= 0 or line.find('was new') >= 0 or line.find(' leak') >= 0:
412                     return WARNINGS
413
414             for line in self.incorrectLayoutLines:
415                 if line.find('flakes') >= 0 or line.find('new passes') >= 0 or line.find('missing results') >= 0:
416                     result = WARNINGS
417                 else:
418                     return FAILURE
419
420         if cmd.rc != 0:
421             return FAILURE
422
423         return result
424
425     def getText(self, cmd, results):
426         return self.getText2(cmd, results)
427
428     def getText2(self, cmd, results):
429         if results != SUCCESS and self.incorrectLayoutLines:
430             return self.incorrectLayoutLines
431
432         return [self.name]
433
434
435 class RunUnitTests(TestWithFailureCount):
436     name = "run-api-tests"
437     description = ["unit tests running"]
438     descriptionDone = ["unit-tests"]
439     command = ["perl", "./Tools/Scripts/run-api-tests", WithProperties("--%(configuration)s"), "--verbose"]
440     failedTestsFormatString = "%d unit tests failed or timed out"
441
442     def start(self):
443         platform = self.getProperty('fullPlatform')
444         if platform.startswith('win'):
445             self.setCommand(self.command + ['--no-build'])
446         if platform.startswith('mac'):
447             self.setCommand(self.command + ['--no-build'])
448         return shell.Test.start(self)
449
450     def countFailures(self, cmd):
451         log_text = cmd.logs['stdio'].getText()
452         count = 0
453
454         split = re.split(r'\sTests that timed out:\s', log_text)
455         if len(split) > 1:
456             count += len(re.findall(r'^\s+\S+$', split[1], flags=re.MULTILINE))
457
458         split = re.split(r'\sTests that failed:\s', split[0])
459         if len(split) > 1:
460             count += len(re.findall(r'^\s+\S+$', split[1], flags=re.MULTILINE))
461
462         return count
463
464
465 class RunPythonTests(TestWithFailureCount):
466     name = "webkitpy-test"
467     description = ["python-tests running"]
468     descriptionDone = ["python-tests"]
469     command = ["python", "./Tools/Scripts/test-webkitpy"]
470     failedTestsFormatString = "%d python tests failed"
471
472     def start(self):
473         platform = self.getProperty('platform')
474         # Python tests are flaky on the GTK builders, running them serially
475         # helps and does not significantly prolong the cycle time.
476         if platform == 'gtk':
477             self.setCommand(self.command + ['--child-processes', '1'])
478         # Python tests fail on windows bots when running more than one child process
479         # https://bugs.webkit.org/show_bug.cgi?id=97465
480         if platform == 'win':
481             self.setCommand(self.command + ['--child-processes', '1'])
482         return shell.Test.start(self)
483
484     def countFailures(self, cmd):
485         logText = cmd.logs['stdio'].getText()
486         # We're looking for the line that looks like this: FAILED (failures=2, errors=1)
487         regex = re.compile(r'^FAILED \((?P<counts>[^)]+)\)')
488         for line in logText.splitlines():
489             match = regex.match(line)
490             if not match:
491                 continue
492             return sum(int(component.split('=')[1]) for component in match.group('counts').split(', '))
493         return 0
494
495
496 class RunPerlTests(TestWithFailureCount):
497     name = "webkitperl-test"
498     description = ["perl-tests running"]
499     descriptionDone = ["perl-tests"]
500     command = ["perl", "./Tools/Scripts/test-webkitperl"]
501     failedTestsFormatString = "%d perl tests failed"
502
503     def countFailures(self, cmd):
504         logText = cmd.logs['stdio'].getText()
505         # We're looking for the line that looks like this: Failed 2/19 test programs. 5/363 subtests failed.
506         regex = re.compile(r'^Failed \d+/\d+ test programs\. (?P<count>\d+)/\d+ subtests failed\.')
507         for line in logText.splitlines():
508             match = regex.match(line)
509             if not match:
510                 continue
511             return int(match.group('count'))
512         return 0
513
514
515 class RunBindingsTests(shell.Test):
516     name = "bindings-generation-tests"
517     description = ["bindings-tests running"]
518     descriptionDone = ["bindings-tests"]
519     command = ["python", "./Tools/Scripts/run-bindings-tests"]
520
521
522 class RunEflAPITests(shell.Test):
523     name = "API tests"
524     description = ["API tests running"]
525     descriptionDone = ["API tests"]
526     command = ["perl", "./Tools/Scripts/run-efl-tests", WithProperties("--%(configuration)s")]
527
528
529 class RunGtkAPITests(shell.Test):
530     name = "API tests"
531     description = ["API tests running"]
532     descriptionDone = ["API tests"]
533     command = ["python", "./Tools/Scripts/run-gtk-tests", "--verbose", WithProperties("--%(configuration)s")]
534
535     def commandComplete(self, cmd):
536         shell.Test.commandComplete(self, cmd)
537
538         logText = cmd.logs['stdio'].getText()
539
540         self.incorrectTests = 0
541         self.timedOutTests = 0
542         self.skippedTests = 0
543         self.statusLine = []
544
545         foundItems = re.findall("Tests failed \((\d+)\):", logText)
546         if (foundItems):
547             self.incorrectTests = int(foundItems[0])
548
549         foundItems = re.findall("Tests that timed out \((\d+)\):", logText)
550         if (foundItems):
551             self.timedOutTests = int(foundItems[0])
552
553         foundItems = re.findall("Tests skipped \((\d+)\):", logText)
554         if (foundItems):
555             self.skippedTests = int(foundItems[0])
556
557         self.totalFailedTests = self.incorrectTests + self.timedOutTests
558
559         if self.totalFailedTests > 0:
560             self.statusLine = [
561                 "%d API tests failed, %d timed out, %d skipped" %
562                 (self.incorrectTests, self.timedOutTests, self.skippedTests)
563             ]
564
565     def evaluateCommand(self, cmd):
566         if self.totalFailedTests > 0:
567             return FAILURE
568
569         if cmd.rc != 0:
570             return FAILURE
571
572         return SUCCESS
573
574     def getText(self, cmd, results):
575         return self.getText2(cmd, results)
576
577     def getText2(self, cmd, results):
578         if results != SUCCESS and self.totalFailedTests > 0:
579             return self.statusLine
580
581         return [self.name]
582
583 class RunQtAPITests(shell.Test):
584     name = "API tests"
585     description = ["API tests running"]
586     descriptionDone = ["API tests"]
587
588     def start(self):
589         self.setCommand( ["python", "./Tools/Scripts/run-qtwebkit-tests", "--output-file=qt-unit-tests.html", "--do-not-open-results", "--timeout=120", "--%s" % self.getProperty('configuration')] )
590         if self.getProperty('fullPlatform').endswith("-wk2"):
591             self.setCommand(self.command + ['--webkit2'])
592         return shell.Test.start(self)
593
594     def commandComplete(self, cmd):
595         shell.Test.commandComplete(self, cmd)
596
597         logText = cmd.logs['stdio'].getText()
598         foundItems = re.findall("TOTALS: (?P<passed>\d+) passed, (?P<failed>\d+) failed, (?P<skipped>\d+) skipped, (?P<crashed>\d+) crashed", logText)
599
600         self.incorrectTests = 0
601         self.crashedTests = 0
602         self.statusLine = []
603
604         if foundItems:
605             self.incorrectTests = int(foundItems[0][1])
606             self.crashedTests = int(foundItems[0][3])
607
608             if self.incorrectTests > 0 or self.crashedTests > 0:
609                 self.statusLine = [
610                     "%s passed, %s failed, %s skipped, %s crashed" % (foundItems[0][0], foundItems[0][1], foundItems[0][2], foundItems[0][3])
611                 ]
612
613     def evaluateCommand(self, cmd):
614         if self.crashedTests:
615             return FAILURE
616
617         if re.findall("Timeout, process", cmd.logs['stdio'].getText()):
618             self.statusLine = ["Failure: timeout occured during testing"]
619             return FAILURE
620
621         if self.incorrectTests:
622             return WARNINGS
623
624         if cmd.rc != 0:
625             return FAILURE
626
627         return SUCCESS
628
629     def getText(self, cmd, results):
630         return self.getText2(cmd, results)
631
632     def getText2(self, cmd, results):
633         if results != SUCCESS and self.incorrectTests:
634             return self.statusLine
635
636         return [self.name]
637
638 class RunWebKitLeakTests(RunWebKitTests):
639     warnOnWarnings = True
640     def start(self):
641         self.setCommand(self.command + ["--leaks"])
642         return RunWebKitTests.start(self)
643
644
645 class RunWebKit2Tests(RunWebKitTests):
646     def start(self):
647         self.setProperty("shouldAbortEarly", False)
648         self.setCommand(self.command + ["--webkit-test-runner"])
649         if self.getProperty('buildername') == "Qt Mountain Lion Release":
650             self.setCommand(self.command + ["-p"])
651
652         return RunWebKitTests.start(self)
653
654
655 class RunAndUploadPerfTests(shell.Test):
656    name = "perf-test"
657    description = ["perf-tests running"]
658    descriptionDone = ["perf-tests"]
659    command = ["python", "./Tools/Scripts/run-perf-tests",
660               "--output-json-path", "perf-test-results.json",
661               "--slave-config-json-path", "../../perf-test-config.json",
662               "--no-show-results",
663               "--reset-results",
664               "--test-results-server", "perf.webkit.org",
665               "--builder-name", WithProperties("%(buildername)s"),
666               "--build-number", WithProperties("%(buildnumber)s"),
667               "--platform", WithProperties("%(fullPlatform)s"),
668               WithProperties("--%(configuration)s")]
669
670    def start(self):
671        self.setCommand(self.command)
672        return shell.Test.start(self)
673
674     def getText(self, cmd, results):
675         return self.getText2(cmd, results)
676
677     def getText2(self, cmd, results):
678         if results != SUCCESS
679             return ["%d perf tests failed" % cmd.rc]
680
681         return [self.name]
682
683
684 class RunAndUploadPerfTestsWebKit2(RunAndUploadPerfTests):
685     def start(self):
686         self.setCommand(self.command + ["--webkit-test-runner"])
687         return RunAndUploadPerfTests.start(self)
688
689
690 class ArchiveTestResults(shell.ShellCommand):
691     command = ["python", "./Tools/BuildSlaveSupport/test-result-archive",
692                WithProperties("--platform=%(platform)s"), WithProperties("--%(configuration)s"), "archive"]
693     name = "archive-test-results"
694     description = ["archiving test results"]
695     descriptionDone = ["archived test results"]
696     haltOnFailure = True
697
698
699 class UploadTestResults(transfer.FileUpload):
700     slavesrc = "layout-test-results.zip"
701     masterdest = WithProperties("public_html/results/%(buildername)s/r%(got_revision)s (%(buildnumber)s).zip")
702
703     def __init__(self, **kwargs):
704         kwargs['slavesrc'] = self.slavesrc
705         kwargs['masterdest'] = self.masterdest
706         kwargs['mode'] = 0644
707         transfer.FileUpload.__init__(self, **kwargs)
708
709
710 class ExtractTestResults(master.MasterShellCommand):
711     zipFile = WithProperties("public_html/results/%(buildername)s/r%(got_revision)s (%(buildnumber)s).zip")
712     resultDirectory = WithProperties("public_html/results/%(buildername)s/r%(got_revision)s (%(buildnumber)s)")
713     descriptionDone = ["uploaded results"]
714
715     def __init__(self, **kwargs):
716         kwargs['command'] = ""
717         master.MasterShellCommand.__init__(self, **kwargs)
718
719     def resultDirectoryURL(self):
720         return self.build.getProperties().render(self.resultDirectory).replace("public_html/", "/") + "/"
721
722     def start(self):
723         self.command = ["unzip", self.build.getProperties().render(self.zipFile), "-d", self.build.getProperties().render(self.resultDirectory)]
724         return master.MasterShellCommand.start(self)
725
726     def addCustomURLs(self):
727         url = self.resultDirectoryURL() + "results.html"
728         self.addURL("view results", url)
729
730     def finished(self, result):
731         self.addCustomURLs()
732         return master.MasterShellCommand.finished(self, result)
733
734
735 class ExtractTestResultsAndLeaks(ExtractTestResults):
736     def addCustomURLs(self):
737         ExtractTestResults.addCustomURLs(self)
738         url = "/LeaksViewer/?url=" + urllib.quote(self.resultDirectoryURL(), safe="")
739         self.addURL("view leaks", url)
740
741
742 class Factory(factory.BuildFactory):
743     def __init__(self, platform, configuration, architectures, buildOnly, SVNMirror):
744         factory.BuildFactory.__init__(self)
745         self.addStep(ConfigureBuild(platform=platform, configuration=configuration, architecture=" ".join(architectures), buildOnly=buildOnly, SVNMirror=SVNMirror))
746         if SVNMirror:
747             self.addStep(WaitForSVNServer())
748         self.addStep(CheckOutSource(SVNMirror=SVNMirror))
749         # There are multiple Qt slaves running on same machines, so buildslaves shouldn't kill the processes of other slaves.
750         if not platform.startswith("qt"):
751             self.addStep(KillOldProcesses())
752         self.addStep(DeleteStaleBuildFiles())
753         if platform == "win":
754             self.addStep(InstallWin32Dependencies())
755         if platform == "gtk":
756             self.addStep(InstallGtkDependencies())
757         if platform == "efl":
758             self.addStep(InstallEflDependencies())
759
760
761 class BuildFactory(Factory):
762     def __init__(self, platform, configuration, architectures, triggers=None, SVNMirror=None):
763         Factory.__init__(self, platform, configuration, architectures, True, SVNMirror)
764         self.addStep(CompileWebKit())
765         if triggers:
766             self.addStep(ArchiveBuiltProduct())
767             self.addStep(UploadBuiltProduct())
768             self.addStep(trigger.Trigger(schedulerNames=triggers))
769
770 def unitTestsSupported(configuration, platform):
771     if platform.startswith('mac') and configuration == "release":
772         return False; # https://bugs.webkit.org/show_bug.cgi?id=82652
773     return platform == 'win' or platform.startswith('mac')
774
775 def pickLatestBuild(builder, requests):
776     return max(requests, key=operator.attrgetter("submittedAt"))
777
778 class TestFactory(Factory):
779     LayoutTestClass = RunWebKitTests
780     ExtractTestResultsClass = ExtractTestResults
781     def __init__(self, platform, configuration, architectures, SVNMirror=None):
782         Factory.__init__(self, platform, configuration, architectures, False, SVNMirror)
783         self.addStep(DownloadBuiltProduct())
784         self.addStep(ExtractBuiltProduct())
785         self.addStep(RunJavaScriptCoreTests(buildJSCTool=False))
786         if self.LayoutTestClass:
787             self.addStep(self.LayoutTestClass(buildJSCTool=(platform != 'win')))
788
789         if unitTestsSupported(configuration, platform): 
790             self.addStep(RunUnitTests())
791         self.addStep(RunPythonTests())
792         self.addStep(RunPerlTests())
793         self.addStep(RunBindingsTests())
794         if self.LayoutTestClass:
795             self.addStep(ArchiveTestResults())
796             self.addStep(UploadTestResults())
797             self.addStep(self.ExtractTestResultsClass())
798         if platform == "efl":
799             self.addStep(RunEflAPITests)
800         if platform == "gtk":
801             self.addStep(RunGtkAPITests())
802         if platform.startswith("qt"):
803             self.addStep(RunQtAPITests)
804
805 class BuildAndTestFactory(Factory):
806     CompileClass = CompileWebKit
807     LayoutTestClass = RunWebKitTests
808     ExtractTestResultsClass = ExtractTestResults
809     def __init__(self, platform, configuration, architectures, triggers=None, SVNMirror=None, **kwargs):
810         Factory.__init__(self, platform, configuration, architectures, False, SVNMirror, **kwargs)
811         self.addStep(self.CompileClass())
812         self.addStep(RunJavaScriptCoreTests())
813         if self.LayoutTestClass:
814             self.addStep(self.LayoutTestClass())
815         if unitTestsSupported(configuration, platform): 
816             self.addStep(RunUnitTests())
817         self.addStep(RunPythonTests())
818         self.addStep(RunPerlTests())
819         self.addStep(RunBindingsTests())
820         if self.LayoutTestClass:
821             self.addStep(ArchiveTestResults())
822             self.addStep(UploadTestResults())
823             self.addStep(self.ExtractTestResultsClass())
824         if platform == "efl":
825             self.addStep(RunEflAPITests())
826         if platform == "gtk":
827             self.addStep(RunGtkAPITests())
828         if platform.startswith("qt"):
829             self.addStep(RunQtAPITests())
830         if triggers:
831             self.addStep(ArchiveBuiltProduct())
832             self.addStep(UploadBuiltProduct())
833             self.addStep(trigger.Trigger(schedulerNames=triggers))
834
835 class BuildAndTestWebKit2Factory(BuildAndTestFactory):
836     CompileClass = CompileWebKit
837     LayoutTestClass = RunWebKit2Tests
838
839 class BuildAndTestWebKit1OnlyFactory(BuildAndTestFactory):
840     CompileClass = CompileWebKit1Only
841
842 class BuildAndTestWebKit2OnlyFactory(BuildAndTestFactory):
843     CompileClass = CompileWebKit2Only
844     LayoutTestClass = RunWebKit2Tests
845
846 class BuildAndNonLayoutTestFactory(BuildAndTestFactory):
847     LayoutTestClass = None
848
849 class TestLeaksFactory(TestFactory):
850     LayoutTestClass = RunWebKitLeakTests
851     ExtractTestResultsClass = ExtractTestResultsAndLeaks
852
853
854 class TestWebKit2Factory(TestFactory):
855     LayoutTestClass = RunWebKit2Tests
856
857 class BuildAndPerfTestFactory(Factory):
858     def __init__(self, platform, configuration, architectures, SVNMirror=None, **kwargs):
859         Factory.__init__(self, platform, configuration, architectures, False, SVNMirror, **kwargs)
860         self.addStep(CompileWebKit())
861         self.addStep(RunAndUploadPerfTests())
862
863 class BuildAndPerfTestWebKit2Factory(Factory):
864     def __init__(self, platform, configuration, architectures, SVNMirror=None, **kwargs):
865         Factory.__init__(self, platform, configuration, architectures, False, SVNMirror, **kwargs)
866         self.addStep(CompileWebKit())
867         self.addStep(RunAndUploadPerfTestsWebKit2())
868
869 class DownloadAndPerfTestFactory(Factory):
870     def __init__(self, platform, configuration, architectures, SVNMirror=None, **kwargs):
871         Factory.__init__(self, platform, configuration, architectures, False, SVNMirror, **kwargs)
872         self.addStep(DownloadBuiltProduct())
873         self.addStep(ExtractBuiltProduct())
874         self.addStep(RunAndUploadPerfTests())
875
876 class DownloadAndPerfTestWebKit2Factory(Factory):
877     def __init__(self, platform, configuration, architectures, SVNMirror=None, **kwargs):
878         Factory.__init__(self, platform, configuration, architectures, False, SVNMirror, **kwargs)
879         self.addStep(DownloadBuiltProduct())
880         self.addStep(ExtractBuiltProduct())
881         self.addStep(RunAndUploadPerfTestsWebKit2())
882
883 class PlatformSpecificScheduler(AnyBranchScheduler):
884     def __init__(self, platform, branch, **kwargs):
885         self.platform = platform
886         filter = ChangeFilter(branch=[branch, None], filter_fn=self.filter)
887         AnyBranchScheduler.__init__(self, name=platform, change_filter=filter, **kwargs)
888
889     def filter(self, change):
890         return wkbuild.should_build(self.platform, change.files)
891
892 trunk_filter = ChangeFilter(branch=["trunk", None])
893
894 def loadBuilderConfig(c):
895     # FIXME: These file handles are leaked.
896     passwords = json.load(open('passwords.json'))
897     config = json.load(open('config.json'))
898
899     c['slaves'] = [BuildSlave(slave['name'], passwords[slave['name']], max_builds=1) for slave in config['slaves']]
900
901     c['schedulers'] = []
902     for scheduler in config['schedulers']:
903         if "change_filter" in scheduler:
904             scheduler["change_filter"] = globals()[scheduler["change_filter"]]
905         kls = globals()[scheduler.pop('type')]
906         # Python 2.6 can't handle unicode keys as keyword arguments:
907         # http://bugs.python.org/issue2646.  Modern versions of json return
908         # unicode strings from json.load, so we map all keys to str objects.
909         scheduler = dict(map(lambda key_value_pair: (str(key_value_pair[0]), key_value_pair[1]), scheduler.items()))
910
911         c['schedulers'].append(kls(**scheduler))
912
913     forceScheduler = ForceScheduler(
914         name="force",
915         builderNames=[builder['name'] for builder in config['builders']],
916         reason=StringParameter(name="reason", default="", size=40),
917
918         # Validate SVN revision: number or emtpy string
919         revision=StringParameter(name="revision", default="", regex=re.compile(r'^(\d*)$')),
920
921         # Disable default enabled input fields: branch, repository, project, additional properties
922         branch=FixedParameter(name="branch"),
923         repository=FixedParameter(name="repository"),
924         project=FixedParameter(name="project"),
925         properties=[]
926     )
927     c['schedulers'].append(forceScheduler)
928
929     c['builders'] = []
930     for builder in config['builders']:
931         for slaveName in builder['slavenames']:
932             for slave in config['slaves']:
933                 if slave['name'] != slaveName or slave['platform'] == '*':
934                     continue
935
936                 if slave['platform'] != builder['platform']:
937                     raise Exception, "Builder %r is for platform %r but has slave %r for platform %r!" % (builder['name'], builder['platform'], slave['name'], slave['platform'])
938
939                 break
940
941         platform = builder['platform']
942
943         builderType = builder.pop('type')
944         factory = globals()["%sFactory" % builderType]
945         factorykwargs = {}
946         for key in "platform", "configuration", "architectures", "triggers", "SVNMirror":
947             value = builder.pop(key, None)
948             if value:
949                 factorykwargs[key] = value
950
951         builder["factory"] = factory(**factorykwargs)
952
953         if platform.startswith('mac'):
954             builder["category"] = 'AppleMac'
955         elif platform == 'win':
956             builder["category"] = 'AppleWin'
957         elif platform.startswith('gtk'):
958             builder["category"] = 'GTK'
959         elif platform.startswith('qt'):
960             builder["category"] = 'Qt'
961         elif platform.startswith('efl'):
962             builder["category"] = "EFL"
963         else:
964             builder["category"] = 'misc'
965
966         if (builder['category'] == 'AppleMac' or builder['category'] == 'AppleWin') and builderType != 'Build':
967             builder['nextBuild'] = pickLatestBuild
968
969         c['builders'].append(builder)
970
971 loadBuilderConfig(c)