d0771032f23c3d49d4b408a5e637f552307c7358
[WebKit-https.git] / Tools / BuildSlaveSupport / build.webkit.org-config / master.cfg
1 # -*- python -*-
2 # ex: set syntax=python:
3
4 from buildbot.buildslave import BuildSlave
5 from buildbot.changes.pb import PBChangeSource
6 from buildbot.scheduler import AnyBranchScheduler, Triggerable
7 from buildbot.schedulers.forcesched import FixedParameter, ForceScheduler, StringParameter, BooleanParameter
8 from buildbot.schedulers.filter import ChangeFilter
9 from buildbot.status import html
10 from buildbot.status.web.authz import Authz
11 from buildbot.process import buildstep, factory, properties
12 from buildbot.steps import master, shell, source, transfer, trigger
13 from buildbot.status.builder import SUCCESS, FAILURE, WARNINGS, SKIPPED
14
15 from twisted.internet import defer
16
17 import os
18 import re
19 import json
20 import operator
21 import cStringIO
22 import urllib
23
24 from committer_auth import CommitterAuth
25 import wkbuild
26
27
28 c = BuildmasterConfig = {}
29
30 c['change_source'] = PBChangeSource()
31
32 # permissions for WebStatus
33 authz = Authz(
34     auth=CommitterAuth('auth.json'),
35     forceBuild='auth',
36     forceAllBuilds='auth',
37     pingBuilder=True,
38     gracefulShutdown=False,
39     stopBuild='auth',
40     stopAllBuilds='auth',
41     cancelPendingBuild='auth',
42     stopChange=True,
43     cleanShutdown=False)
44
45 c['status'] = []
46 c['status'].append(html.WebStatus(http_port=8710,
47                                   revlink="https://trac.webkit.org/changeset/%s", 
48                                   changecommentlink=(r"(https://bugs\.webkit\.org/show_bug\.cgi\?id=|webkit\.org/b/)(\d+)", r"https://bugs.webkit.org/show_bug.cgi?id=\2"),
49                                   authz=authz))
50
51 c['slavePortnum'] = 17000
52 c['projectName'] = "WebKit"
53 c['projectURL'] = "https://webkit.org"
54 c['buildbotURL'] = "https://build.webkit.org/"
55
56 c['buildHorizon'] = 1000
57 c['logHorizon'] = 500
58 c['eventHorizon'] = 200
59 c['buildCacheSize'] = 60
60
61 WithProperties = properties.WithProperties
62
63
64 class TestWithFailureCount(shell.Test):
65     failedTestsFormatString = "%d test%s failed"
66
67     def countFailures(self, cmd):
68         return 0
69
70     def commandComplete(self, cmd):
71         shell.Test.commandComplete(self, cmd)
72         self.failedTestCount = self.countFailures(cmd)
73         self.failedTestPluralSuffix = "" if self.failedTestCount == 1 else "s"
74
75     def evaluateCommand(self, cmd):
76         if self.failedTestCount:
77             return FAILURE
78
79         if cmd.rc != 0:
80             return FAILURE
81
82         return SUCCESS
83
84     def getText(self, cmd, results):
85         return self.getText2(cmd, results)
86
87     def getText2(self, cmd, results):
88         if results != SUCCESS and self.failedTestCount:
89             return [self.failedTestsFormatString % (self.failedTestCount, self.failedTestPluralSuffix)]
90
91         return [self.name]
92
93
94 class ConfigureBuild(buildstep.BuildStep):
95     name = "configure build"
96     description = ["configuring build"]
97     descriptionDone = ["configured build"]
98     def __init__(self, platform, configuration, architecture, buildOnly, additionalArguments, SVNMirror, *args, **kwargs):
99         buildstep.BuildStep.__init__(self, *args, **kwargs)
100         self.platform = platform
101         if platform != 'jsc-only':
102             self.platform = platform.split('-', 1)[0]
103         self.fullPlatform = platform
104         self.configuration = configuration
105         self.architecture = architecture
106         self.buildOnly = buildOnly
107         self.additionalArguments = additionalArguments
108         self.SVNMirror = SVNMirror
109         self.addFactoryArguments(platform=platform, configuration=configuration, architecture=architecture, buildOnly=buildOnly, additionalArguments=additionalArguments, SVNMirror=SVNMirror)
110
111     def start(self):
112         self.setProperty("platform", self.platform)
113         self.setProperty("fullPlatform", self.fullPlatform)
114         self.setProperty("configuration", self.configuration)
115         self.setProperty("architecture", self.architecture)
116         self.setProperty("buildOnly", self.buildOnly)
117         self.setProperty("additionalArguments", self.additionalArguments)
118         self.setProperty("SVNMirror", self.SVNMirror)
119         self.finished(SUCCESS)
120         return defer.succeed(None)
121
122
123 class CheckOutSource(source.SVN):
124     mode = "update"
125     def __init__(self, SVNMirror, **kwargs):
126         kwargs['baseURL'] = SVNMirror or "http://svn.webkit.org/repository/webkit/"
127         kwargs['defaultBranch'] = "trunk"
128         kwargs['mode'] = self.mode
129         source.SVN.__init__(self, **kwargs)
130         self.addFactoryArguments(SVNMirror=SVNMirror)
131
132 class WaitForSVNServer(shell.ShellCommand):
133     name = "wait-for-svn-server"
134     command = ["python", "./Tools/BuildSlaveSupport/wait-for-SVN-server.py", "-r", WithProperties("%(revision)s"), "-s", WithProperties("%(SVNMirror)s")]
135     description = ["waiting for SVN server"]
136     descriptionDone = ["SVN server is ready"]
137     warnOnFailure = True
138
139     def evaluateCommand(self, cmd):
140         if cmd.rc != 0:
141             return WARNINGS
142         return SUCCESS
143
144 class InstallWin32Dependencies(shell.Compile):
145     description = ["installing dependencies"]
146     descriptionDone = ["installed dependencies"]
147     command = ["perl", "./Tools/Scripts/update-webkit-auxiliary-libs"]
148
149 class KillOldProcesses(shell.Compile):
150     name = "kill old processes"
151     description = ["killing old processes"]
152     descriptionDone = ["killed old processes"]
153     command = ["python", "./Tools/BuildSlaveSupport/kill-old-processes"]
154
155 class CleanBuildIfScheduled(shell.Compile):
156     name = "delete WebKitBuild directory"
157     description = ["deleting WebKitBuild directory"]
158     descriptionDone = ["deleted WebKitBuild directory"]
159     command = ["python", "./Tools/BuildSlaveSupport/clean-build", WithProperties("--platform=%(fullPlatform)s"), WithProperties("--%(configuration)s")]
160
161     def start(self):
162         if not self.getProperty('is_clean'):
163             self.hideStepIf = True
164             return SKIPPED
165         return shell.Compile.start(self)
166
167 class DeleteStaleBuildFiles(shell.Compile):
168     name = "delete stale build files"
169     description = ["deleting stale build files"]
170     descriptionDone = ["deleted stale build files"]
171     command = ["python", "./Tools/BuildSlaveSupport/delete-stale-build-files", WithProperties("--platform=%(fullPlatform)s"), WithProperties("--%(configuration)s")]
172
173     def start(self):
174         if self.getProperty('is_clean'): # Nothing to be done if WebKitBuild had been removed.
175             self.hideStepIf = True
176             return SKIPPED
177         return shell.Compile.start(self)
178
179 class InstallEflDependencies(shell.ShellCommand):
180     name = "jhbuild"
181     description = ["updating efl dependencies"]
182     descriptionDone = ["updated efl dependencies"]
183     command = ["perl", "./Tools/Scripts/update-webkitefl-libs"]
184     haltOnFailure = True
185
186 class InstallGtkDependencies(shell.ShellCommand):
187     name = "jhbuild"
188     description = ["updating gtk dependencies"]
189     descriptionDone = ["updated gtk dependencies"]
190     command = ["perl", "./Tools/Scripts/update-webkitgtk-libs"]
191     haltOnFailure = True
192
193 def appendCustomBuildFlags(step, platform, fullPlatform):
194     if platform not in ('efl', 'gtk', 'wincairo', 'ios', 'jsc-only'):
195         return
196     if fullPlatform.startswith('ios-simulator'):
197         platform = 'ios-simulator'
198     elif platform == 'ios':
199         platform = 'device'
200     step.setCommand(step.command + ['--' + platform])
201
202 class CompileWebKit(shell.Compile):
203     command = ["perl", "./Tools/Scripts/build-webkit", WithProperties("--%(configuration)s")]
204     env = {'MFLAGS':''}
205     name = "compile-webkit"
206     description = ["compiling"]
207     descriptionDone = ["compiled"]
208     warningPattern = ".*arning: .*"
209
210     def start(self):
211         platform = self.getProperty('platform')
212         buildOnly = self.getProperty('buildOnly')
213         architecture = self.getProperty('architecture')
214         additionalArguments = self.getProperty('additionalArguments')
215
216         if additionalArguments:
217             self.setCommand(self.command + additionalArguments)
218         if platform in ('mac', 'ios') and architecture:
219             self.setCommand(self.command + ['ARCHS=' + architecture])
220             if platform == 'ios':
221                 self.setCommand(self.command + ['ONLY_ACTIVE_ARCH=NO'])
222         # Generating dSYM files is slow, but these are needed to have line numbers in crash reports on testers.
223         # Debug builds on Yosemite can't use dSYMs, because crash logs end up unsymbolicated.
224         if platform in ('mac', 'ios') and buildOnly and (self.getProperty('fullPlatform') != "mac-yosemite" or self.getProperty('configuration') != "debug"):
225             self.setCommand(self.command + ['DEBUG_INFORMATION_FORMAT=dwarf-with-dsym'])
226
227         appendCustomBuildFlags(self, platform, self.getProperty('fullPlatform'))
228
229         return shell.Compile.start(self)
230
231     def createSummary(self, log):
232         platform = self.getProperty('platform')
233         if platform.startswith('mac'):    
234             warnings = []
235             errors = []
236             sio = cStringIO.StringIO(log.getText())
237             for line in sio.readlines():
238                 if "arning:" in line:
239                     warnings.append(line)
240                 if "rror:" in line:
241                     errors.append(line)
242             if warnings:
243                 self.addCompleteLog('warnings', "".join(warnings))
244             if errors:
245                 self.addCompleteLog('errors', "".join(errors))
246
247
248 class CompileLLINTCLoop(CompileWebKit):
249     command = ["perl", "./Tools/Scripts/build-jsc", "--cloop", WithProperties("--%(configuration)s")]
250
251 class Compile32bitJSC(CompileWebKit):
252     command = ["perl", "./Tools/Scripts/build-jsc", "--32-bit", WithProperties("--%(configuration)s")]
253
254 class CompileJSCOnly(CompileWebKit):
255     command = ["perl", "./Tools/Scripts/build-jsc", WithProperties("--%(configuration)s")]
256
257 class ArchiveBuiltProduct(shell.ShellCommand):
258     command = ["python", "./Tools/BuildSlaveSupport/built-product-archive",
259                WithProperties("--platform=%(fullPlatform)s"), WithProperties("--%(configuration)s"), "archive"]
260     name = "archive-built-product"
261     description = ["archiving built product"]
262     descriptionDone = ["archived built product"]
263     haltOnFailure = True
264
265
266 class ExtractBuiltProduct(shell.ShellCommand):
267     command = ["python", "./Tools/BuildSlaveSupport/built-product-archive",
268                WithProperties("--platform=%(fullPlatform)s"), WithProperties("--%(configuration)s"), "extract"]
269     name = "extract-built-product"
270     description = ["extracting built product"]
271     descriptionDone = ["extracted built product"]
272     haltOnFailure = True
273
274
275 class UploadBuiltProduct(transfer.FileUpload):
276     slavesrc = WithProperties("WebKitBuild/%(configuration)s.zip")
277     masterdest = WithProperties("archives/%(fullPlatform)s-%(architecture)s-%(configuration)s/%(got_revision)s.zip")
278     haltOnFailure = True
279
280     def __init__(self, **kwargs):
281         kwargs['slavesrc'] = self.slavesrc
282         kwargs['masterdest'] = self.masterdest
283         kwargs['mode'] = 0644
284         kwargs['blocksize'] = 1024*256
285         transfer.FileUpload.__init__(self, **kwargs)
286
287
288 class DownloadBuiltProduct(shell.ShellCommand):
289     command = ["python", "./Tools/BuildSlaveSupport/download-built-product",
290         WithProperties("--platform=%(platform)s"), WithProperties("--%(configuration)s"),
291         WithProperties(c["buildbotURL"] + "archives/%(fullPlatform)s-%(architecture)s-%(configuration)s/%(got_revision)s.zip")]
292     name = "download-built-product"
293     description = ["downloading built product"]
294     descriptionDone = ["downloaded built product"]
295     haltOnFailure = True
296     flunkOnFailure = True
297
298
299 class RunJavaScriptCoreTests(TestWithFailureCount):
300     name = "jscore-test"
301     description = ["jscore-tests running"]
302     descriptionDone = ["jscore-tests"]
303     jsonFileName = "jsc_results.json"
304     command = ["perl", "./Tools/Scripts/run-javascriptcore-tests", "--no-build", "--no-fail-fast", "--json-output={0}".format(jsonFileName), WithProperties("--%(configuration)s")]
305     failedTestsFormatString = "%d JSC test%s failed"
306     logfiles = {"json": jsonFileName}
307
308     def start(self):
309         appendCustomBuildFlags(self, self.getProperty('platform'), self.getProperty('fullPlatform'))
310         return shell.Test.start(self)
311
312     def countFailures(self, cmd):
313         logText = cmd.logs['stdio'].getText()
314
315         match = re.search(r'^Results for JSC stress tests:\r?\n\s+(\d+) failure', logText, re.MULTILINE)
316         if match:
317             return int(match.group(1))
318
319         match = re.search(r'^Results for Mozilla tests:\r?\n\s+(\d+) regression', logText, re.MULTILINE)
320         if match:
321             return int(match.group(1))
322
323         return 0
324
325
326 class RunRemoteJavaScriptCoreTests(RunJavaScriptCoreTests):
327     def start(self):
328         self.setCommand(self.command + ["--remote-config-file", "../../remote-jsc-tests-config.json"])
329         return RunJavaScriptCoreTests.start(self)
330
331
332 class RunWebKitTests(shell.Test):
333     name = "layout-test"
334     description = ["layout-tests running"]
335     descriptionDone = ["layout-tests"]
336     command = ["python", "./Tools/Scripts/run-webkit-tests",
337                "--no-build",
338                "--no-show-results",
339                "--no-new-test-results",
340                "--no-sample-on-timeout",
341                "--results-directory", "layout-test-results",
342                "--builder-name", WithProperties("%(buildername)s"),
343                "--build-number", WithProperties("%(buildnumber)s"),
344                "--master-name", "webkit.org",
345                "--test-results-server", "webkit-test-results.webkit.org",
346                "--exit-after-n-crashes-or-timeouts", "50",
347                "--exit-after-n-failures", "500",
348                WithProperties("--%(configuration)s")]
349
350     def start(self):
351         platform = self.getProperty('platform')
352         appendCustomBuildFlags(self, platform, self.getProperty('fullPlatform'))
353         additionalArguments = self.getProperty('additionalArguments')
354
355         if platform != "efl":
356             self.setCommand(self.command + ['--debug-rwt-logging'])
357
358         if platform == "win":
359             self.setCommand(self.command + ['--batch-size', '100', '--root=' + os.path.join("WebKitBuild", self.getProperty('configuration'), "bin32")])
360
361         if additionalArguments:
362             self.setCommand(self.command + additionalArguments)
363         return shell.Test.start(self)
364
365     # FIXME: This will break if run-webkit-tests changes its default log formatter.
366     nrwt_log_message_regexp = re.compile(r'\d{2}:\d{2}:\d{2}(\.\d+)?\s+\d+\s+(?P<message>.*)')
367
368     def _strip_python_logging_prefix(self, line):
369         match_object = self.nrwt_log_message_regexp.match(line)
370         if match_object:
371             return match_object.group('message')
372         return line
373
374     def _parseRunWebKitTestsOutput(self, logText):
375         incorrectLayoutLines = []
376         expressions = [
377             ('flakes', re.compile(r'Unexpected flakiness.+\((\d+)\)')),
378             ('new passes', re.compile(r'Expected to .+, but passed:\s+\((\d+)\)')),
379             ('missing results', re.compile(r'Regressions: Unexpected missing results\s+\((\d+)\)')),
380             ('failures', re.compile(r'Regressions: Unexpected.+\((\d+)\)')),
381         ]
382         testFailures = {}
383
384         for line in logText.splitlines():
385             if line.find('Exiting early') >= 0 or line.find('leaks found') >= 0:
386                 incorrectLayoutLines.append(self._strip_python_logging_prefix(line))
387                 continue
388             for name, expression in expressions:
389                 match = expression.search(line)
390
391                 if match:
392                     testFailures[name] = testFailures.get(name, 0) + int(match.group(1))
393                     break
394
395                 # FIXME: Parse file names and put them in results
396
397         for name in testFailures:
398             incorrectLayoutLines.append(str(testFailures[name]) + ' ' + name)
399
400         self.incorrectLayoutLines = incorrectLayoutLines
401
402     def commandComplete(self, cmd):
403         shell.Test.commandComplete(self, cmd)
404
405         logText = cmd.logs['stdio'].getText()
406         self._parseRunWebKitTestsOutput(logText)
407
408     def evaluateCommand(self, cmd):
409         result = SUCCESS
410
411         if self.incorrectLayoutLines:
412             if len(self.incorrectLayoutLines) == 1:
413                 line = self.incorrectLayoutLines[0]
414                 if line.find('were new') >= 0 or line.find('was new') >= 0 or line.find(' leak') >= 0:
415                     return WARNINGS
416
417             for line in self.incorrectLayoutLines:
418                 if line.find('flakes') >= 0 or line.find('new passes') >= 0 or line.find('missing results') >= 0:
419                     result = WARNINGS
420                 else:
421                     return FAILURE
422
423         if cmd.rc != 0:
424             return FAILURE
425
426         return result
427
428     def getText(self, cmd, results):
429         return self.getText2(cmd, results)
430
431     def getText2(self, cmd, results):
432         if results != SUCCESS and self.incorrectLayoutLines:
433             return self.incorrectLayoutLines
434
435         return [self.name]
436
437
438 class RunUnitTests(TestWithFailureCount):
439     name = "run-api-tests"
440     description = ["unit tests running"]
441     descriptionDone = ["unit-tests"]
442     command = ["perl", "./Tools/Scripts/run-api-tests", "--no-build", WithProperties("--%(configuration)s"), "--verbose"]
443     failedTestsFormatString = "%d unit test%s failed or timed out"
444
445     def start(self):
446         appendCustomBuildFlags(self, self.getProperty('platform'), self.getProperty('fullPlatform'))
447         return shell.Test.start(self)
448
449     def countFailures(self, cmd):
450         log_text = cmd.logs['stdio'].getText()
451         count = 0
452
453         split = re.split(r'\sTests that timed out:\s', log_text)
454         if len(split) > 1:
455             count += len(re.findall(r'^\s+\S+$', split[1], flags=re.MULTILINE))
456
457         split = re.split(r'\sTests that failed:\s', split[0])
458         if len(split) > 1:
459             count += len(re.findall(r'^\s+\S+$', split[1], flags=re.MULTILINE))
460
461         return count
462
463
464 class RunPythonTests(TestWithFailureCount):
465     name = "webkitpy-test"
466     description = ["python-tests running"]
467     descriptionDone = ["python-tests"]
468     command = ["python", "./Tools/Scripts/test-webkitpy", "--verbose"]
469     failedTestsFormatString = "%d python test%s failed"
470
471     def start(self):
472         platform = self.getProperty('platform')
473         # Python tests are flaky on the GTK builders, running them serially
474         # helps and does not significantly prolong the cycle time.
475         if platform == 'gtk':
476             self.setCommand(self.command + ['--child-processes', '1'])
477         # Python tests fail on windows bots when running more than one child process
478         # https://bugs.webkit.org/show_bug.cgi?id=97465
479         if platform == 'win':
480             self.setCommand(self.command + ['--child-processes', '1'])
481         return shell.Test.start(self)
482
483     def countFailures(self, cmd):
484         logText = cmd.logs['stdio'].getText()
485         # We're looking for the line that looks like this: FAILED (failures=2, errors=1)
486         regex = re.compile(r'^FAILED \((?P<counts>[^)]+)\)')
487         for line in logText.splitlines():
488             match = regex.match(line)
489             if not match:
490                 continue
491             return sum(int(component.split('=')[1]) for component in match.group('counts').split(', '))
492         return 0
493
494
495 class RunPerlTests(TestWithFailureCount):
496     name = "webkitperl-test"
497     description = ["perl-tests running"]
498     descriptionDone = ["perl-tests"]
499     command = ["perl", "./Tools/Scripts/test-webkitperl"]
500     failedTestsFormatString = "%d perl test%s failed"
501
502     def countFailures(self, cmd):
503         logText = cmd.logs['stdio'].getText()
504         # We're looking for the line that looks like this: Failed 2/19 test programs. 5/363 subtests failed.
505         regex = re.compile(r'^Failed \d+/\d+ test programs\. (?P<count>\d+)/\d+ subtests failed\.')
506         for line in logText.splitlines():
507             match = regex.match(line)
508             if not match:
509                 continue
510             return int(match.group('count'))
511         return 0
512
513
514 class RunLLINTCLoopTests(TestWithFailureCount):
515     name = "webkit-jsc-cloop-test"
516     description = ["cloop-tests running"]
517     descriptionDone = ["cloop-tests"]
518     jsonFileName = "jsc_cloop.json"
519     command = ["perl", "./Tools/Scripts/run-javascriptcore-tests", "--cloop", "--no-build", "--no-jsc-stress", "--no-fail-fast", "--json-output={0}".format(jsonFileName), WithProperties("--%(configuration)s")]
520     failedTestsFormatString = "%d regression%s found."
521     logfiles = {"json": jsonFileName}
522
523     def countFailures(self, cmd):
524         logText = cmd.logs['stdio'].getText()
525         # We're looking for the line that looks like this: 0 regressions found.
526         regex = re.compile(r'\s*(?P<count>\d+) regressions? found.')
527         for line in logText.splitlines():
528             match = regex.match(line)
529             if not match:
530                 continue
531             return int(match.group('count'))
532         return 0
533
534
535 class Run32bitJSCTests(TestWithFailureCount):
536     name = "webkit-32bit-jsc-test"
537     description = ["32bit-jsc-tests running"]
538     descriptionDone = ["32bit-jsc-tests"]
539     jsonFileName = "jsc_32bit.json"
540     command = ["perl", "./Tools/Scripts/run-javascriptcore-tests", "--32-bit", "--no-build", "--no-fail-fast", "--json-output={0}".format(jsonFileName), WithProperties("--%(configuration)s")]
541     failedTestsFormatString = "%d regression%s found."
542     logfiles = {"json": jsonFileName}
543
544     def countFailures(self, cmd):
545         logText = cmd.logs['stdio'].getText()
546         # We're looking for the line that looks like this: 0 failures found.
547         regex = re.compile(r'\s*(?P<count>\d+) failures? found.')
548         for line in logText.splitlines():
549             match = regex.match(line)
550             if not match:
551                 continue
552             return int(match.group('count'))
553         return 0
554
555
556 class RunBindingsTests(shell.Test):
557     name = "bindings-generation-tests"
558     description = ["bindings-tests running"]
559     descriptionDone = ["bindings-tests"]
560     command = ["python", "./Tools/Scripts/run-bindings-tests"]
561
562
563 class RunEflAPITests(shell.Test):
564     name = "API tests"
565     description = ["API tests running"]
566     descriptionDone = ["API tests"]
567     command = ["perl", "./Tools/Scripts/run-efl-tests", WithProperties("--%(configuration)s")]
568
569
570 class RunGtkAPITests(shell.Test):
571     name = "API tests"
572     description = ["API tests running"]
573     descriptionDone = ["API tests"]
574     command = ["python", "./Tools/Scripts/run-gtk-tests", "--verbose", WithProperties("--%(configuration)s")]
575
576     def commandComplete(self, cmd):
577         shell.Test.commandComplete(self, cmd)
578
579         logText = cmd.logs['stdio'].getText()
580
581         self.incorrectTests = 0
582         self.crashedTests = 0
583         self.timedOutTests = 0
584         self.skippedTests = 0
585         self.statusLine = []
586
587         foundItems = re.findall("Tests failed \((\d+)\):", logText)
588         if (foundItems):
589             self.incorrectTests = int(foundItems[0])
590
591         foundItems = re.findall("Tests that crashed \((\d+)\):", logText)
592         if (foundItems):
593             self.crashedTests = int(foundItems[0])
594
595         foundItems = re.findall("Tests that timed out \((\d+)\):", logText)
596         if (foundItems):
597             self.timedOutTests = int(foundItems[0])
598
599         foundItems = re.findall("Tests skipped \((\d+)\):", logText)
600         if (foundItems):
601             self.skippedTests = int(foundItems[0])
602
603         self.totalFailedTests = self.incorrectTests + self.crashedTests + self.timedOutTests
604
605         if self.totalFailedTests > 0:
606             self.statusLine = [
607                 "%d API tests failed, %d crashed, %d timed out, %d skipped" %
608                 (self.incorrectTests, self.crashedTests, self.timedOutTests, self.skippedTests)
609             ]
610
611     def evaluateCommand(self, cmd):
612         if self.totalFailedTests > 0:
613             return FAILURE
614
615         if cmd.rc != 0:
616             return FAILURE
617
618         return SUCCESS
619
620     def getText(self, cmd, results):
621         return self.getText2(cmd, results)
622
623     def getText2(self, cmd, results):
624         if results != SUCCESS and self.totalFailedTests > 0:
625             return self.statusLine
626
627         return [self.name]
628
629 class RunGtkWebKitGObjectDOMBindingsAPIBreakTests(shell.Test):
630     name = "WebKit GObject DOM bindings API break tests"
631     description = ["WebKit GObject DOM bindings API break tests running"]
632     descriptionDone = ["WebKit GObject DOM bindings API break tests"]
633     command = ["./Tools/gtk/check-for-webkitdom-api-breaks"]
634
635     def commandComplete(self, cmd):
636         shell.Test.commandComplete(self, cmd)
637
638         logText = cmd.logs['stdio'].getText()
639
640         self.missingAPI = bool(re.findall("Missing API", logText))
641         self.newAPI = bool(re.findall("New API", logText))
642
643         line = ""
644         if self.newAPI:
645             line += "Found new API."
646         if self.missingAPI:
647             line += "Found missing API."
648         if line:
649             self.statusLine = [ line ]
650
651     def evaluateCommand(self, cmd):
652         if self.missingAPI:
653             return FAILURE
654         if self.newAPI:
655             return WARNINGS
656         if cmd.rc != 0:
657             return FAILURE
658         return SUCCESS
659
660 class RunWebKit1Tests(RunWebKitTests):
661     def start(self):
662         self.setCommand(self.command + ["--dump-render-tree"])
663
664         return RunWebKitTests.start(self)
665
666 class RunWebKit1LeakTests(RunWebKit1Tests):
667     want_stdout = False
668     want_stderr = False
669     warnOnWarnings = True
670     def start(self):
671         self.setCommand(self.command + ["--leaks"])
672         return RunWebKit1Tests.start(self)
673
674 class RunAndUploadPerfTests(shell.Test):
675     name = "perf-test"
676     description = ["perf-tests running"]
677     descriptionDone = ["perf-tests"]
678     command = ["python", "./Tools/Scripts/run-perf-tests",
679                "--output-json-path", "perf-test-results.json",
680                "--slave-config-json-path", "../../perf-test-config.json",
681                "--no-show-results",
682                "--reset-results",
683                "--test-results-server", "perf.webkit.org",
684                "--builder-name", WithProperties("%(buildername)s"),
685                "--build-number", WithProperties("%(buildnumber)s"),
686                "--platform", WithProperties("%(fullPlatform)s"),
687                "--no-build",
688                WithProperties("--%(configuration)s")]
689
690     def start(self):
691         self.setCommand(self.command)
692         return shell.Test.start(self)
693
694     def getText(self, cmd, results):
695         return self.getText2(cmd, results)
696
697     def getText2(self, cmd, results):
698         if results != SUCCESS:
699             if cmd.rc == -1 & 0xff:
700                 return ["build not up to date"]
701             elif cmd.rc == -2 & 0xff:
702                 return ["slave config JSON error"]
703             elif cmd.rc == -3 & 0xff:
704                 return ["output JSON merge error"]
705             elif cmd.rc == -4 & 0xff:
706                 return ["upload error"]
707             elif cmd.rc == -5 & 0xff:
708                 return ["system dependency error"]
709             elif cmd.rc == -1:
710                 return ["timeout"]
711             else:
712                 return ["%d perf tests failed" % cmd.rc]
713
714         return [self.name]
715
716 class RunBenchmarkTests(shell.Test):
717     name = "benchmark-test"
718     description = ["benchmark tests running"]
719     descriptionDone = ["benchmark tests"]
720     # Buildbot default timeout without output for a step is 1200.
721     # The current maximum timeout for a benchmark plan is also 1200.
722     # So raise the buildbot timeout to avoid aborting this whole step when a test timeouts.
723     timeout = 1500
724     command = ["python", "./Tools/Scripts/run-benchmark", "--allplans"]
725
726     def start(self):
727         platform = self.getProperty("platform")
728         if platform == "gtk":
729             self.command += ["--platform", "gtk", "--browser", "minibrowser"]
730         self.setCommand(self.command)
731         return shell.Test.start(self)
732
733     def getText(self, cmd, results):
734         return self.getText2(cmd, results)
735
736     def getText2(self, cmd, results):
737         if results != SUCCESS:
738             return ["%d benchmark tests failed" % cmd.rc]
739         return [self.name]
740
741 class ArchiveTestResults(shell.ShellCommand):
742     command = ["python", "./Tools/BuildSlaveSupport/test-result-archive",
743                WithProperties("--platform=%(platform)s"), WithProperties("--%(configuration)s"), "archive"]
744     name = "archive-test-results"
745     description = ["archiving test results"]
746     descriptionDone = ["archived test results"]
747     haltOnFailure = True
748
749
750 class UploadTestResults(transfer.FileUpload):
751     slavesrc = "layout-test-results.zip"
752     masterdest = WithProperties("public_html/results/%(buildername)s/r%(got_revision)s (%(buildnumber)s).zip")
753
754     def __init__(self, **kwargs):
755         kwargs['slavesrc'] = self.slavesrc
756         kwargs['masterdest'] = self.masterdest
757         kwargs['mode'] = 0644
758         transfer.FileUpload.__init__(self, **kwargs)
759
760
761 class ExtractTestResults(master.MasterShellCommand):
762     zipFile = WithProperties("public_html/results/%(buildername)s/r%(got_revision)s (%(buildnumber)s).zip")
763     resultDirectory = WithProperties("public_html/results/%(buildername)s/r%(got_revision)s (%(buildnumber)s)")
764     descriptionDone = ["uploaded results"]
765
766     def __init__(self, **kwargs):
767         kwargs['command'] = ""
768         master.MasterShellCommand.__init__(self, **kwargs)
769
770     def resultDirectoryURL(self):
771         return self.build.getProperties().render(self.resultDirectory).replace("public_html/", "/") + "/"
772
773     def start(self):
774         self.command = ["unzip", self.build.getProperties().render(self.zipFile), "-d", self.build.getProperties().render(self.resultDirectory)]
775         return master.MasterShellCommand.start(self)
776
777     def addCustomURLs(self):
778         url = self.resultDirectoryURL() + "results.html"
779         self.addURL("view results", url)
780
781     def finished(self, result):
782         self.addCustomURLs()
783         return master.MasterShellCommand.finished(self, result)
784
785
786 class ExtractTestResultsAndLeaks(ExtractTestResults):
787     def addCustomURLs(self):
788         ExtractTestResults.addCustomURLs(self)
789         url = "/LeaksViewer/?url=" + urllib.quote(self.resultDirectoryURL(), safe="")
790         self.addURL("view leaks", url)
791
792
793 class Factory(factory.BuildFactory):
794     def __init__(self, platform, configuration, architectures, buildOnly, additionalArguments, SVNMirror):
795         factory.BuildFactory.__init__(self)
796         self.addStep(ConfigureBuild(platform=platform, configuration=configuration, architecture=" ".join(architectures), buildOnly=buildOnly, additionalArguments=additionalArguments, SVNMirror=SVNMirror))
797         if SVNMirror:
798             self.addStep(WaitForSVNServer())
799         self.addStep(CheckOutSource(SVNMirror=SVNMirror))
800         if not (platform == "jsc-only"):
801             self.addStep(KillOldProcesses())
802         self.addStep(CleanBuildIfScheduled())
803         self.addStep(DeleteStaleBuildFiles())
804         if platform == "win":
805             self.addStep(InstallWin32Dependencies())
806         if platform == "gtk":
807             self.addStep(InstallGtkDependencies())
808         if platform == "efl":
809             self.addStep(InstallEflDependencies())
810
811
812 class BuildFactory(Factory):
813     def __init__(self, platform, configuration, architectures, triggers=None, additionalArguments=None, SVNMirror=None):
814         Factory.__init__(self, platform, configuration, architectures, True, additionalArguments, SVNMirror)
815
816         if platform == "win":
817             self.addStep(CompileWebKit(timeout=2*60*60))
818         else:
819             self.addStep(CompileWebKit())
820
821         if triggers:
822             self.addStep(ArchiveBuiltProduct())
823             self.addStep(UploadBuiltProduct())
824             self.addStep(trigger.Trigger(schedulerNames=triggers))
825
826 def pickLatestBuild(builder, requests):
827     return max(requests, key=operator.attrgetter("submittedAt"))
828
829 class TestFactory(Factory):
830     JSCTestClass = RunJavaScriptCoreTests
831     LayoutTestClass = RunWebKitTests
832
833     def getProduct(self):
834         self.addStep(DownloadBuiltProduct())
835         self.addStep(ExtractBuiltProduct())
836
837     def __init__(self, platform, configuration, architectures, additionalArguments=None, SVNMirror=None, **kwargs):
838         Factory.__init__(self, platform, configuration, architectures, False, additionalArguments, SVNMirror, **kwargs)
839         self.getProduct()
840         if self.JSCTestClass:
841             self.addStep(self.JSCTestClass())
842         if self.LayoutTestClass:
843             self.addStep(self.LayoutTestClass())
844
845         if platform == 'win' or platform.startswith('mac') or platform.startswith('ios-simulator'):
846             self.addStep(RunUnitTests())
847         self.addStep(RunPythonTests())
848         self.addStep(RunPerlTests())
849         self.addStep(RunBindingsTests())
850         if self.LayoutTestClass:
851             self.addStep(ArchiveTestResults())
852             self.addStep(UploadTestResults())
853             self.addStep(ExtractTestResults())
854         if platform == "efl":
855             self.addStep(RunEflAPITests())
856         if platform == "gtk":
857             self.addStep(RunGtkAPITests())
858             self.addStep(RunGtkWebKitGObjectDOMBindingsAPIBreakTests())
859
860 class BuildAndTestFactory(TestFactory):
861     def getProduct(self):
862         self.addStep(CompileWebKit())
863
864     def __init__(self, platform, configuration, architectures, triggers=None, additionalArguments=None, SVNMirror=None, **kwargs):
865         TestFactory.__init__(self, platform, configuration, architectures, additionalArguments, SVNMirror, **kwargs)
866         if triggers:
867             self.addStep(ArchiveBuiltProduct())
868             self.addStep(UploadBuiltProduct())
869             self.addStep(trigger.Trigger(schedulerNames=triggers))
870
871 class BuildAndTestLLINTCLoopFactory(Factory):
872     def __init__(self, platform, configuration, architectures, triggers=None, additionalArguments=None, SVNMirror=None, **kwargs):
873         Factory.__init__(self, platform, configuration, architectures, False, additionalArguments, SVNMirror, **kwargs)
874         self.addStep(CompileLLINTCLoop())
875         self.addStep(RunLLINTCLoopTests())
876
877 class BuildAndTest32bitJSCFactory(Factory):
878     def __init__(self, platform, configuration, architectures, triggers=None, additionalArguments=None, SVNMirror=None, **kwargs):
879         Factory.__init__(self, platform, configuration, architectures, False, additionalArguments, SVNMirror, **kwargs)
880         self.addStep(Compile32bitJSC())
881         self.addStep(Run32bitJSCTests())
882
883 class BuildAndNonLayoutTestFactory(BuildAndTestFactory):
884     LayoutTestClass = None
885
886 class BuildAndRemoteJSCTestsFactory(Factory):
887     def __init__(self, platform, configuration, architectures, triggers=None, additionalArguments=None, SVNMirror=None):
888         Factory.__init__(self, platform, configuration, architectures, False, additionalArguments, SVNMirror)
889         self.addStep(CompileJSCOnly())
890         self.addStep(RunRemoteJavaScriptCoreTests())
891
892 class TestWebKit1LeaksFactory(Factory):
893     def __init__(self, platform, configuration, architectures, additionalArguments=None, SVNMirror=None):
894         Factory.__init__(self, platform, configuration, architectures, False, additionalArguments, SVNMirror)
895         self.addStep(DownloadBuiltProduct())
896         self.addStep(ExtractBuiltProduct())
897         self.addStep(RunWebKit1LeakTests())
898         self.addStep(ArchiveTestResults())
899         self.addStep(UploadTestResults())
900         self.addStep(ExtractTestResultsAndLeaks())
901
902 class TestAllButJSCFactory(TestFactory):
903     JSCTestClass = None
904
905 class TestJSCFactory(Factory):
906     def __init__(self, platform, configuration, architectures, additionalArguments=None, SVNMirror=None):
907         Factory.__init__(self, platform, configuration, architectures, False, additionalArguments, SVNMirror)
908         self.addStep(DownloadBuiltProduct())
909         self.addStep(ExtractBuiltProduct())
910         self.addStep(RunJavaScriptCoreTests())
911
912 class TestWebKit1Factory(TestFactory):
913     LayoutTestClass = RunWebKit1Tests
914
915 class TestWebKit1AllButJSCFactory(TestWebKit1Factory):
916     JSCTestClass = None
917
918 class BuildAndPerfTestFactory(Factory):
919     def __init__(self, platform, configuration, architectures, additionalArguments=None, SVNMirror=None, **kwargs):
920         Factory.__init__(self, platform, configuration, architectures, False, additionalArguments, SVNMirror, **kwargs)
921         self.addStep(CompileWebKit())
922         self.addStep(RunAndUploadPerfTests())
923         if platform == "gtk":
924             self.addStep(RunBenchmarkTests())
925
926 class DownloadAndPerfTestFactory(Factory):
927     def __init__(self, platform, configuration, architectures, additionalArguments=None, SVNMirror=None, **kwargs):
928         Factory.__init__(self, platform, configuration, architectures, False, additionalArguments, SVNMirror, **kwargs)
929         self.addStep(DownloadBuiltProduct())
930         self.addStep(ExtractBuiltProduct())
931         self.addStep(RunAndUploadPerfTests())
932         if platform == "gtk":
933             self.addStep(RunBenchmarkTests())
934
935 class PlatformSpecificScheduler(AnyBranchScheduler):
936     def __init__(self, platform, branch, **kwargs):
937         self.platform = platform
938         filter = ChangeFilter(branch=[branch, None], filter_fn=self.filter)
939         AnyBranchScheduler.__init__(self, name=platform, change_filter=filter, **kwargs)
940
941     def filter(self, change):
942         return wkbuild.should_build(self.platform, change.files)
943
944 trunk_filter = ChangeFilter(branch=["trunk", None])
945
946 def loadBuilderConfig(c):
947     # FIXME: These file handles are leaked.
948     passwords = json.load(open('passwords.json'))
949     config = json.load(open('config.json'))
950
951     c['slaves'] = [BuildSlave(slave['name'], passwords[slave['name']], max_builds=1) for slave in config['slaves']]
952
953     c['schedulers'] = []
954     for scheduler in config['schedulers']:
955         if "change_filter" in scheduler:
956             scheduler["change_filter"] = globals()[scheduler["change_filter"]]
957         kls = globals()[scheduler.pop('type')]
958         # Python 2.6 can't handle unicode keys as keyword arguments:
959         # http://bugs.python.org/issue2646.  Modern versions of json return
960         # unicode strings from json.load, so we map all keys to str objects.
961         scheduler = dict(map(lambda key_value_pair: (str(key_value_pair[0]), key_value_pair[1]), scheduler.items()))
962
963         c['schedulers'].append(kls(**scheduler))
964
965     forceScheduler = ForceScheduler(
966         name="force",
967         builderNames=[str(builder['name']) for builder in config['builders']],
968         reason=StringParameter(name="reason", default="", size=40),
969
970         # Validate SVN revision: number or empty string
971         revision=StringParameter(name="revision", default="", regex=re.compile(r'^(\d*)$')),
972
973         # Disable default enabled input fields: branch, repository, project, additional properties
974         branch=FixedParameter(name="branch"),
975         repository=FixedParameter(name="repository"),
976         project=FixedParameter(name="project"),
977         properties=[BooleanParameter(name="is_clean", label="Force Clean build")]
978     )
979     c['schedulers'].append(forceScheduler)
980
981     c['builders'] = []
982     for builder in config['builders']:
983         for slaveName in builder['slavenames']:
984             for slave in config['slaves']:
985                 if slave['name'] != slaveName or slave['platform'] == '*':
986                     continue
987
988                 if slave['platform'] != builder['platform']:
989                     raise Exception, "Builder %r is for platform %r but has slave %r for platform %r!" % (builder['name'], builder['platform'], slave['name'], slave['platform'])
990
991                 break
992
993         platform = builder['platform']
994
995         builderType = builder.pop('type')
996         factory = globals()["%sFactory" % builderType]
997         factorykwargs = {}
998         for key in "platform", "configuration", "architectures", "triggers", "additionalArguments", "SVNMirror":
999             value = builder.pop(key, None)
1000             if value:
1001                 factorykwargs[key] = value
1002
1003         builder["factory"] = factory(**factorykwargs)
1004
1005         if platform.startswith('mac'):
1006             builder["category"] = 'AppleMac'
1007         elif platform.startswith('ios'):
1008             builder['category'] = 'iOS'
1009         elif platform == 'win':
1010             builder["category"] = 'AppleWin'
1011         elif platform.startswith('gtk'):
1012             builder["category"] = 'GTK'
1013         elif platform.startswith('efl'):
1014             builder["category"] = "EFL"
1015         else:
1016             builder["category"] = 'misc'
1017
1018         if (builder['category'] in ('AppleMac', 'AppleWin', 'iOS')) and builderType != 'Build':
1019             builder['nextBuild'] = pickLatestBuild
1020
1021         c['builders'].append(builder)
1022
1023 loadBuilderConfig(c)