Support running unit tests on iOS Simulator test bot
[WebKit-https.git] / Tools / BuildSlaveSupport / build.webkit.org-config / master.cfg
1 # -*- python -*-
2 # ex: set syntax=python:
3
4 from buildbot.buildslave import BuildSlave
5 from buildbot.changes.pb import PBChangeSource
6 from buildbot.scheduler import AnyBranchScheduler, Triggerable
7 from buildbot.schedulers.forcesched import FixedParameter, ForceScheduler, StringParameter, BooleanParameter
8 from buildbot.schedulers.filter import ChangeFilter
9 from buildbot.status import html
10 from buildbot.status.web.authz import Authz
11 from buildbot.process import buildstep, factory, properties
12 from buildbot.steps import master, shell, source, transfer, trigger
13 from buildbot.status.builder import SUCCESS, FAILURE, WARNINGS, SKIPPED
14
15 from twisted.internet import defer
16
17 import os
18 import re
19 import json
20 import operator
21 import cStringIO
22 import urllib
23
24 from committer_auth import CommitterAuth
25 import wkbuild
26
27
28 c = BuildmasterConfig = {}
29
30 c['change_source'] = PBChangeSource()
31
32 # permissions for WebStatus
33 authz = Authz(
34     auth=CommitterAuth('auth.json'),
35     forceBuild='auth',
36     forceAllBuilds='auth',
37     pingBuilder=True,
38     gracefulShutdown=False,
39     stopBuild='auth',
40     stopAllBuilds='auth',
41     cancelPendingBuild='auth',
42     stopChange=True,
43     cleanShutdown=False)
44
45 c['status'] = []
46 c['status'].append(html.WebStatus(http_port=8710,
47                                   revlink="https://trac.webkit.org/changeset/%s", 
48                                   changecommentlink=(r"(https://bugs\.webkit\.org/show_bug\.cgi\?id=|webkit\.org/b/)(\d+)", r"https://bugs.webkit.org/show_bug.cgi?id=\2"),
49                                   authz=authz))
50
51 c['slavePortnum'] = 17000
52 c['projectName'] = "WebKit"
53 c['projectURL'] = "https://webkit.org"
54 c['buildbotURL'] = "https://build.webkit.org/"
55
56 c['buildHorizon'] = 1000
57 c['logHorizon'] = 500
58 c['eventHorizon'] = 200
59 c['buildCacheSize'] = 60
60
61 WithProperties = properties.WithProperties
62
63
64 class TestWithFailureCount(shell.Test):
65     failedTestsFormatString = "%d test%s failed"
66
67     def countFailures(self, cmd):
68         return 0
69
70     def commandComplete(self, cmd):
71         shell.Test.commandComplete(self, cmd)
72         self.failedTestCount = self.countFailures(cmd)
73         self.failedTestPluralSuffix = "" if self.failedTestCount == 1 else "s"
74
75     def evaluateCommand(self, cmd):
76         if self.failedTestCount:
77             return FAILURE
78
79         if cmd.rc != 0:
80             return FAILURE
81
82         return SUCCESS
83
84     def getText(self, cmd, results):
85         return self.getText2(cmd, results)
86
87     def getText2(self, cmd, results):
88         if results != SUCCESS and self.failedTestCount:
89             return [self.failedTestsFormatString % (self.failedTestCount, self.failedTestPluralSuffix)]
90
91         return [self.name]
92
93
94 class ConfigureBuild(buildstep.BuildStep):
95     name = "configure build"
96     description = ["configuring build"]
97     descriptionDone = ["configured build"]
98     def __init__(self, platform, configuration, architecture, buildOnly, additionalArguments, SVNMirror, *args, **kwargs):
99         buildstep.BuildStep.__init__(self, *args, **kwargs)
100         self.platform = platform.split('-', 1)[0]
101         self.fullPlatform = platform
102         self.configuration = configuration
103         self.architecture = architecture
104         self.buildOnly = buildOnly
105         self.additionalArguments = additionalArguments
106         self.SVNMirror = SVNMirror
107         self.addFactoryArguments(platform=platform, configuration=configuration, architecture=architecture, buildOnly=buildOnly, additionalArguments=additionalArguments, SVNMirror=SVNMirror)
108
109     def start(self):
110         self.setProperty("platform", self.platform)
111         self.setProperty("fullPlatform", self.fullPlatform)
112         self.setProperty("configuration", self.configuration)
113         self.setProperty("architecture", self.architecture)
114         self.setProperty("buildOnly", self.buildOnly)
115         self.setProperty("additionalArguments", self.additionalArguments)
116         self.setProperty("SVNMirror", self.SVNMirror)
117         self.finished(SUCCESS)
118         return defer.succeed(None)
119
120
121 class CheckOutSource(source.SVN):
122     mode = "update"
123     def __init__(self, SVNMirror, **kwargs):
124         kwargs['baseURL'] = SVNMirror or "http://svn.webkit.org/repository/webkit/"
125         kwargs['defaultBranch'] = "trunk"
126         kwargs['mode'] = self.mode
127         source.SVN.__init__(self, **kwargs)
128         self.addFactoryArguments(SVNMirror=SVNMirror)
129
130 class WaitForSVNServer(shell.ShellCommand):
131     name = "wait-for-svn-server"
132     command = ["python", "./Tools/BuildSlaveSupport/wait-for-SVN-server.py", "-r", WithProperties("%(revision)s"), "-s", WithProperties("%(SVNMirror)s")]
133     description = ["waiting for SVN server"]
134     descriptionDone = ["SVN server is ready"]
135     warnOnFailure = True
136
137     def evaluateCommand(self, cmd):
138         if cmd.rc != 0:
139             return WARNINGS
140         return SUCCESS
141
142 class InstallWin32Dependencies(shell.Compile):
143     description = ["installing dependencies"]
144     descriptionDone = ["installed dependencies"]
145     command = ["perl", "./Tools/Scripts/update-webkit-auxiliary-libs"]
146
147 class KillOldProcesses(shell.Compile):
148     name = "kill old processes"
149     description = ["killing old processes"]
150     descriptionDone = ["killed old processes"]
151     command = ["python", "./Tools/BuildSlaveSupport/kill-old-processes"]
152
153 class CleanBuildIfScheduled(shell.Compile):
154     name = "delete WebKitBuild directory"
155     description = ["deleting WebKitBuild directory"]
156     descriptionDone = ["deleted WebKitBuild directory"]
157     command = ["python", "./Tools/BuildSlaveSupport/clean-build", WithProperties("--platform=%(fullPlatform)s"), WithProperties("--%(configuration)s")]
158
159     def start(self):
160         if not self.getProperty('is_clean'):
161             self.hideStepIf = True
162             return SKIPPED
163         return shell.Compile.start(self)
164
165 class DeleteStaleBuildFiles(shell.Compile):
166     name = "delete stale build files"
167     description = ["deleting stale build files"]
168     descriptionDone = ["deleted stale build files"]
169     command = ["python", "./Tools/BuildSlaveSupport/delete-stale-build-files", WithProperties("--platform=%(fullPlatform)s"), WithProperties("--%(configuration)s")]
170
171     def start(self):
172         if self.getProperty('is_clean'): # Nothing to be done if WebKitBuild had been removed.
173             self.hideStepIf = True
174             return SKIPPED
175         return shell.Compile.start(self)
176
177 class InstallEflDependencies(shell.ShellCommand):
178     name = "jhbuild"
179     description = ["updating efl dependencies"]
180     descriptionDone = ["updated efl dependencies"]
181     command = ["perl", "./Tools/Scripts/update-webkitefl-libs"]
182     haltOnFailure = True
183
184 class InstallGtkDependencies(shell.ShellCommand):
185     name = "jhbuild"
186     description = ["updating gtk dependencies"]
187     descriptionDone = ["updated gtk dependencies"]
188     command = ["perl", "./Tools/Scripts/update-webkitgtk-libs"]
189     haltOnFailure = True
190
191 def appendCustomBuildFlags(step, platform, fullPlatform):
192     if platform not in ('efl', 'gtk', 'wincairo', 'ios'):
193         return
194     if fullPlatform.startswith('ios-simulator'):
195         platform = 'ios-simulator'
196     elif platform == 'ios':
197         platform = 'device'
198     step.setCommand(step.command + ['--' + platform])
199
200 class CompileWebKit(shell.Compile):
201     command = ["perl", "./Tools/Scripts/build-webkit", WithProperties("--%(configuration)s")]
202     env = {'MFLAGS':''}
203     name = "compile-webkit"
204     description = ["compiling"]
205     descriptionDone = ["compiled"]
206     warningPattern = ".*arning: .*"
207
208     def start(self):
209         platform = self.getProperty('platform')
210         buildOnly = self.getProperty('buildOnly')
211         architecture = self.getProperty('architecture')
212         if platform == 'mac' and architecture == 'i386':
213             self.setCommand(self.command + ['--32-bit'])
214         if platform == 'mac' and buildOnly:
215             self.setCommand(self.command + ['DEBUG_INFORMATION_FORMAT=dwarf-with-dsym'])
216
217         appendCustomBuildFlags(self, platform, self.getProperty('fullPlatform'))
218
219         return shell.Compile.start(self)
220
221     def createSummary(self, log):
222         platform = self.getProperty('platform')
223         if platform.startswith('mac'):    
224             warnings = []
225             errors = []
226             sio = cStringIO.StringIO(log.getText())
227             for line in sio.readlines():
228                 if "arning:" in line:
229                     warnings.append(line)
230                 if "rror:" in line:
231                     errors.append(line)
232             if warnings:
233                 self.addCompleteLog('warnings', "".join(warnings))
234             if errors:
235                 self.addCompleteLog('errors', "".join(errors))
236
237
238 class CompileLLINTCLoop(CompileWebKit):
239     command = ["perl", "./Tools/Scripts/build-jsc", "--cloop", WithProperties("--%(configuration)s")]
240
241 class Compile32bitJSC(CompileWebKit):
242     command = ["perl", "./Tools/Scripts/build-jsc", "--32-bit", WithProperties("--%(configuration)s")]
243
244
245 class ArchiveBuiltProduct(shell.ShellCommand):
246     command = ["python", "./Tools/BuildSlaveSupport/built-product-archive",
247                WithProperties("--platform=%(fullPlatform)s"), WithProperties("--%(configuration)s"), "archive"]
248     name = "archive-built-product"
249     description = ["archiving built product"]
250     descriptionDone = ["archived built product"]
251     haltOnFailure = True
252
253
254 class ExtractBuiltProduct(shell.ShellCommand):
255     command = ["python", "./Tools/BuildSlaveSupport/built-product-archive",
256                WithProperties("--platform=%(fullPlatform)s"), WithProperties("--%(configuration)s"), "extract"]
257     name = "extract-built-product"
258     description = ["extracting built product"]
259     descriptionDone = ["extracted built product"]
260     haltOnFailure = True
261
262
263 class UploadBuiltProduct(transfer.FileUpload):
264     slavesrc = WithProperties("WebKitBuild/%(configuration)s.zip")
265     masterdest = WithProperties("archives/%(fullPlatform)s-%(architecture)s-%(configuration)s/%(got_revision)s.zip")
266     haltOnFailure = True
267
268     def __init__(self, **kwargs):
269         kwargs['slavesrc'] = self.slavesrc
270         kwargs['masterdest'] = self.masterdest
271         kwargs['mode'] = 0644
272         kwargs['blocksize'] = 1024*256
273         transfer.FileUpload.__init__(self, **kwargs)
274
275
276 class DownloadBuiltProduct(shell.ShellCommand):
277     command = ["python", "./Tools/BuildSlaveSupport/download-built-product",
278         WithProperties("--platform=%(platform)s"), WithProperties("--%(configuration)s"),
279         WithProperties(c["buildbotURL"] + "archives/%(fullPlatform)s-%(architecture)s-%(configuration)s/%(got_revision)s.zip")]
280     name = "download-built-product"
281     description = ["downloading built product"]
282     descriptionDone = ["downloaded built product"]
283     haltOnFailure = True
284     flunkOnFailure = True
285
286
287 class RunJavaScriptCoreTests(TestWithFailureCount):
288     name = "jscore-test"
289     description = ["jscore-tests running"]
290     descriptionDone = ["jscore-tests"]
291     command = ["perl", "./Tools/Scripts/run-javascriptcore-tests", "--no-build", WithProperties("--%(configuration)s")]
292     failedTestsFormatString = "%d JSC test%s failed"
293
294     def start(self):
295         appendCustomBuildFlags(self, self.getProperty('platform'), self.getProperty('fullPlatform'))
296         return shell.Test.start(self)
297
298     def countFailures(self, cmd):
299         logText = cmd.logs['stdio'].getText()
300
301         match = re.search(r'^Results for JSC stress tests:\r?\n\s+(\d+) failure', logText, re.MULTILINE)
302         if match:
303             return int(match.group(1))
304
305         match = re.search(r'^Results for Mozilla tests:\r?\n\s+(\d+) regression', logText, re.MULTILINE)
306         if match:
307             return int(match.group(1))
308
309         return 0
310
311
312 class RunRemoteJavaScriptCoreTests(RunJavaScriptCoreTests):
313     def start(self):
314         self.setCommand(self.command + ["--remote-config-file", "../../remote-jsc-tests-config.json"])
315         return RunJavaScriptCoreTests.start(self)
316
317
318 class RunWebKitTests(shell.Test):
319     name = "layout-test"
320     description = ["layout-tests running"]
321     descriptionDone = ["layout-tests"]
322     command = ["python", "./Tools/Scripts/run-webkit-tests",
323                "--no-build",
324                "--no-show-results",
325                "--no-new-test-results",
326                "--no-sample-on-timeout",
327                "--results-directory", "layout-test-results",
328                "--builder-name", WithProperties("%(buildername)s"),
329                "--build-number", WithProperties("%(buildnumber)s"),
330                "--master-name", "webkit.org",
331                "--test-results-server", "webkit-test-results.webkit.org",
332                "--exit-after-n-crashes-or-timeouts", "50",
333                "--exit-after-n-failures", "500",
334                WithProperties("--%(configuration)s")]
335
336     def start(self):
337         platform = self.getProperty('platform')
338         appendCustomBuildFlags(self, platform, self.getProperty('fullPlatform'))
339         additionalArguments = self.getProperty('additionalArguments')
340
341         if platform != "efl":
342             self.setCommand(self.command + ['--debug-rwt-logging'])
343
344         if platform == "win":
345             self.setCommand(self.command + ['--batch-size', '100', '--root=' + os.path.join("WebKitBuild", self.getProperty('configuration'), "bin32")])
346
347         if additionalArguments:
348             self.setCommand(self.command + additionalArguments)
349         return shell.Test.start(self)
350
351     # FIXME: This will break if run-webkit-tests changes its default log formatter.
352     nrwt_log_message_regexp = re.compile(r'\d{2}:\d{2}:\d{2}(\.\d+)?\s+\d+\s+(?P<message>.*)')
353
354     def _strip_python_logging_prefix(self, line):
355         match_object = self.nrwt_log_message_regexp.match(line)
356         if match_object:
357             return match_object.group('message')
358         return line
359
360     def _parseRunWebKitTestsOutput(self, logText):
361         incorrectLayoutLines = []
362         expressions = [
363             ('flakes', re.compile(r'Unexpected flakiness.+\((\d+)\)')),
364             ('new passes', re.compile(r'Expected to .+, but passed:\s+\((\d+)\)')),
365             ('missing results', re.compile(r'Regressions: Unexpected missing results\s+\((\d+)\)')),
366             ('failures', re.compile(r'Regressions: Unexpected.+\((\d+)\)')),
367         ]
368         testFailures = {}
369
370         for line in logText.splitlines():
371             if line.find('Exiting early') >= 0 or line.find('leaks found') >= 0:
372                 incorrectLayoutLines.append(self._strip_python_logging_prefix(line))
373                 continue
374             for name, expression in expressions:
375                 match = expression.search(line)
376
377                 if match:
378                     testFailures[name] = testFailures.get(name, 0) + int(match.group(1))
379                     break
380
381                 # FIXME: Parse file names and put them in results
382
383         for name in testFailures:
384             incorrectLayoutLines.append(str(testFailures[name]) + ' ' + name)
385
386         self.incorrectLayoutLines = incorrectLayoutLines
387
388     def commandComplete(self, cmd):
389         shell.Test.commandComplete(self, cmd)
390
391         logText = cmd.logs['stdio'].getText()
392         self._parseRunWebKitTestsOutput(logText)
393
394     def evaluateCommand(self, cmd):
395         result = SUCCESS
396
397         if self.incorrectLayoutLines:
398             if len(self.incorrectLayoutLines) == 1:
399                 line = self.incorrectLayoutLines[0]
400                 if line.find('were new') >= 0 or line.find('was new') >= 0 or line.find(' leak') >= 0:
401                     return WARNINGS
402
403             for line in self.incorrectLayoutLines:
404                 if line.find('flakes') >= 0 or line.find('new passes') >= 0 or line.find('missing results') >= 0:
405                     result = WARNINGS
406                 else:
407                     return FAILURE
408
409         if cmd.rc != 0:
410             return FAILURE
411
412         return result
413
414     def getText(self, cmd, results):
415         return self.getText2(cmd, results)
416
417     def getText2(self, cmd, results):
418         if results != SUCCESS and self.incorrectLayoutLines:
419             return self.incorrectLayoutLines
420
421         return [self.name]
422
423
424 class RunUnitTests(TestWithFailureCount):
425     name = "run-api-tests"
426     description = ["unit tests running"]
427     descriptionDone = ["unit-tests"]
428     command = ["perl", "./Tools/Scripts/run-api-tests", "--no-build", WithProperties("--%(configuration)s"), "--verbose"]
429     failedTestsFormatString = "%d unit test%s failed or timed out"
430
431     def start(self):
432         appendCustomBuildFlags(self, self.getProperty('platform'), self.getProperty('fullPlatform'))
433         return shell.Test.start(self)
434
435     def countFailures(self, cmd):
436         log_text = cmd.logs['stdio'].getText()
437         count = 0
438
439         split = re.split(r'\sTests that timed out:\s', log_text)
440         if len(split) > 1:
441             count += len(re.findall(r'^\s+\S+$', split[1], flags=re.MULTILINE))
442
443         split = re.split(r'\sTests that failed:\s', split[0])
444         if len(split) > 1:
445             count += len(re.findall(r'^\s+\S+$', split[1], flags=re.MULTILINE))
446
447         return count
448
449
450 class RunPythonTests(TestWithFailureCount):
451     name = "webkitpy-test"
452     description = ["python-tests running"]
453     descriptionDone = ["python-tests"]
454     command = ["python", "./Tools/Scripts/test-webkitpy", "--verbose"]
455     failedTestsFormatString = "%d python test%s failed"
456
457     def start(self):
458         platform = self.getProperty('platform')
459         # Python tests are flaky on the GTK builders, running them serially
460         # helps and does not significantly prolong the cycle time.
461         if platform == 'gtk':
462             self.setCommand(self.command + ['--child-processes', '1'])
463         # Python tests fail on windows bots when running more than one child process
464         # https://bugs.webkit.org/show_bug.cgi?id=97465
465         if platform == 'win':
466             self.setCommand(self.command + ['--child-processes', '1'])
467         return shell.Test.start(self)
468
469     def countFailures(self, cmd):
470         logText = cmd.logs['stdio'].getText()
471         # We're looking for the line that looks like this: FAILED (failures=2, errors=1)
472         regex = re.compile(r'^FAILED \((?P<counts>[^)]+)\)')
473         for line in logText.splitlines():
474             match = regex.match(line)
475             if not match:
476                 continue
477             return sum(int(component.split('=')[1]) for component in match.group('counts').split(', '))
478         return 0
479
480
481 class RunPerlTests(TestWithFailureCount):
482     name = "webkitperl-test"
483     description = ["perl-tests running"]
484     descriptionDone = ["perl-tests"]
485     command = ["perl", "./Tools/Scripts/test-webkitperl"]
486     failedTestsFormatString = "%d perl test%s failed"
487
488     def countFailures(self, cmd):
489         logText = cmd.logs['stdio'].getText()
490         # We're looking for the line that looks like this: Failed 2/19 test programs. 5/363 subtests failed.
491         regex = re.compile(r'^Failed \d+/\d+ test programs\. (?P<count>\d+)/\d+ subtests failed\.')
492         for line in logText.splitlines():
493             match = regex.match(line)
494             if not match:
495                 continue
496             return int(match.group('count'))
497         return 0
498
499
500 class RunLLINTCLoopTests(TestWithFailureCount):
501     name = "webkit-jsc-cloop-test"
502     description = ["cloop-tests running"]
503     descriptionDone = ["cloop-tests"]
504     command = ["perl", "./Tools/Scripts/run-javascriptcore-tests", "--cloop", "--no-build", "--no-jsc-stress", WithProperties("--%(configuration)s")]
505     failedTestsFormatString = "%d regression%s found."
506
507     def countFailures(self, cmd):
508         logText = cmd.logs['stdio'].getText()
509         # We're looking for the line that looks like this: 0 regressions found.
510         regex = re.compile(r'\s*(?P<count>\d+) regressions? found.')
511         for line in logText.splitlines():
512             match = regex.match(line)
513             if not match:
514                 continue
515             return int(match.group('count'))
516         return 0
517
518
519 class Run32bitJSCTests(TestWithFailureCount):
520     name = "webkit-32bit-jsc-test"
521     description = ["32bit-jsc-tests running"]
522     descriptionDone = ["32bit-jsc-tests"]
523     command = ["perl", "./Tools/Scripts/run-javascriptcore-tests", "--32-bit", "--no-build", WithProperties("--%(configuration)s")]
524     failedTestsFormatString = "%d regression%s found."
525
526     def countFailures(self, cmd):
527         logText = cmd.logs['stdio'].getText()
528         # We're looking for the line that looks like this: 0 failures found.
529         regex = re.compile(r'\s*(?P<count>\d+) failures? found.')
530         for line in logText.splitlines():
531             match = regex.match(line)
532             if not match:
533                 continue
534             return int(match.group('count'))
535         return 0
536
537
538 class RunBindingsTests(shell.Test):
539     name = "bindings-generation-tests"
540     description = ["bindings-tests running"]
541     descriptionDone = ["bindings-tests"]
542     command = ["python", "./Tools/Scripts/run-bindings-tests"]
543
544
545 class RunEflAPITests(shell.Test):
546     name = "API tests"
547     description = ["API tests running"]
548     descriptionDone = ["API tests"]
549     command = ["perl", "./Tools/Scripts/run-efl-tests", WithProperties("--%(configuration)s")]
550
551
552 class RunGtkAPITests(shell.Test):
553     name = "API tests"
554     description = ["API tests running"]
555     descriptionDone = ["API tests"]
556     command = ["python", "./Tools/Scripts/run-gtk-tests", "--verbose", WithProperties("--%(configuration)s")]
557
558     def commandComplete(self, cmd):
559         shell.Test.commandComplete(self, cmd)
560
561         logText = cmd.logs['stdio'].getText()
562
563         self.incorrectTests = 0
564         self.crashedTests = 0
565         self.timedOutTests = 0
566         self.skippedTests = 0
567         self.statusLine = []
568
569         foundItems = re.findall("Tests failed \((\d+)\):", logText)
570         if (foundItems):
571             self.incorrectTests = int(foundItems[0])
572
573         foundItems = re.findall("Tests that crashed \((\d+)\):", logText)
574         if (foundItems):
575             self.crashedTests = int(foundItems[0])
576
577         foundItems = re.findall("Tests that timed out \((\d+)\):", logText)
578         if (foundItems):
579             self.timedOutTests = int(foundItems[0])
580
581         foundItems = re.findall("Tests skipped \((\d+)\):", logText)
582         if (foundItems):
583             self.skippedTests = int(foundItems[0])
584
585         self.totalFailedTests = self.incorrectTests + self.crashedTests + self.timedOutTests
586
587         if self.totalFailedTests > 0:
588             self.statusLine = [
589                 "%d API tests failed, %d crashed, %d timed out, %d skipped" %
590                 (self.incorrectTests, self.crashedTests, self.timedOutTests, self.skippedTests)
591             ]
592
593     def evaluateCommand(self, cmd):
594         if self.totalFailedTests > 0:
595             return FAILURE
596
597         if cmd.rc != 0:
598             return FAILURE
599
600         return SUCCESS
601
602     def getText(self, cmd, results):
603         return self.getText2(cmd, results)
604
605     def getText2(self, cmd, results):
606         if results != SUCCESS and self.totalFailedTests > 0:
607             return self.statusLine
608
609         return [self.name]
610
611 class RunGtkWebKitGObjectDOMBindingsAPIBreakTests(shell.Test):
612     name = "WebKit GObject DOM bindings API break tests"
613     description = ["WebKit GObject DOM bindings API break tests running"]
614     descriptionDone = ["WebKit GObject DOM bindings API break tests"]
615     command = ["./Tools/gtk/check-for-webkitdom-api-breaks"]
616
617     def commandComplete(self, cmd):
618         shell.Test.commandComplete(self, cmd)
619
620         logText = cmd.logs['stdio'].getText()
621
622         self.missingAPI = bool(re.findall("Missing API", logText))
623         self.newAPI = bool(re.findall("New API", logText))
624
625         line = ""
626         if self.newAPI:
627             line += "Found new API."
628         if self.missingAPI:
629             line += "Found missing API."
630         if line:
631             self.statusLine = [ line ]
632
633     def evaluateCommand(self, cmd):
634         if self.missingAPI:
635             return FAILURE
636         if self.newAPI:
637             return WARNINGS
638         if cmd.rc != 0:
639             return FAILURE
640         return SUCCESS
641
642 class RunWebKit1Tests(RunWebKitTests):
643     def start(self):
644         self.setCommand(self.command + ["--dump-render-tree"])
645
646         return RunWebKitTests.start(self)
647
648 class RunWebKit1LeakTests(RunWebKit1Tests):
649     warnOnWarnings = True
650     def start(self):
651         self.setCommand(self.command + ["--leaks"])
652         return RunWebKit1Tests.start(self)
653
654 class RunAndUploadPerfTests(shell.Test):
655     name = "perf-test"
656     description = ["perf-tests running"]
657     descriptionDone = ["perf-tests"]
658     command = ["python", "./Tools/Scripts/run-perf-tests",
659                "--output-json-path", "perf-test-results.json",
660                "--slave-config-json-path", "../../perf-test-config.json",
661                "--no-show-results",
662                "--reset-results",
663                "--test-results-server", "perf.webkit.org",
664                "--builder-name", WithProperties("%(buildername)s"),
665                "--build-number", WithProperties("%(buildnumber)s"),
666                "--platform", WithProperties("%(fullPlatform)s"),
667                "--no-build",
668                WithProperties("--%(configuration)s")]
669
670     def start(self):
671         self.setCommand(self.command)
672         return shell.Test.start(self)
673
674     def getText(self, cmd, results):
675         return self.getText2(cmd, results)
676
677     def getText2(self, cmd, results):
678         if results != SUCCESS:
679             if cmd.rc == -1 & 0xff:
680                 return ["build not up to date"]
681             elif cmd.rc == -2 & 0xff:
682                 return ["slave config JSON error"]
683             elif cmd.rc == -3 & 0xff:
684                 return ["output JSON merge error"]
685             elif cmd.rc == -4 & 0xff:
686                 return ["upload error"]
687             elif cmd.rc == -5 & 0xff:
688                 return ["system dependency error"]
689             elif cmd.rc == -1:
690                 return ["timeout"]
691             else:
692                 return ["%d perf tests failed" % cmd.rc]
693
694         return [self.name]
695
696
697 class ArchiveTestResults(shell.ShellCommand):
698     command = ["python", "./Tools/BuildSlaveSupport/test-result-archive",
699                WithProperties("--platform=%(platform)s"), WithProperties("--%(configuration)s"), "archive"]
700     name = "archive-test-results"
701     description = ["archiving test results"]
702     descriptionDone = ["archived test results"]
703     haltOnFailure = True
704
705
706 class UploadTestResults(transfer.FileUpload):
707     slavesrc = "layout-test-results.zip"
708     masterdest = WithProperties("public_html/results/%(buildername)s/r%(got_revision)s (%(buildnumber)s).zip")
709
710     def __init__(self, **kwargs):
711         kwargs['slavesrc'] = self.slavesrc
712         kwargs['masterdest'] = self.masterdest
713         kwargs['mode'] = 0644
714         transfer.FileUpload.__init__(self, **kwargs)
715
716
717 class ExtractTestResults(master.MasterShellCommand):
718     zipFile = WithProperties("public_html/results/%(buildername)s/r%(got_revision)s (%(buildnumber)s).zip")
719     resultDirectory = WithProperties("public_html/results/%(buildername)s/r%(got_revision)s (%(buildnumber)s)")
720     descriptionDone = ["uploaded results"]
721
722     def __init__(self, **kwargs):
723         kwargs['command'] = ""
724         master.MasterShellCommand.__init__(self, **kwargs)
725
726     def resultDirectoryURL(self):
727         return self.build.getProperties().render(self.resultDirectory).replace("public_html/", "/") + "/"
728
729     def start(self):
730         self.command = ["unzip", self.build.getProperties().render(self.zipFile), "-d", self.build.getProperties().render(self.resultDirectory)]
731         return master.MasterShellCommand.start(self)
732
733     def addCustomURLs(self):
734         url = self.resultDirectoryURL() + "results.html"
735         self.addURL("view results", url)
736
737     def finished(self, result):
738         self.addCustomURLs()
739         return master.MasterShellCommand.finished(self, result)
740
741
742 class ExtractTestResultsAndLeaks(ExtractTestResults):
743     def addCustomURLs(self):
744         ExtractTestResults.addCustomURLs(self)
745         url = "/LeaksViewer/?url=" + urllib.quote(self.resultDirectoryURL(), safe="")
746         self.addURL("view leaks", url)
747
748
749 class Factory(factory.BuildFactory):
750     def __init__(self, platform, configuration, architectures, buildOnly, additionalArguments, SVNMirror):
751         factory.BuildFactory.__init__(self)
752         self.addStep(ConfigureBuild(platform=platform, configuration=configuration, architecture=" ".join(architectures), buildOnly=buildOnly, additionalArguments=additionalArguments, SVNMirror=SVNMirror))
753         if SVNMirror:
754             self.addStep(WaitForSVNServer())
755         self.addStep(CheckOutSource(SVNMirror=SVNMirror))
756         if not (platform == "efl" and ('armv7' in architectures or 'aarch64' in architectures)):
757             self.addStep(KillOldProcesses())
758         self.addStep(CleanBuildIfScheduled())
759         self.addStep(DeleteStaleBuildFiles())
760         if platform == "win":
761             self.addStep(InstallWin32Dependencies())
762         if platform == "gtk":
763             self.addStep(InstallGtkDependencies())
764         if platform == "efl":
765             self.addStep(InstallEflDependencies())
766
767
768 class BuildFactory(Factory):
769     def __init__(self, platform, configuration, architectures, triggers=None, additionalArguments=None, SVNMirror=None):
770         Factory.__init__(self, platform, configuration, architectures, True, additionalArguments, SVNMirror)
771
772         if platform == "win":
773             self.addStep(CompileWebKit(timeout=2*60*60))
774         else:
775             self.addStep(CompileWebKit())
776
777         if triggers:
778             self.addStep(ArchiveBuiltProduct())
779             self.addStep(UploadBuiltProduct())
780             self.addStep(trigger.Trigger(schedulerNames=triggers))
781
782 def pickLatestBuild(builder, requests):
783     return max(requests, key=operator.attrgetter("submittedAt"))
784
785 class TestFactory(Factory):
786     JSCTestClass = RunJavaScriptCoreTests
787     LayoutTestClass = RunWebKitTests
788
789     def getProduct(self):
790         self.addStep(DownloadBuiltProduct())
791         self.addStep(ExtractBuiltProduct())
792
793     def __init__(self, platform, configuration, architectures, additionalArguments=None, SVNMirror=None, **kwargs):
794         Factory.__init__(self, platform, configuration, architectures, False, additionalArguments, SVNMirror, **kwargs)
795         self.getProduct()
796         if self.JSCTestClass:
797             self.addStep(self.JSCTestClass())
798         if self.LayoutTestClass:
799             self.addStep(self.LayoutTestClass())
800
801         if platform == 'win' or platform.startswith('mac') or platform.startswith('ios-simulator'):
802             self.addStep(RunUnitTests())
803         self.addStep(RunPythonTests())
804         self.addStep(RunPerlTests())
805         self.addStep(RunBindingsTests())
806         if self.LayoutTestClass:
807             self.addStep(ArchiveTestResults())
808             self.addStep(UploadTestResults())
809             self.addStep(ExtractTestResults())
810         if platform == "efl":
811             self.addStep(RunEflAPITests())
812         if platform == "gtk":
813             self.addStep(RunGtkAPITests())
814             self.addStep(RunGtkWebKitGObjectDOMBindingsAPIBreakTests())
815
816 class BuildAndTestFactory(TestFactory):
817     def getProduct(self):
818         self.addStep(CompileWebKit())
819
820     def __init__(self, platform, configuration, architectures, triggers=None, additionalArguments=None, SVNMirror=None, **kwargs):
821         TestFactory.__init__(self, platform, configuration, architectures, additionalArguments, SVNMirror, **kwargs)
822         if triggers:
823             self.addStep(ArchiveBuiltProduct())
824             self.addStep(UploadBuiltProduct())
825             self.addStep(trigger.Trigger(schedulerNames=triggers))
826
827 class BuildAndTestLLINTCLoopFactory(Factory):
828     def __init__(self, platform, configuration, architectures, triggers=None, additionalArguments=None, SVNMirror=None, **kwargs):
829         Factory.__init__(self, platform, configuration, architectures, False, additionalArguments, SVNMirror, **kwargs)
830         self.addStep(CompileLLINTCLoop())
831         self.addStep(RunLLINTCLoopTests())
832
833 class BuildAndTest32bitJSCFactory(Factory):
834     def __init__(self, platform, configuration, architectures, triggers=None, additionalArguments=None, SVNMirror=None, **kwargs):
835         Factory.__init__(self, platform, configuration, architectures, False, additionalArguments, SVNMirror, **kwargs)
836         self.addStep(Compile32bitJSC())
837         self.addStep(Run32bitJSCTests())
838
839 class BuildAndNonLayoutTestFactory(BuildAndTestFactory):
840     LayoutTestClass = None
841
842 class BuildAndRemoteJSCTestsFactory(BuildFactory):
843     def __init__(self, platform, configuration, architectures, triggers=None, additionalArguments=None, SVNMirror=None):
844         BuildFactory.__init__(self, platform, configuration, architectures, False, additionalArguments, SVNMirror)
845         self.addStep(RunRemoteJavaScriptCoreTests())
846
847 class TestWebKit1LeaksFactory(Factory):
848     def __init__(self, platform, configuration, architectures, additionalArguments=None, SVNMirror=None):
849         Factory.__init__(self, platform, configuration, architectures, False, additionalArguments, SVNMirror)
850         self.addStep(DownloadBuiltProduct())
851         self.addStep(ExtractBuiltProduct())
852         self.addStep(RunWebKit1LeakTests())
853         self.addStep(ArchiveTestResults())
854         self.addStep(UploadTestResults())
855         self.addStep(ExtractTestResultsAndLeaks())
856
857 class TestAllButJSCFactory(TestFactory):
858     JSCTestClass = None
859
860 class TestJSCFactory(Factory):
861     def __init__(self, platform, configuration, architectures, additionalArguments=None, SVNMirror=None):
862         Factory.__init__(self, platform, configuration, architectures, False, additionalArguments, SVNMirror)
863         self.addStep(DownloadBuiltProduct())
864         self.addStep(ExtractBuiltProduct())
865         self.addStep(RunJavaScriptCoreTests())
866
867 class TestWebKit1Factory(TestFactory):
868     LayoutTestClass = RunWebKit1Tests
869
870 class TestWebKit1AllButJSCFactory(TestWebKit1Factory):
871     JSCTestClass = None
872
873 class BuildAndPerfTestFactory(Factory):
874     def __init__(self, platform, configuration, architectures, additionalArguments=None, SVNMirror=None, **kwargs):
875         Factory.__init__(self, platform, configuration, architectures, False, additionalArguments, SVNMirror, **kwargs)
876         self.addStep(CompileWebKit())
877         self.addStep(RunAndUploadPerfTests())
878
879 class DownloadAndPerfTestFactory(Factory):
880     def __init__(self, platform, configuration, architectures, additionalArguments=None, SVNMirror=None, **kwargs):
881         Factory.__init__(self, platform, configuration, architectures, False, additionalArguments, SVNMirror, **kwargs)
882         self.addStep(DownloadBuiltProduct())
883         self.addStep(ExtractBuiltProduct())
884         self.addStep(RunAndUploadPerfTests())
885
886 class PlatformSpecificScheduler(AnyBranchScheduler):
887     def __init__(self, platform, branch, **kwargs):
888         self.platform = platform
889         filter = ChangeFilter(branch=[branch, None], filter_fn=self.filter)
890         AnyBranchScheduler.__init__(self, name=platform, change_filter=filter, **kwargs)
891
892     def filter(self, change):
893         return wkbuild.should_build(self.platform, change.files)
894
895 trunk_filter = ChangeFilter(branch=["trunk", None])
896
897 def loadBuilderConfig(c):
898     # FIXME: These file handles are leaked.
899     passwords = json.load(open('passwords.json'))
900     config = json.load(open('config.json'))
901
902     c['slaves'] = [BuildSlave(slave['name'], passwords[slave['name']], max_builds=1) for slave in config['slaves']]
903
904     c['schedulers'] = []
905     for scheduler in config['schedulers']:
906         if "change_filter" in scheduler:
907             scheduler["change_filter"] = globals()[scheduler["change_filter"]]
908         kls = globals()[scheduler.pop('type')]
909         # Python 2.6 can't handle unicode keys as keyword arguments:
910         # http://bugs.python.org/issue2646.  Modern versions of json return
911         # unicode strings from json.load, so we map all keys to str objects.
912         scheduler = dict(map(lambda key_value_pair: (str(key_value_pair[0]), key_value_pair[1]), scheduler.items()))
913
914         c['schedulers'].append(kls(**scheduler))
915
916     forceScheduler = ForceScheduler(
917         name="force",
918         builderNames=[str(builder['name']) for builder in config['builders']],
919         reason=StringParameter(name="reason", default="", size=40),
920
921         # Validate SVN revision: number or empty string
922         revision=StringParameter(name="revision", default="", regex=re.compile(r'^(\d*)$')),
923
924         # Disable default enabled input fields: branch, repository, project, additional properties
925         branch=FixedParameter(name="branch"),
926         repository=FixedParameter(name="repository"),
927         project=FixedParameter(name="project"),
928         properties=[BooleanParameter(name="is_clean", label="Force Clean build")]
929     )
930     c['schedulers'].append(forceScheduler)
931
932     c['builders'] = []
933     for builder in config['builders']:
934         for slaveName in builder['slavenames']:
935             for slave in config['slaves']:
936                 if slave['name'] != slaveName or slave['platform'] == '*':
937                     continue
938
939                 if slave['platform'] != builder['platform']:
940                     raise Exception, "Builder %r is for platform %r but has slave %r for platform %r!" % (builder['name'], builder['platform'], slave['name'], slave['platform'])
941
942                 break
943
944         platform = builder['platform']
945
946         builderType = builder.pop('type')
947         factory = globals()["%sFactory" % builderType]
948         factorykwargs = {}
949         for key in "platform", "configuration", "architectures", "triggers", "additionalArguments", "SVNMirror":
950             value = builder.pop(key, None)
951             if value:
952                 factorykwargs[key] = value
953
954         builder["factory"] = factory(**factorykwargs)
955
956         if platform.startswith('mac'):
957             builder["category"] = 'AppleMac'
958         elif platform == 'win':
959             builder["category"] = 'AppleWin'
960         elif platform.startswith('gtk'):
961             builder["category"] = 'GTK'
962         elif platform.startswith('efl'):
963             builder["category"] = "EFL"
964         else:
965             builder["category"] = 'misc'
966
967         if (builder['category'] == 'AppleMac' or builder['category'] == 'AppleWin') and builderType != 'Build':
968             builder['nextBuild'] = pickLatestBuild
969
970         c['builders'].append(builder)
971
972 loadBuilderConfig(c)