Add support for test262 JavaScriptCore tests
[WebKit-https.git] / Tools / BuildSlaveSupport / build.webkit.org-config / master.cfg
1 # -*- python -*-
2 # ex: set syntax=python:
3
4 from buildbot.buildslave import BuildSlave
5 from buildbot.changes.pb import PBChangeSource
6 from buildbot.scheduler import AnyBranchScheduler, Triggerable
7 from buildbot.schedulers.forcesched import FixedParameter, ForceScheduler, StringParameter, BooleanParameter
8 from buildbot.schedulers.filter import ChangeFilter
9 from buildbot.status import html
10 from buildbot.status.web.authz import Authz
11 from buildbot.process import buildstep, factory, properties
12 from buildbot.steps import master, shell, source, transfer, trigger
13 from buildbot.status.builder import SUCCESS, FAILURE, WARNINGS, SKIPPED, EXCEPTION
14
15 from twisted.internet import defer
16
17 import os
18 import re
19 import json
20 import operator
21 import cStringIO
22 import urllib
23
24 from committer_auth import CommitterAuth
25 import wkbuild
26
27
28 c = BuildmasterConfig = {}
29
30 c['change_source'] = PBChangeSource(port=16000)
31
32 # permissions for WebStatus
33 authz = Authz(
34     auth=CommitterAuth('auth.json'),
35     forceBuild='auth',
36     forceAllBuilds='auth',
37     pingBuilder=True,
38     gracefulShutdown=False,
39     stopBuild='auth',
40     stopAllBuilds='auth',
41     cancelPendingBuild='auth',
42     stopChange=True,
43     cleanShutdown=False)
44
45 c['status'] = []
46 c['status'].append(html.WebStatus(http_port=8710,
47                                   revlink="https://trac.webkit.org/changeset/%s", 
48                                   changecommentlink=(r"(https://bugs\.webkit\.org/show_bug\.cgi\?id=|webkit\.org/b/)(\d+)", r"https://bugs.webkit.org/show_bug.cgi?id=\2"),
49                                   authz=authz))
50
51 c['slavePortnum'] = 17000
52 c['projectName'] = "WebKit"
53 c['projectURL'] = "https://webkit.org"
54 c['buildbotURL'] = "https://build.webkit.org/"
55
56 c['buildHorizon'] = 1000
57 c['logHorizon'] = 500
58 c['eventHorizon'] = 200
59 c['buildCacheSize'] = 60
60
61 WithProperties = properties.WithProperties
62
63
64 class TestWithFailureCount(shell.Test):
65     failedTestsFormatString = "%d test%s failed"
66
67     def countFailures(self, cmd):
68         return 0
69
70     def commandComplete(self, cmd):
71         shell.Test.commandComplete(self, cmd)
72         self.failedTestCount = self.countFailures(cmd)
73         self.failedTestPluralSuffix = "" if self.failedTestCount == 1 else "s"
74
75     def evaluateCommand(self, cmd):
76         if self.failedTestCount:
77             return FAILURE
78
79         if cmd.rc != 0:
80             return FAILURE
81
82         return SUCCESS
83
84     def getText(self, cmd, results):
85         return self.getText2(cmd, results)
86
87     def getText2(self, cmd, results):
88         if results != SUCCESS and self.failedTestCount:
89             return [self.failedTestsFormatString % (self.failedTestCount, self.failedTestPluralSuffix)]
90
91         return [self.name]
92
93
94 class ConfigureBuild(buildstep.BuildStep):
95     name = "configure build"
96     description = ["configuring build"]
97     descriptionDone = ["configured build"]
98     def __init__(self, platform, configuration, architecture, buildOnly, additionalArguments, SVNMirror, *args, **kwargs):
99         buildstep.BuildStep.__init__(self, *args, **kwargs)
100         self.platform = platform
101         if platform != 'jsc-only':
102             self.platform = platform.split('-', 1)[0]
103         self.fullPlatform = platform
104         self.configuration = configuration
105         self.architecture = architecture
106         self.buildOnly = buildOnly
107         self.additionalArguments = additionalArguments
108         self.SVNMirror = SVNMirror
109         self.addFactoryArguments(platform=platform, configuration=configuration, architecture=architecture, buildOnly=buildOnly, additionalArguments=additionalArguments, SVNMirror=SVNMirror)
110
111     def start(self):
112         self.setProperty("platform", self.platform)
113         self.setProperty("fullPlatform", self.fullPlatform)
114         self.setProperty("configuration", self.configuration)
115         self.setProperty("architecture", self.architecture)
116         self.setProperty("buildOnly", self.buildOnly)
117         self.setProperty("additionalArguments", self.additionalArguments)
118         self.setProperty("SVNMirror", self.SVNMirror)
119         self.finished(SUCCESS)
120         return defer.succeed(None)
121
122
123 class CheckOutSource(source.SVN):
124     mode = "update"
125     def __init__(self, SVNMirror, **kwargs):
126         kwargs['baseURL'] = SVNMirror or "https://svn.webkit.org/repository/webkit/"
127         kwargs['defaultBranch'] = "trunk"
128         kwargs['mode'] = self.mode
129         source.SVN.__init__(self, **kwargs)
130         self.addFactoryArguments(SVNMirror=SVNMirror)
131
132 class WaitForSVNServer(shell.ShellCommand):
133     name = "wait-for-svn-server"
134     command = ["python", "./Tools/BuildSlaveSupport/wait-for-SVN-server.py", "-r", WithProperties("%(revision)s"), "-s", WithProperties("%(SVNMirror)s")]
135     description = ["waiting for SVN server"]
136     descriptionDone = ["SVN server is ready"]
137     warnOnFailure = True
138
139     def evaluateCommand(self, cmd):
140         if cmd.rc != 0:
141             return WARNINGS
142         return SUCCESS
143
144 class InstallWin32Dependencies(shell.Compile):
145     description = ["installing dependencies"]
146     descriptionDone = ["installed dependencies"]
147     command = ["perl", "./Tools/Scripts/update-webkit-auxiliary-libs"]
148
149 class KillOldProcesses(shell.Compile):
150     name = "kill old processes"
151     description = ["killing old processes"]
152     descriptionDone = ["killed old processes"]
153     command = ["python", "./Tools/BuildSlaveSupport/kill-old-processes"]
154
155 class CleanBuildIfScheduled(shell.Compile):
156     name = "delete WebKitBuild directory"
157     description = ["deleting WebKitBuild directory"]
158     descriptionDone = ["deleted WebKitBuild directory"]
159     command = ["python", "./Tools/BuildSlaveSupport/clean-build", WithProperties("--platform=%(fullPlatform)s"), WithProperties("--%(configuration)s")]
160
161     def start(self):
162         if not self.getProperty('is_clean'):
163             self.hideStepIf = True
164             return SKIPPED
165         return shell.Compile.start(self)
166
167 class DeleteStaleBuildFiles(shell.Compile):
168     name = "delete stale build files"
169     description = ["deleting stale build files"]
170     descriptionDone = ["deleted stale build files"]
171     command = ["python", "./Tools/BuildSlaveSupport/delete-stale-build-files", WithProperties("--platform=%(fullPlatform)s"), WithProperties("--%(configuration)s")]
172
173     def start(self):
174         if self.getProperty('is_clean'): # Nothing to be done if WebKitBuild had been removed.
175             self.hideStepIf = True
176             return SKIPPED
177         return shell.Compile.start(self)
178
179 class InstallGtkDependencies(shell.ShellCommand):
180     name = "jhbuild"
181     description = ["updating gtk dependencies"]
182     descriptionDone = ["updated gtk dependencies"]
183     command = ["perl", "./Tools/Scripts/update-webkitgtk-libs"]
184     haltOnFailure = True
185
186 def appendCustomBuildFlags(step, platform, fullPlatform):
187     if platform not in ('gtk', 'wincairo', 'ios', 'jsc-only'):
188         return
189     if fullPlatform.startswith('ios-simulator'):
190         platform = 'ios-simulator'
191     elif platform == 'ios':
192         platform = 'device'
193     step.setCommand(step.command + ['--' + platform])
194
195 class CompileWebKit(shell.Compile):
196     command = ["perl", "./Tools/Scripts/build-webkit", WithProperties("--%(configuration)s")]
197     env = {'MFLAGS':''}
198     name = "compile-webkit"
199     description = ["compiling"]
200     descriptionDone = ["compiled"]
201     warningPattern = ".*arning: .*"
202
203     def start(self):
204         platform = self.getProperty('platform')
205         buildOnly = self.getProperty('buildOnly')
206         architecture = self.getProperty('architecture')
207         additionalArguments = self.getProperty('additionalArguments')
208
209         if additionalArguments:
210             self.setCommand(self.command + additionalArguments)
211         if platform in ('mac', 'ios') and architecture:
212             self.setCommand(self.command + ['ARCHS=' + architecture])
213             if platform == 'ios':
214                 self.setCommand(self.command + ['ONLY_ACTIVE_ARCH=NO'])
215         # Generating dSYM files is slow, but these are needed to have line numbers in crash reports on testers.
216         # Debug builds on Yosemite can't use dSYMs, because crash logs end up unsymbolicated.
217         if platform in ('mac', 'ios') and buildOnly and (self.getProperty('fullPlatform') != "mac-yosemite" or self.getProperty('configuration') != "debug"):
218             self.setCommand(self.command + ['DEBUG_INFORMATION_FORMAT=dwarf-with-dsym'])
219
220         appendCustomBuildFlags(self, platform, self.getProperty('fullPlatform'))
221
222         return shell.Compile.start(self)
223
224     def createSummary(self, log):
225         platform = self.getProperty('platform')
226         if platform.startswith('mac'):    
227             warnings = []
228             errors = []
229             sio = cStringIO.StringIO(log.getText())
230             for line in sio.readlines():
231                 if "arning:" in line:
232                     warnings.append(line)
233                 if "rror:" in line:
234                     errors.append(line)
235             if warnings:
236                 self.addCompleteLog('warnings', "".join(warnings))
237             if errors:
238                 self.addCompleteLog('errors', "".join(errors))
239
240
241 class CompileLLINTCLoop(CompileWebKit):
242     command = ["perl", "./Tools/Scripts/build-jsc", "--cloop", WithProperties("--%(configuration)s")]
243
244 class Compile32bitJSC(CompileWebKit):
245     command = ["perl", "./Tools/Scripts/build-jsc", "--32-bit", WithProperties("--%(configuration)s")]
246
247 class CompileJSCOnly(CompileWebKit):
248     command = ["perl", "./Tools/Scripts/build-jsc", WithProperties("--%(configuration)s")]
249
250 class ArchiveBuiltProduct(shell.ShellCommand):
251     command = ["python", "./Tools/BuildSlaveSupport/built-product-archive",
252                WithProperties("--platform=%(fullPlatform)s"), WithProperties("--%(configuration)s"), "archive"]
253     name = "archive-built-product"
254     description = ["archiving built product"]
255     descriptionDone = ["archived built product"]
256     haltOnFailure = True
257
258
259 class ExtractBuiltProduct(shell.ShellCommand):
260     command = ["python", "./Tools/BuildSlaveSupport/built-product-archive",
261                WithProperties("--platform=%(fullPlatform)s"), WithProperties("--%(configuration)s"), "extract"]
262     name = "extract-built-product"
263     description = ["extracting built product"]
264     descriptionDone = ["extracted built product"]
265     haltOnFailure = True
266
267
268 class UploadBuiltProduct(transfer.FileUpload):
269     slavesrc = WithProperties("WebKitBuild/%(configuration)s.zip")
270     masterdest = WithProperties("archives/%(fullPlatform)s-%(architecture)s-%(configuration)s/%(got_revision)s.zip")
271     haltOnFailure = True
272
273     def __init__(self, **kwargs):
274         kwargs['slavesrc'] = self.slavesrc
275         kwargs['masterdest'] = self.masterdest
276         kwargs['mode'] = 0644
277         kwargs['blocksize'] = 1024*256
278         transfer.FileUpload.__init__(self, **kwargs)
279
280
281 class DownloadBuiltProduct(shell.ShellCommand):
282     command = ["python", "./Tools/BuildSlaveSupport/download-built-product",
283         WithProperties("--platform=%(platform)s"), WithProperties("--%(configuration)s"),
284         WithProperties(c["buildbotURL"] + "archives/%(fullPlatform)s-%(architecture)s-%(configuration)s/%(got_revision)s.zip")]
285     name = "download-built-product"
286     description = ["downloading built product"]
287     descriptionDone = ["downloaded built product"]
288     haltOnFailure = True
289     flunkOnFailure = True
290
291
292 class RunJavaScriptCoreTests(TestWithFailureCount):
293     name = "jscore-test"
294     description = ["jscore-tests running"]
295     descriptionDone = ["jscore-tests"]
296     jsonFileName = "jsc_results.json"
297     command = ["perl", "./Tools/Scripts/run-javascriptcore-tests", "--no-build", "--no-fail-fast", "--json-output={0}".format(jsonFileName), WithProperties("--%(configuration)s")]
298     failedTestsFormatString = "%d JSC test%s failed"
299     logfiles = {"json": jsonFileName}
300
301     def start(self):
302         appendCustomBuildFlags(self, self.getProperty('platform'), self.getProperty('fullPlatform'))
303         return shell.Test.start(self)
304
305     def countFailures(self, cmd):
306         logText = cmd.logs['stdio'].getText()
307
308         match = re.search(r'^Results for JSC stress tests:\r?\n\s+(\d+) failure', logText, re.MULTILINE)
309         if match:
310             return int(match.group(1))
311
312         match = re.search(r'^Results for Mozilla tests:\r?\n\s+(\d+) regression', logText, re.MULTILINE)
313         if match:
314             return int(match.group(1))
315
316         return 0
317
318
319 class RunRemoteJavaScriptCoreTests(RunJavaScriptCoreTests):
320     def start(self):
321         self.setCommand(self.command + ["--memory-limited", "--remote-config-file", "../../remote-jsc-tests-config.json"])
322         return RunJavaScriptCoreTests.start(self)
323
324
325 class RunTest262Tests(TestWithFailureCount):
326     name = "test262-test"
327     description = ["test262-tests running"]
328     descriptionDone = ["test262-tests"]
329     failedTestsFormatString = "%d Test262 test%s failed"
330     command = ["perl", "./Tools/Scripts/run-jsc-stress-tests", WithProperties("--%(configuration)s"), "JSTests/test262.yaml"]
331
332     def start(self):
333         appendCustomBuildFlags(self, self.getProperty('platform'), self.getProperty('fullPlatform'))
334         return shell.Test.start(self)
335
336     def countFailures(self, cmd):
337         logText = cmd.logs['stdio'].getText()
338         matches = re.findall(r'\(failed (\d+)\)', logText)
339         if matches:
340             return int(matches[-1])
341         return 0
342
343
344 class RunWebKitTests(shell.Test):
345     name = "layout-test"
346     description = ["layout-tests running"]
347     descriptionDone = ["layout-tests"]
348     resultDirectory = "layout-test-results"
349     command = ["python", "./Tools/Scripts/run-webkit-tests",
350                "--no-build",
351                "--no-show-results",
352                "--no-new-test-results",
353                "--builder-name", WithProperties("%(buildername)s"),
354                "--build-number", WithProperties("%(buildnumber)s"),
355                "--master-name", "webkit.org",
356                "--test-results-server", "webkit-test-results.webkit.org",
357                "--exit-after-n-crashes-or-timeouts", "50",
358                "--exit-after-n-failures", "500",
359                WithProperties("--%(configuration)s")]
360
361     def start(self):
362         platform = self.getProperty('platform')
363         appendCustomBuildFlags(self, platform, self.getProperty('fullPlatform'))
364         additionalArguments = self.getProperty('additionalArguments')
365
366         self.setCommand(self.command + ["--results-directory", self.resultDirectory])
367         self.setCommand(self.command + ['--debug-rwt-logging'])
368
369         if platform == "win":
370             self.setCommand(self.command + ['--batch-size', '100', '--root=' + os.path.join("WebKitBuild", self.getProperty('configuration'), "bin32")])
371
372         if additionalArguments:
373             self.setCommand(self.command + additionalArguments)
374         return shell.Test.start(self)
375
376     # FIXME: This will break if run-webkit-tests changes its default log formatter.
377     nrwt_log_message_regexp = re.compile(r'\d{2}:\d{2}:\d{2}(\.\d+)?\s+\d+\s+(?P<message>.*)')
378
379     def _strip_python_logging_prefix(self, line):
380         match_object = self.nrwt_log_message_regexp.match(line)
381         if match_object:
382             return match_object.group('message')
383         return line
384
385     def _parseRunWebKitTestsOutput(self, logText):
386         incorrectLayoutLines = []
387         expressions = [
388             ('flakes', re.compile(r'Unexpected flakiness.+\((\d+)\)')),
389             ('new passes', re.compile(r'Expected to .+, but passed:\s+\((\d+)\)')),
390             ('missing results', re.compile(r'Regressions: Unexpected missing results\s+\((\d+)\)')),
391             ('failures', re.compile(r'Regressions: Unexpected.+\((\d+)\)')),
392         ]
393         testFailures = {}
394
395         for line in logText.splitlines():
396             if line.find('Exiting early') >= 0 or line.find('leaks found') >= 0:
397                 incorrectLayoutLines.append(self._strip_python_logging_prefix(line))
398                 continue
399             for name, expression in expressions:
400                 match = expression.search(line)
401
402                 if match:
403                     testFailures[name] = testFailures.get(name, 0) + int(match.group(1))
404                     break
405
406                 # FIXME: Parse file names and put them in results
407
408         for name in testFailures:
409             incorrectLayoutLines.append(str(testFailures[name]) + ' ' + name)
410
411         self.incorrectLayoutLines = incorrectLayoutLines
412
413     def commandComplete(self, cmd):
414         shell.Test.commandComplete(self, cmd)
415
416         logText = cmd.logs['stdio'].getText()
417         self._parseRunWebKitTestsOutput(logText)
418
419     def evaluateCommand(self, cmd):
420         result = SUCCESS
421
422         if self.incorrectLayoutLines:
423             if len(self.incorrectLayoutLines) == 1:
424                 line = self.incorrectLayoutLines[0]
425                 if line.find('were new') >= 0 or line.find('was new') >= 0 or line.find(' leak') >= 0:
426                     return WARNINGS
427
428             for line in self.incorrectLayoutLines:
429                 if line.find('flakes') >= 0 or line.find('new passes') >= 0 or line.find('missing results') >= 0:
430                     result = WARNINGS
431                 else:
432                     return FAILURE
433
434         # Return code from Tools/Scripts/layout_tests/run_webkit_tests.py.
435         # This means that an exception was raised when running run-webkit-tests and
436         # was never handled.
437         if cmd.rc == 254:
438             return EXCEPTION
439         if cmd.rc != 0:
440             return FAILURE
441
442         return result
443
444     def getText(self, cmd, results):
445         return self.getText2(cmd, results)
446
447     def getText2(self, cmd, results):
448         if results != SUCCESS and self.incorrectLayoutLines:
449             return self.incorrectLayoutLines
450
451         return [self.name]
452
453
454 class RunDashboardTests(RunWebKitTests):
455     name = "dashboard-tests"
456     description = ["dashboard-tests running"]
457     descriptionDone = ["dashboard-tests"]
458     resultDirectory = os.path.join(RunWebKitTests.resultDirectory, "dashboard-layout-test-results")
459
460     def start(self):
461         self.setCommand(self.command + ["--layout-tests-directory", "./Tools/BuildSlaveSupport/build.webkit.org-config/public_html/dashboard/Scripts/tests"])
462         return RunWebKitTests.start(self)
463
464
465 class RunUnitTests(TestWithFailureCount):
466     name = "run-api-tests"
467     description = ["unit tests running"]
468     descriptionDone = ["unit-tests"]
469     command = ["perl", "./Tools/Scripts/run-api-tests", "--no-build", WithProperties("--%(configuration)s"), "--verbose"]
470     failedTestsFormatString = "%d unit test%s failed or timed out"
471
472     def start(self):
473         appendCustomBuildFlags(self, self.getProperty('platform'), self.getProperty('fullPlatform'))
474         return shell.Test.start(self)
475
476     def countFailures(self, cmd):
477         log_text = cmd.logs['stdio'].getText()
478         count = 0
479
480         split = re.split(r'\sTests that timed out:\s', log_text)
481         if len(split) > 1:
482             count += len(re.findall(r'^\s+\S+$', split[1], flags=re.MULTILINE))
483
484         split = re.split(r'\sTests that failed:\s', split[0])
485         if len(split) > 1:
486             count += len(re.findall(r'^\s+\S+$', split[1], flags=re.MULTILINE))
487
488         return count
489
490
491 class RunPythonTests(TestWithFailureCount):
492     name = "webkitpy-test"
493     description = ["python-tests running"]
494     descriptionDone = ["python-tests"]
495     command = ["python", "./Tools/Scripts/test-webkitpy", "--verbose"]
496     failedTestsFormatString = "%d python test%s failed"
497
498     def start(self):
499         platform = self.getProperty('platform')
500         # Python tests are flaky on the GTK builders, running them serially
501         # helps and does not significantly prolong the cycle time.
502         if platform == 'gtk':
503             self.setCommand(self.command + ['--child-processes', '1'])
504         # Python tests fail on windows bots when running more than one child process
505         # https://bugs.webkit.org/show_bug.cgi?id=97465
506         if platform == 'win':
507             self.setCommand(self.command + ['--child-processes', '1'])
508         return shell.Test.start(self)
509
510     def countFailures(self, cmd):
511         logText = cmd.logs['stdio'].getText()
512         # We're looking for the line that looks like this: FAILED (failures=2, errors=1)
513         regex = re.compile(r'^FAILED \((?P<counts>[^)]+)\)')
514         for line in logText.splitlines():
515             match = regex.match(line)
516             if not match:
517                 continue
518             return sum(int(component.split('=')[1]) for component in match.group('counts').split(', '))
519         return 0
520
521
522 class RunPerlTests(TestWithFailureCount):
523     name = "webkitperl-test"
524     description = ["perl-tests running"]
525     descriptionDone = ["perl-tests"]
526     command = ["perl", "./Tools/Scripts/test-webkitperl"]
527     failedTestsFormatString = "%d perl test%s failed"
528
529     def countFailures(self, cmd):
530         logText = cmd.logs['stdio'].getText()
531         # We're looking for the line that looks like this: Failed 2/19 test programs. 5/363 subtests failed.
532         regex = re.compile(r'^Failed \d+/\d+ test programs\. (?P<count>\d+)/\d+ subtests failed\.')
533         for line in logText.splitlines():
534             match = regex.match(line)
535             if not match:
536                 continue
537             return int(match.group('count'))
538         return 0
539
540
541 class RunLLINTCLoopTests(TestWithFailureCount):
542     name = "webkit-jsc-cloop-test"
543     description = ["cloop-tests running"]
544     descriptionDone = ["cloop-tests"]
545     jsonFileName = "jsc_cloop.json"
546     command = ["perl", "./Tools/Scripts/run-javascriptcore-tests", "--cloop", "--no-build", "--no-jsc-stress", "--no-fail-fast", "--json-output={0}".format(jsonFileName), WithProperties("--%(configuration)s")]
547     failedTestsFormatString = "%d regression%s found."
548     logfiles = {"json": jsonFileName}
549
550     def countFailures(self, cmd):
551         logText = cmd.logs['stdio'].getText()
552         # We're looking for the line that looks like this: 0 regressions found.
553         regex = re.compile(r'\s*(?P<count>\d+) regressions? found.')
554         for line in logText.splitlines():
555             match = regex.match(line)
556             if not match:
557                 continue
558             return int(match.group('count'))
559         return 0
560
561
562 class Run32bitJSCTests(TestWithFailureCount):
563     name = "webkit-32bit-jsc-test"
564     description = ["32bit-jsc-tests running"]
565     descriptionDone = ["32bit-jsc-tests"]
566     jsonFileName = "jsc_32bit.json"
567     command = ["perl", "./Tools/Scripts/run-javascriptcore-tests", "--32-bit", "--no-build", "--no-fail-fast", "--json-output={0}".format(jsonFileName), WithProperties("--%(configuration)s")]
568     failedTestsFormatString = "%d regression%s found."
569     logfiles = {"json": jsonFileName}
570
571     def countFailures(self, cmd):
572         logText = cmd.logs['stdio'].getText()
573         # We're looking for the line that looks like this: 0 failures found.
574         regex = re.compile(r'\s*(?P<count>\d+) failures? found.')
575         for line in logText.splitlines():
576             match = regex.match(line)
577             if not match:
578                 continue
579             return int(match.group('count'))
580         return 0
581
582
583 class RunBindingsTests(shell.Test):
584     name = "bindings-generation-tests"
585     description = ["bindings-tests running"]
586     descriptionDone = ["bindings-tests"]
587     command = ["python", "./Tools/Scripts/run-bindings-tests"]
588
589
590 class RunBuiltinsTests(shell.Test):
591     name = "builtins-generator-tests"
592     description = ["builtins-generator-tests running"]
593     descriptionDone = ["builtins-generator-tests"]
594     command = ["python", "./Tools/Scripts/run-builtins-generator-tests"]
595
596
597 class RunGtkAPITests(shell.Test):
598     name = "API tests"
599     description = ["API tests running"]
600     descriptionDone = ["API tests"]
601     command = ["python", "./Tools/Scripts/run-gtk-tests", "--verbose", WithProperties("--%(configuration)s")]
602
603     def start(self):
604         additionalArguments = self.getProperty("additionalArguments")
605         if additionalArguments:
606             self.command += additionalArguments
607         self.setCommand(self.command)
608         return shell.Test.start(self)
609
610     def commandComplete(self, cmd):
611         shell.Test.commandComplete(self, cmd)
612
613         logText = cmd.logs['stdio'].getText()
614
615         self.incorrectTests = 0
616         self.crashedTests = 0
617         self.timedOutTests = 0
618         self.skippedTests = 0
619         self.statusLine = []
620
621         foundItems = re.findall("Tests failed \((\d+)\):", logText)
622         if (foundItems):
623             self.incorrectTests = int(foundItems[0])
624
625         foundItems = re.findall("Tests that crashed \((\d+)\):", logText)
626         if (foundItems):
627             self.crashedTests = int(foundItems[0])
628
629         foundItems = re.findall("Tests that timed out \((\d+)\):", logText)
630         if (foundItems):
631             self.timedOutTests = int(foundItems[0])
632
633         foundItems = re.findall("Tests skipped \((\d+)\):", logText)
634         if (foundItems):
635             self.skippedTests = int(foundItems[0])
636
637         self.totalFailedTests = self.incorrectTests + self.crashedTests + self.timedOutTests
638
639         if self.totalFailedTests > 0:
640             self.statusLine = [
641                 "%d API tests failed, %d crashed, %d timed out, %d skipped" %
642                 (self.incorrectTests, self.crashedTests, self.timedOutTests, self.skippedTests)
643             ]
644
645     def evaluateCommand(self, cmd):
646         if self.totalFailedTests > 0:
647             return FAILURE
648
649         if cmd.rc != 0:
650             return FAILURE
651
652         return SUCCESS
653
654     def getText(self, cmd, results):
655         return self.getText2(cmd, results)
656
657     def getText2(self, cmd, results):
658         if results != SUCCESS and self.totalFailedTests > 0:
659             return self.statusLine
660
661         return [self.name]
662
663 class RunWebKit1Tests(RunWebKitTests):
664     def start(self):
665         self.setCommand(self.command + ["--dump-render-tree"])
666
667         return RunWebKitTests.start(self)
668
669 class RunWebKit1LeakTests(RunWebKit1Tests):
670     want_stdout = False
671     want_stderr = False
672     warnOnWarnings = True
673     def start(self):
674         self.setCommand(self.command + ["--leaks"])
675         return RunWebKit1Tests.start(self)
676
677 class RunAndUploadPerfTests(shell.Test):
678     name = "perf-test"
679     description = ["perf-tests running"]
680     descriptionDone = ["perf-tests"]
681     command = ["python", "./Tools/Scripts/run-perf-tests",
682                "--output-json-path", "perf-test-results.json",
683                "--slave-config-json-path", "../../perf-test-config.json",
684                "--no-show-results",
685                "--reset-results",
686                "--test-results-server", "perf.webkit.org",
687                "--builder-name", WithProperties("%(buildername)s"),
688                "--build-number", WithProperties("%(buildnumber)s"),
689                "--platform", WithProperties("%(fullPlatform)s"),
690                "--no-build",
691                WithProperties("--%(configuration)s")]
692
693     def start(self):
694         additionalArguments = self.getProperty("additionalArguments")
695         if additionalArguments:
696             self.command += additionalArguments
697         self.setCommand(self.command)
698         return shell.Test.start(self)
699
700     def getText(self, cmd, results):
701         return self.getText2(cmd, results)
702
703     def getText2(self, cmd, results):
704         if results != SUCCESS:
705             if cmd.rc == -1 & 0xff:
706                 return ["build not up to date"]
707             elif cmd.rc == -2 & 0xff:
708                 return ["slave config JSON error"]
709             elif cmd.rc == -3 & 0xff:
710                 return ["output JSON merge error"]
711             elif cmd.rc == -4 & 0xff:
712                 return ["upload error"]
713             elif cmd.rc == -5 & 0xff:
714                 return ["system dependency error"]
715             elif cmd.rc == -1:
716                 return ["timeout"]
717             else:
718                 return ["%d perf tests failed" % cmd.rc]
719
720         return [self.name]
721
722 class RunBenchmarkTests(shell.Test):
723     name = "benchmark-test"
724     description = ["benchmark tests running"]
725     descriptionDone = ["benchmark tests"]
726     # Buildbot default timeout without output for a step is 1200.
727     # The current maximum timeout for a benchmark plan is also 1200.
728     # So raise the buildbot timeout to avoid aborting this whole step when a test timeouts.
729     timeout = 1500
730     command = ["python", "./Tools/Scripts/run-benchmark", "--allplans"]
731
732     def start(self):
733         platform = self.getProperty("platform")
734         if platform == "gtk":
735             self.command += ["--browser", "minibrowser-gtk"]
736         self.setCommand(self.command)
737         return shell.Test.start(self)
738
739     def getText(self, cmd, results):
740         return self.getText2(cmd, results)
741
742     def getText2(self, cmd, results):
743         if results != SUCCESS:
744             return ["%d benchmark tests failed" % cmd.rc]
745         return [self.name]
746
747 class ArchiveTestResults(shell.ShellCommand):
748     command = ["python", "./Tools/BuildSlaveSupport/test-result-archive",
749                WithProperties("--platform=%(platform)s"), WithProperties("--%(configuration)s"), "archive"]
750     name = "archive-test-results"
751     description = ["archiving test results"]
752     descriptionDone = ["archived test results"]
753     haltOnFailure = True
754
755
756 class UploadTestResults(transfer.FileUpload):
757     slavesrc = "layout-test-results.zip"
758     masterdest = WithProperties("public_html/results/%(buildername)s/r%(got_revision)s (%(buildnumber)s).zip")
759
760     def __init__(self, **kwargs):
761         kwargs['slavesrc'] = self.slavesrc
762         kwargs['masterdest'] = self.masterdest
763         kwargs['mode'] = 0644
764         transfer.FileUpload.__init__(self, **kwargs)
765
766
767 class ExtractTestResults(master.MasterShellCommand):
768     zipFile = WithProperties("public_html/results/%(buildername)s/r%(got_revision)s (%(buildnumber)s).zip")
769     resultDirectory = WithProperties("public_html/results/%(buildername)s/r%(got_revision)s (%(buildnumber)s)")
770     descriptionDone = ["uploaded results"]
771
772     def __init__(self, **kwargs):
773         kwargs['command'] = ""
774         master.MasterShellCommand.__init__(self, **kwargs)
775
776     def resultDirectoryURL(self):
777         return self.build.getProperties().render(self.resultDirectory).replace("public_html/", "/") + "/"
778
779     def start(self):
780         self.command = ["unzip", self.build.getProperties().render(self.zipFile), "-d", self.build.getProperties().render(self.resultDirectory)]
781         return master.MasterShellCommand.start(self)
782
783     def addCustomURLs(self):
784         self.addURL("view layout test results", self.resultDirectoryURL() + "results.html")
785         self.addURL("view dashboard test results", self.resultDirectoryURL() + "dashboard-layout-test-results/results.html")
786
787     def finished(self, result):
788         self.addCustomURLs()
789         return master.MasterShellCommand.finished(self, result)
790
791
792 class ExtractTestResultsAndLeaks(ExtractTestResults):
793     def addCustomURLs(self):
794         ExtractTestResults.addCustomURLs(self)
795         url = "/LeaksViewer/?url=" + urllib.quote(self.resultDirectoryURL(), safe="")
796         self.addURL("view leaks", url)
797
798
799 class Factory(factory.BuildFactory):
800     def __init__(self, platform, configuration, architectures, buildOnly, additionalArguments, SVNMirror):
801         factory.BuildFactory.__init__(self)
802         self.addStep(ConfigureBuild(platform=platform, configuration=configuration, architecture=" ".join(architectures), buildOnly=buildOnly, additionalArguments=additionalArguments, SVNMirror=SVNMirror))
803         if SVNMirror:
804             self.addStep(WaitForSVNServer())
805         self.addStep(CheckOutSource(SVNMirror=SVNMirror))
806         if not (platform == "jsc-only"):
807             self.addStep(KillOldProcesses())
808         self.addStep(CleanBuildIfScheduled())
809         self.addStep(DeleteStaleBuildFiles())
810         if platform == "win":
811             self.addStep(InstallWin32Dependencies())
812         if platform == "gtk" and additionalArguments != ["--default-cmake-features"]:
813             self.addStep(InstallGtkDependencies())
814
815
816 class BuildFactory(Factory):
817     def __init__(self, platform, configuration, architectures, triggers=None, additionalArguments=None, SVNMirror=None):
818         Factory.__init__(self, platform, configuration, architectures, True, additionalArguments, SVNMirror)
819
820         if platform == "win":
821             self.addStep(CompileWebKit(timeout=2*60*60))
822         else:
823             self.addStep(CompileWebKit())
824
825         if triggers:
826             self.addStep(ArchiveBuiltProduct())
827             self.addStep(UploadBuiltProduct())
828             self.addStep(trigger.Trigger(schedulerNames=triggers))
829
830 def pickLatestBuild(builder, requests):
831     return max(requests, key=operator.attrgetter("submittedAt"))
832
833 class TestFactory(Factory):
834     JSCTestClass = RunJavaScriptCoreTests
835     LayoutTestClass = RunWebKitTests
836
837     def getProduct(self):
838         self.addStep(DownloadBuiltProduct())
839         self.addStep(ExtractBuiltProduct())
840
841     def __init__(self, platform, configuration, architectures, additionalArguments=None, SVNMirror=None, **kwargs):
842         Factory.__init__(self, platform, configuration, architectures, False, additionalArguments, SVNMirror, **kwargs)
843         self.getProduct()
844         if self.JSCTestClass:
845             self.addStep(self.JSCTestClass())
846         if self.LayoutTestClass:
847             self.addStep(self.LayoutTestClass())
848
849         if platform == 'win' or platform.startswith('mac') or platform.startswith('ios-simulator'):
850             self.addStep(RunUnitTests())
851         self.addStep(RunPythonTests())
852         self.addStep(RunPerlTests())
853         self.addStep(RunBindingsTests())
854         self.addStep(RunBuiltinsTests())
855         self.addStep(RunDashboardTests())
856         if self.LayoutTestClass:
857             self.addStep(ArchiveTestResults())
858             self.addStep(UploadTestResults())
859             self.addStep(ExtractTestResults())
860         if platform == "gtk":
861             self.addStep(RunGtkAPITests())
862
863 class BuildAndTestFactory(TestFactory):
864     def getProduct(self):
865         self.addStep(CompileWebKit())
866
867     def __init__(self, platform, configuration, architectures, triggers=None, additionalArguments=None, SVNMirror=None, **kwargs):
868         TestFactory.__init__(self, platform, configuration, architectures, additionalArguments, SVNMirror, **kwargs)
869         if triggers:
870             self.addStep(ArchiveBuiltProduct())
871             self.addStep(UploadBuiltProduct())
872             self.addStep(trigger.Trigger(schedulerNames=triggers))
873
874 class BuildAndTestLLINTCLoopFactory(Factory):
875     def __init__(self, platform, configuration, architectures, triggers=None, additionalArguments=None, SVNMirror=None, **kwargs):
876         Factory.__init__(self, platform, configuration, architectures, False, additionalArguments, SVNMirror, **kwargs)
877         self.addStep(CompileLLINTCLoop())
878         self.addStep(RunLLINTCLoopTests())
879
880 class BuildAndTest32bitJSCFactory(Factory):
881     def __init__(self, platform, configuration, architectures, triggers=None, additionalArguments=None, SVNMirror=None, **kwargs):
882         Factory.__init__(self, platform, configuration, architectures, False, additionalArguments, SVNMirror, **kwargs)
883         self.addStep(Compile32bitJSC())
884         self.addStep(Run32bitJSCTests())
885
886 class BuildAndNonLayoutTestFactory(BuildAndTestFactory):
887     LayoutTestClass = None
888
889 class BuildAndRemoteJSCTestsFactory(Factory):
890     def __init__(self, platform, configuration, architectures, triggers=None, additionalArguments=None, SVNMirror=None):
891         Factory.__init__(self, platform, configuration, architectures, False, additionalArguments, SVNMirror)
892         self.addStep(CompileJSCOnly())
893         self.addStep(RunRemoteJavaScriptCoreTests())
894
895 class TestWebKit1LeaksFactory(Factory):
896     def __init__(self, platform, configuration, architectures, additionalArguments=None, SVNMirror=None):
897         Factory.__init__(self, platform, configuration, architectures, False, additionalArguments, SVNMirror)
898         self.addStep(DownloadBuiltProduct())
899         self.addStep(ExtractBuiltProduct())
900         self.addStep(RunWebKit1LeakTests())
901         self.addStep(ArchiveTestResults())
902         self.addStep(UploadTestResults())
903         self.addStep(ExtractTestResultsAndLeaks())
904
905 class TestAllButJSCFactory(TestFactory):
906     JSCTestClass = None
907
908 class TestJSCFactory(Factory):
909     def __init__(self, platform, configuration, architectures, additionalArguments=None, SVNMirror=None):
910         Factory.__init__(self, platform, configuration, architectures, False, additionalArguments, SVNMirror)
911         self.addStep(DownloadBuiltProduct())
912         self.addStep(ExtractBuiltProduct())
913         self.addStep(RunJavaScriptCoreTests())
914
915 class Test262Factory(Factory):
916     def __init__(self, platform, configuration, architectures, additionalArguments=None, SVNMirror=None):
917         Factory.__init__(self, platform, configuration, architectures, False, additionalArguments, SVNMirror)
918         self.addStep(DownloadBuiltProduct())
919         self.addStep(ExtractBuiltProduct())
920         self.addStep(RunTest262Tests())
921
922 class TestWebKit1Factory(TestFactory):
923     LayoutTestClass = RunWebKit1Tests
924
925 class TestWebKit1AllButJSCFactory(TestWebKit1Factory):
926     JSCTestClass = None
927
928 class BuildAndPerfTestFactory(Factory):
929     def __init__(self, platform, configuration, architectures, additionalArguments=None, SVNMirror=None, **kwargs):
930         Factory.__init__(self, platform, configuration, architectures, False, additionalArguments, SVNMirror, **kwargs)
931         self.addStep(CompileWebKit())
932         self.addStep(RunAndUploadPerfTests())
933         if platform == "gtk":
934             self.addStep(RunBenchmarkTests())
935
936 class DownloadAndPerfTestFactory(Factory):
937     def __init__(self, platform, configuration, architectures, additionalArguments=None, SVNMirror=None, **kwargs):
938         Factory.__init__(self, platform, configuration, architectures, False, additionalArguments, SVNMirror, **kwargs)
939         self.addStep(DownloadBuiltProduct())
940         self.addStep(ExtractBuiltProduct())
941         self.addStep(RunAndUploadPerfTests())
942         if platform == "gtk":
943             self.addStep(RunBenchmarkTests())
944
945 class PlatformSpecificScheduler(AnyBranchScheduler):
946     def __init__(self, platform, branch, **kwargs):
947         self.platform = platform
948         filter = ChangeFilter(branch=[branch, None], filter_fn=self.filter)
949         AnyBranchScheduler.__init__(self, name=platform, change_filter=filter, **kwargs)
950
951     def filter(self, change):
952         return wkbuild.should_build(self.platform, change.files)
953
954 trunk_filter = ChangeFilter(branch=["trunk", None])
955
956 def loadBuilderConfig(c):
957     # FIXME: These file handles are leaked.
958     passwords = json.load(open('passwords.json'))
959     config = json.load(open('config.json'))
960
961     c['slaves'] = [BuildSlave(slave['name'], passwords[slave['name']], max_builds=1) for slave in config['slaves']]
962
963     c['schedulers'] = []
964     for scheduler in config['schedulers']:
965         if "change_filter" in scheduler:
966             scheduler["change_filter"] = globals()[scheduler["change_filter"]]
967         kls = globals()[scheduler.pop('type')]
968         # Python 2.6 can't handle unicode keys as keyword arguments:
969         # http://bugs.python.org/issue2646.  Modern versions of json return
970         # unicode strings from json.load, so we map all keys to str objects.
971         scheduler = dict(map(lambda key_value_pair: (str(key_value_pair[0]), key_value_pair[1]), scheduler.items()))
972
973         c['schedulers'].append(kls(**scheduler))
974
975     forceScheduler = ForceScheduler(
976         name="force",
977         builderNames=[str(builder['name']) for builder in config['builders']],
978         reason=StringParameter(name="reason", default="", size=40),
979
980         # Validate SVN revision: number or empty string
981         revision=StringParameter(name="revision", default="", regex=re.compile(r'^(\d*)$')),
982
983         # Disable default enabled input fields: branch, repository, project, additional properties
984         branch=FixedParameter(name="branch"),
985         repository=FixedParameter(name="repository"),
986         project=FixedParameter(name="project"),
987         properties=[BooleanParameter(name="is_clean", label="Force Clean build")]
988     )
989     c['schedulers'].append(forceScheduler)
990
991     c['builders'] = []
992     for builder in config['builders']:
993         for slaveName in builder['slavenames']:
994             for slave in config['slaves']:
995                 if slave['name'] != slaveName or slave['platform'] == '*':
996                     continue
997
998                 if slave['platform'] != builder['platform']:
999                     raise Exception, "Builder %r is for platform %r but has slave %r for platform %r!" % (builder['name'], builder['platform'], slave['name'], slave['platform'])
1000
1001                 break
1002
1003         platform = builder['platform']
1004
1005         builderType = builder.pop('type')
1006         factory = globals()["%sFactory" % builderType]
1007         factorykwargs = {}
1008         for key in "platform", "configuration", "architectures", "triggers", "additionalArguments", "SVNMirror":
1009             value = builder.pop(key, None)
1010             if value:
1011                 factorykwargs[key] = value
1012
1013         builder["factory"] = factory(**factorykwargs)
1014
1015         if platform.startswith('mac'):
1016             builder["category"] = 'AppleMac'
1017         elif platform.startswith('ios'):
1018             builder['category'] = 'iOS'
1019         elif platform == 'win':
1020             builder["category"] = 'AppleWin'
1021         elif platform.startswith('gtk'):
1022             builder["category"] = 'GTK'
1023         else:
1024             builder["category"] = 'misc'
1025
1026         if (builder['category'] in ('AppleMac', 'AppleWin', 'iOS')) and builderType != 'Build':
1027             builder['nextBuild'] = pickLatestBuild
1028
1029         c['builders'].append(builder)
1030
1031 loadBuilderConfig(c)