54bb1f0280748b1b55ccf57479fb41e5ebf746c2
[WebKit-https.git] / Tools / BuildSlaveSupport / build.webkit.org-config / steps.py
1 # Copyright (C) 2017 Apple Inc. All rights reserved.
2 #
3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions
5 # are met:
6 # 1.  Redistributions of source code must retain the above copyright
7 #     notice, this list of conditions and the following disclaimer.
8 # 2.  Redistributions in binary form must reproduce the above copyright
9 #     notice, this list of conditions and the following disclaimer in the
10 #     documentation and/or other materials provided with the distribution.
11 #
12 # THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND
13 # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
14 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
15 # DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR
16 # ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
17 # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
18 # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
19 # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
20 # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
21 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
22
23 from buildbot.process import buildstep, factory, properties
24 from buildbot.steps import master, shell, source, transfer, trigger
25 from buildbot.status.builder import SUCCESS, FAILURE, WARNINGS, SKIPPED, EXCEPTION
26
27 from twisted.internet import defer
28
29 import os
30 import re
31 import json
32 import cStringIO
33 import urllib
34
35 APPLE_WEBKIT_AWS_PROXY = "http://proxy01.webkit.org:3128"
36 S3URL = "https://s3-us-west-2.amazonaws.com/"
37 WithProperties = properties.WithProperties
38
39
40 class TestWithFailureCount(shell.Test):
41     failedTestsFormatString = "%d test%s failed"
42
43     def countFailures(self, cmd):
44         return 0
45
46     def commandComplete(self, cmd):
47         shell.Test.commandComplete(self, cmd)
48         self.failedTestCount = self.countFailures(cmd)
49         self.failedTestPluralSuffix = "" if self.failedTestCount == 1 else "s"
50
51     def evaluateCommand(self, cmd):
52         if self.failedTestCount:
53             return FAILURE
54
55         if cmd.rc != 0:
56             return FAILURE
57
58         return SUCCESS
59
60     def getText(self, cmd, results):
61         return self.getText2(cmd, results)
62
63     def getText2(self, cmd, results):
64         if results != SUCCESS and self.failedTestCount:
65             return [self.failedTestsFormatString % (self.failedTestCount, self.failedTestPluralSuffix)]
66
67         return [self.name]
68
69
70 class ConfigureBuild(buildstep.BuildStep):
71     name = "configure build"
72     description = ["configuring build"]
73     descriptionDone = ["configured build"]
74
75     def __init__(self, platform, configuration, architecture, buildOnly, additionalArguments, SVNMirror, *args, **kwargs):
76         buildstep.BuildStep.__init__(self, *args, **kwargs)
77         self.platform = platform
78         if platform != 'jsc-only':
79             self.platform = platform.split('-', 1)[0]
80         self.fullPlatform = platform
81         self.configuration = configuration
82         self.architecture = architecture
83         self.buildOnly = buildOnly
84         self.additionalArguments = additionalArguments
85         self.SVNMirror = SVNMirror
86         self.addFactoryArguments(platform=platform, configuration=configuration, architecture=architecture, buildOnly=buildOnly, additionalArguments=additionalArguments, SVNMirror=SVNMirror)
87
88     def start(self):
89         self.setProperty("platform", self.platform)
90         self.setProperty("fullPlatform", self.fullPlatform)
91         self.setProperty("configuration", self.configuration)
92         self.setProperty("architecture", self.architecture)
93         self.setProperty("buildOnly", self.buildOnly)
94         self.setProperty("additionalArguments", self.additionalArguments)
95         self.setProperty("SVNMirror", self.SVNMirror)
96         self.finished(SUCCESS)
97         return defer.succeed(None)
98
99
100 class CheckOutSource(source.SVN):
101     mode = "update"
102
103     def __init__(self, SVNMirror, **kwargs):
104         kwargs['baseURL'] = SVNMirror or "https://svn.webkit.org/repository/webkit/"
105         kwargs['defaultBranch'] = "trunk"
106         kwargs['mode'] = self.mode
107         source.SVN.__init__(self, **kwargs)
108         self.addFactoryArguments(SVNMirror=SVNMirror)
109
110
111 class WaitForSVNServer(shell.ShellCommand):
112     name = "wait-for-svn-server"
113     command = ["python", "./Tools/BuildSlaveSupport/wait-for-SVN-server.py", "-r", WithProperties("%(revision)s"), "-s", WithProperties("%(SVNMirror)s")]
114     description = ["waiting for SVN server"]
115     descriptionDone = ["SVN server is ready"]
116     warnOnFailure = True
117
118     def evaluateCommand(self, cmd):
119         if cmd.rc != 0:
120             return WARNINGS
121         return SUCCESS
122
123
124 class InstallWin32Dependencies(shell.Compile):
125     description = ["installing dependencies"]
126     descriptionDone = ["installed dependencies"]
127     command = ["perl", "./Tools/Scripts/update-webkit-auxiliary-libs"]
128
129
130 class KillOldProcesses(shell.Compile):
131     name = "kill old processes"
132     description = ["killing old processes"]
133     descriptionDone = ["killed old processes"]
134     command = ["python", "./Tools/BuildSlaveSupport/kill-old-processes", "buildbot"]
135
136
137 class CleanBuildIfScheduled(shell.Compile):
138     name = "delete WebKitBuild directory"
139     description = ["deleting WebKitBuild directory"]
140     descriptionDone = ["deleted WebKitBuild directory"]
141     command = ["python", "./Tools/BuildSlaveSupport/clean-build", WithProperties("--platform=%(fullPlatform)s"), WithProperties("--%(configuration)s")]
142
143     def start(self):
144         if not self.getProperty('is_clean'):
145             self.hideStepIf = True
146             return SKIPPED
147         return shell.Compile.start(self)
148
149
150 class DeleteStaleBuildFiles(shell.Compile):
151     name = "delete stale build files"
152     description = ["deleting stale build files"]
153     descriptionDone = ["deleted stale build files"]
154     command = ["python", "./Tools/BuildSlaveSupport/delete-stale-build-files", WithProperties("--platform=%(fullPlatform)s"), WithProperties("--%(configuration)s")]
155
156     def start(self):
157         if self.getProperty('is_clean'):  # Nothing to be done if WebKitBuild had been removed.
158             self.hideStepIf = True
159             return SKIPPED
160         return shell.Compile.start(self)
161
162
163 class InstallWinCairoDependencies(shell.ShellCommand):
164     name = 'wincairo-requirements'
165     description = ['updating wincairo dependencies']
166     descriptionDone = ['updated wincairo dependencies']
167     command = ['python', './Tools/Scripts/update-webkit-wincairo-libs.py']
168     haltOnFailure = True
169
170
171 class InstallGtkDependencies(shell.ShellCommand):
172     name = "jhbuild"
173     description = ["updating gtk dependencies"]
174     descriptionDone = ["updated gtk dependencies"]
175     command = ["perl", "./Tools/Scripts/update-webkitgtk-libs"]
176     haltOnFailure = True
177
178
179 class InstallWpeDependencies(shell.ShellCommand):
180     name = "jhbuild"
181     description = ["updating wpe dependencies"]
182     descriptionDone = ["updated wpe dependencies"]
183     command = ["perl", "./Tools/Scripts/update-webkitwpe-libs"]
184     haltOnFailure = True
185
186
187 def appendCustomBuildFlags(step, platform, fullPlatform):
188     if platform not in ('gtk', 'wincairo', 'ios', 'jsc-only', 'wpe'):
189         return
190     if fullPlatform.startswith('ios-simulator'):
191         platform = 'ios-simulator'
192     elif platform == 'ios':
193         platform = 'device'
194     step.setCommand(step.command + ['--' + platform])
195
196
197 class CompileWebKit(shell.Compile):
198     command = ["perl", "./Tools/Scripts/build-webkit", WithProperties("--%(configuration)s")]
199     env = {'MFLAGS': ''}
200     name = "compile-webkit"
201     description = ["compiling"]
202     descriptionDone = ["compiled"]
203     warningPattern = ".*arning: .*"
204
205     def start(self):
206         platform = self.getProperty('platform')
207         buildOnly = self.getProperty('buildOnly')
208         architecture = self.getProperty('architecture')
209         additionalArguments = self.getProperty('additionalArguments')
210
211         if additionalArguments:
212             self.setCommand(self.command + additionalArguments)
213         if platform in ('mac', 'ios') and architecture:
214             self.setCommand(self.command + ['ARCHS=' + architecture])
215             if platform == 'ios':
216                 self.setCommand(self.command + ['ONLY_ACTIVE_ARCH=NO'])
217         if platform in ('mac', 'ios') and buildOnly:
218             # For build-only bots, the expectation is that tests will be run on separate machines,
219             # so we need to package debug info as dSYMs. Only generating line tables makes
220             # this much faster than full debug info, and crash logs still have line numbers.
221             self.setCommand(self.command + ['DEBUG_INFORMATION_FORMAT=dwarf-with-dsym'])
222             self.setCommand(self.command + ['CLANG_DEBUG_INFORMATION_LEVEL=line-tables-only'])
223
224         appendCustomBuildFlags(self, platform, self.getProperty('fullPlatform'))
225
226         return shell.Compile.start(self)
227
228     def createSummary(self, log):
229         platform = self.getProperty('platform')
230         if platform.startswith('mac'):
231             warnings = []
232             errors = []
233             sio = cStringIO.StringIO(log.getText())
234             for line in sio.readlines():
235                 if "arning:" in line:
236                     warnings.append(line)
237                 if "rror:" in line:
238                     errors.append(line)
239             if warnings:
240                 self.addCompleteLog('warnings', "".join(warnings))
241             if errors:
242                 self.addCompleteLog('errors', "".join(errors))
243
244
245 class CompileLLINTCLoop(CompileWebKit):
246     command = ["perl", "./Tools/Scripts/build-jsc", "--cloop", WithProperties("--%(configuration)s")]
247
248
249 class Compile32bitJSC(CompileWebKit):
250     command = ["perl", "./Tools/Scripts/build-jsc", "--32-bit", WithProperties("--%(configuration)s")]
251
252
253 class CompileJSCOnly(CompileWebKit):
254     command = ["perl", "./Tools/Scripts/build-jsc", WithProperties("--%(configuration)s")]
255
256
257 class ArchiveBuiltProduct(shell.ShellCommand):
258     command = ["python", "./Tools/BuildSlaveSupport/built-product-archive",
259                WithProperties("--platform=%(fullPlatform)s"), WithProperties("--%(configuration)s"), "archive"]
260     name = "archive-built-product"
261     description = ["archiving built product"]
262     descriptionDone = ["archived built product"]
263     haltOnFailure = True
264
265
266 class ArchiveMinifiedBuiltProduct(ArchiveBuiltProduct):
267     command = ["python", "./Tools/BuildSlaveSupport/built-product-archive",
268                WithProperties("--platform=%(fullPlatform)s"), WithProperties("--%(configuration)s"), "archive", "--minify"]
269
270
271 class GenerateJSCBundle(shell.ShellCommand):
272     command = ["python", "./Tools/Scripts/generate-jsc-bundle", "--builder-name", WithProperties("%(buildername)s"),
273                WithProperties("--platform=%(fullPlatform)s"), WithProperties("--%(configuration)s"),
274                WithProperties("--revision=%(got_revision)s"), "--remote-config-file", "../../remote-jsc-bundle-upload-config.json"]
275     name = "generate-jsc-bundle"
276     description = ["generating jsc bundle"]
277     descriptionDone = ["generated jsc bundle"]
278     haltOnFailure = False
279
280 class ExtractBuiltProduct(shell.ShellCommand):
281     command = ["python", "./Tools/BuildSlaveSupport/built-product-archive",
282                WithProperties("--platform=%(fullPlatform)s"), WithProperties("--%(configuration)s"), "extract"]
283     name = "extract-built-product"
284     description = ["extracting built product"]
285     descriptionDone = ["extracted built product"]
286     haltOnFailure = True
287
288
289 class UploadBuiltProduct(transfer.FileUpload):
290     slavesrc = WithProperties("WebKitBuild/%(configuration)s.zip")
291     masterdest = WithProperties("archives/%(fullPlatform)s-%(architecture)s-%(configuration)s/%(got_revision)s.zip")
292     haltOnFailure = True
293
294     def __init__(self, **kwargs):
295         kwargs['slavesrc'] = self.slavesrc
296         kwargs['masterdest'] = self.masterdest
297         kwargs['mode'] = 0644
298         kwargs['blocksize'] = 1024 * 256
299         transfer.FileUpload.__init__(self, **kwargs)
300
301
302 class UploadMinifiedBuiltProduct(UploadBuiltProduct):
303     slavesrc = WithProperties("WebKitBuild/minified-%(configuration)s.zip")
304     masterdest = WithProperties("archives/%(fullPlatform)s-%(architecture)s-%(configuration)s/minified-%(got_revision)s.zip")
305
306
307 class DownloadBuiltProduct(shell.ShellCommand):
308     command = ["python", "./Tools/BuildSlaveSupport/download-built-product",
309         WithProperties("--platform=%(platform)s"), WithProperties("--%(configuration)s"),
310         WithProperties(S3URL + "archives.webkit.org/%(fullPlatform)s-%(architecture)s-%(configuration)s/%(got_revision)s.zip")]
311     name = "download-built-product"
312     description = ["downloading built product"]
313     descriptionDone = ["downloaded built product"]
314     haltOnFailure = True
315     flunkOnFailure = True
316
317     def start(self):
318         if 'apple' in self.getProperty('buildername').lower():
319             self.slaveEnvironment['HTTPS_PROXY'] = APPLE_WEBKIT_AWS_PROXY  # curl env var to use a proxy
320         return shell.ShellCommand.start(self)
321
322
323 class RunJavaScriptCoreTests(TestWithFailureCount):
324     name = "jscore-test"
325     description = ["jscore-tests running"]
326     descriptionDone = ["jscore-tests"]
327     jsonFileName = "jsc_results.json"
328     command = ["perl", "./Tools/Scripts/run-javascriptcore-tests", "--no-build", "--no-fail-fast", "--json-output={0}".format(jsonFileName), WithProperties("--%(configuration)s")]
329     failedTestsFormatString = "%d JSC test%s failed"
330     logfiles = {"json": jsonFileName}
331
332     def start(self):
333         platform = self.getProperty('platform')
334         # Linux bots have currently problems with JSC tests that try to use large amounts of memory.
335         # Check: https://bugs.webkit.org/show_bug.cgi?id=175140
336         if platform in ('gtk', 'wpe'):
337             self.setCommand(self.command + ['--memory-limited'])
338         # WinCairo uses the Windows command prompt, not Cygwin.
339         elif platform == 'wincairo':
340             self.setCommand(self.command + ['--test-writer=ruby'])
341
342         appendCustomBuildFlags(self, platform, self.getProperty('fullPlatform'))
343         return shell.Test.start(self)
344
345     def countFailures(self, cmd):
346         logText = cmd.logs['stdio'].getText()
347
348         match = re.search(r'^Results for JSC stress tests:\r?\n\s+(\d+) failure', logText, re.MULTILINE)
349         if match:
350             return int(match.group(1))
351
352         match = re.search(r'^Results for Mozilla tests:\r?\n\s+(\d+) regression', logText, re.MULTILINE)
353         if match:
354             return int(match.group(1))
355
356         return 0
357
358
359 class RunRemoteJavaScriptCoreTests(RunJavaScriptCoreTests):
360     def start(self):
361         self.setCommand(self.command + ["--memory-limited", "--remote-config-file", "../../remote-jsc-tests-config.json"])
362         return RunJavaScriptCoreTests.start(self)
363
364
365 class RunTest262Tests(TestWithFailureCount):
366     name = "test262-test"
367     description = ["test262-tests running"]
368     descriptionDone = ["test262-tests"]
369     failedTestsFormatString = "%d Test262 test%s failed"
370     command = ["perl", "./Tools/Scripts/test262-runner", "--verbose", WithProperties("--%(configuration)s")]
371
372     def start(self):
373         appendCustomBuildFlags(self, self.getProperty('platform'), self.getProperty('fullPlatform'))
374         return shell.Test.start(self)
375
376     def countFailures(self, cmd):
377         logText = cmd.logs['stdio'].getText()
378         matches = re.findall(r'^\! NEW FAIL', logText, flags=re.MULTILINE)
379         if matches:
380             return len(matches)
381         return 0
382
383
384 class RunWebKitTests(shell.Test):
385     name = "layout-test"
386     description = ["layout-tests running"]
387     descriptionDone = ["layout-tests"]
388     resultDirectory = "layout-test-results"
389     command = ["python", "./Tools/Scripts/run-webkit-tests",
390                "--no-build",
391                "--no-show-results",
392                "--no-new-test-results",
393                "--builder-name", WithProperties("%(buildername)s"),
394                "--build-number", WithProperties("%(buildnumber)s"),
395                "--master-name", "webkit.org",
396                "--test-results-server", "webkit-test-results.webkit.org",
397                "--exit-after-n-crashes-or-timeouts", "50",
398                "--exit-after-n-failures", "500",
399                WithProperties("--%(configuration)s")]
400
401     def start(self):
402         platform = self.getProperty('platform')
403         appendCustomBuildFlags(self, platform, self.getProperty('fullPlatform'))
404         additionalArguments = self.getProperty('additionalArguments')
405
406         self.setCommand(self.command + ["--results-directory", self.resultDirectory])
407         self.setCommand(self.command + ['--debug-rwt-logging'])
408
409         if platform == "win":
410             self.setCommand(self.command + ['--batch-size', '100', '--root=' + os.path.join("WebKitBuild", self.getProperty('configuration'), "bin32")])
411
412         if additionalArguments:
413             self.setCommand(self.command + additionalArguments)
414         return shell.Test.start(self)
415
416     # FIXME: This will break if run-webkit-tests changes its default log formatter.
417     nrwt_log_message_regexp = re.compile(r'\d{2}:\d{2}:\d{2}(\.\d+)?\s+\d+\s+(?P<message>.*)')
418
419     def _strip_python_logging_prefix(self, line):
420         match_object = self.nrwt_log_message_regexp.match(line)
421         if match_object:
422             return match_object.group('message')
423         return line
424
425     def _parseRunWebKitTestsOutput(self, logText):
426         incorrectLayoutLines = []
427         expressions = [
428             ('flakes', re.compile(r'Unexpected flakiness.+\((\d+)\)')),
429             ('new passes', re.compile(r'Expected to .+, but passed:\s+\((\d+)\)')),
430             ('missing results', re.compile(r'Regressions: Unexpected missing results\s+\((\d+)\)')),
431             ('failures', re.compile(r'Regressions: Unexpected.+\((\d+)\)')),
432         ]
433         testFailures = {}
434
435         for line in logText.splitlines():
436             if line.find('Exiting early') >= 0 or line.find('leaks found') >= 0:
437                 incorrectLayoutLines.append(self._strip_python_logging_prefix(line))
438                 continue
439             for name, expression in expressions:
440                 match = expression.search(line)
441
442                 if match:
443                     testFailures[name] = testFailures.get(name, 0) + int(match.group(1))
444                     break
445
446                 # FIXME: Parse file names and put them in results
447
448         for name in testFailures:
449             incorrectLayoutLines.append(str(testFailures[name]) + ' ' + name)
450
451         self.incorrectLayoutLines = incorrectLayoutLines
452
453     def commandComplete(self, cmd):
454         shell.Test.commandComplete(self, cmd)
455
456         logText = cmd.logs['stdio'].getText()
457         self._parseRunWebKitTestsOutput(logText)
458
459     def evaluateCommand(self, cmd):
460         result = SUCCESS
461
462         if self.incorrectLayoutLines:
463             if len(self.incorrectLayoutLines) == 1:
464                 line = self.incorrectLayoutLines[0]
465                 if line.find('were new') >= 0 or line.find('was new') >= 0 or line.find(' leak') >= 0:
466                     return WARNINGS
467
468             for line in self.incorrectLayoutLines:
469                 if line.find('flakes') >= 0 or line.find('new passes') >= 0 or line.find('missing results') >= 0:
470                     result = WARNINGS
471                 else:
472                     return FAILURE
473
474         # Return code from Tools/Scripts/layout_tests/run_webkit_tests.py.
475         # This means that an exception was raised when running run-webkit-tests and
476         # was never handled.
477         if cmd.rc == 254:
478             return EXCEPTION
479         if cmd.rc != 0:
480             return FAILURE
481
482         return result
483
484     def getText(self, cmd, results):
485         return self.getText2(cmd, results)
486
487     def getText2(self, cmd, results):
488         if results != SUCCESS and self.incorrectLayoutLines:
489             return self.incorrectLayoutLines
490
491         return [self.name]
492
493
494 class RunDashboardTests(RunWebKitTests):
495     name = "dashboard-tests"
496     description = ["dashboard-tests running"]
497     descriptionDone = ["dashboard-tests"]
498     resultDirectory = os.path.join(RunWebKitTests.resultDirectory, "dashboard-layout-test-results")
499
500     def start(self):
501         self.setCommand(self.command + ["--layout-tests-directory", "./Tools/BuildSlaveSupport/build.webkit.org-config/public_html/dashboard/Scripts/tests"])
502         return RunWebKitTests.start(self)
503
504
505 class RunUnitTests(TestWithFailureCount):
506     name = "run-api-tests"
507     description = ["unit tests running"]
508     descriptionDone = ["unit-tests"]
509     command = ["python", "./Tools/Scripts/run-api-tests", "--no-build", WithProperties("--%(configuration)s"), "--verbose"]
510     failedTestsFormatString = "%d unit test%s failed or timed out"
511
512     def start(self):
513         appendCustomBuildFlags(self, self.getProperty('platform'), self.getProperty('fullPlatform'))
514         return shell.Test.start(self)
515
516     def countFailures(self, cmd):
517         log_text = cmd.logs['stdio'].getText()
518
519         match = re.search(r'Ran (?P<ran>\d+) tests of (?P<total>\d+) with (?P<passed>\d+) successful', log_text)
520         if not match:
521             return -1
522         return int(match.group('ran')) - int(match.group('passed'))
523
524
525 class RunPythonTests(TestWithFailureCount):
526     name = "webkitpy-test"
527     description = ["python-tests running"]
528     descriptionDone = ["python-tests"]
529     command = ["python", "./Tools/Scripts/test-webkitpy", "--verbose", WithProperties("--%(configuration)s")]
530     failedTestsFormatString = "%d python test%s failed"
531
532     def start(self):
533         platform = self.getProperty('platform')
534         # Python tests are flaky on the GTK builders, running them serially
535         # helps and does not significantly prolong the cycle time.
536         if platform == 'gtk':
537             self.setCommand(self.command + ['--child-processes', '1'])
538         # Python tests fail on windows bots when running more than one child process
539         # https://bugs.webkit.org/show_bug.cgi?id=97465
540         if platform == 'win':
541             self.setCommand(self.command + ['--child-processes', '1'])
542         return shell.Test.start(self)
543
544     def countFailures(self, cmd):
545         logText = cmd.logs['stdio'].getText()
546         # We're looking for the line that looks like this: FAILED (failures=2, errors=1)
547         regex = re.compile(r'^FAILED \((?P<counts>[^)]+)\)')
548         for line in logText.splitlines():
549             match = regex.match(line)
550             if not match:
551                 continue
552             return sum(int(component.split('=')[1]) for component in match.group('counts').split(', '))
553         return 0
554
555
556 class RunPerlTests(TestWithFailureCount):
557     name = "webkitperl-test"
558     description = ["perl-tests running"]
559     descriptionDone = ["perl-tests"]
560     command = ["perl", "./Tools/Scripts/test-webkitperl"]
561     failedTestsFormatString = "%d perl test%s failed"
562
563     def countFailures(self, cmd):
564         logText = cmd.logs['stdio'].getText()
565         # We're looking for the line that looks like this: Failed 2/19 test programs. 5/363 subtests failed.
566         regex = re.compile(r'^Failed \d+/\d+ test programs\. (?P<count>\d+)/\d+ subtests failed\.')
567         for line in logText.splitlines():
568             match = regex.match(line)
569             if not match:
570                 continue
571             return int(match.group('count'))
572         return 0
573
574
575 class RunLLINTCLoopTests(TestWithFailureCount):
576     name = "webkit-jsc-cloop-test"
577     description = ["cloop-tests running"]
578     descriptionDone = ["cloop-tests"]
579     jsonFileName = "jsc_cloop.json"
580     command = ["perl", "./Tools/Scripts/run-javascriptcore-tests", "--cloop", "--no-build", "--no-jsc-stress", "--no-fail-fast", "--json-output={0}".format(jsonFileName), WithProperties("--%(configuration)s")]
581     failedTestsFormatString = "%d regression%s found."
582     logfiles = {"json": jsonFileName}
583
584     def countFailures(self, cmd):
585         logText = cmd.logs['stdio'].getText()
586         # We're looking for the line that looks like this: 0 regressions found.
587         regex = re.compile(r'\s*(?P<count>\d+) regressions? found.')
588         for line in logText.splitlines():
589             match = regex.match(line)
590             if not match:
591                 continue
592             return int(match.group('count'))
593         return 0
594
595
596 class Run32bitJSCTests(TestWithFailureCount):
597     name = "webkit-32bit-jsc-test"
598     description = ["32bit-jsc-tests running"]
599     descriptionDone = ["32bit-jsc-tests"]
600     jsonFileName = "jsc_32bit.json"
601     command = ["perl", "./Tools/Scripts/run-javascriptcore-tests", "--32-bit", "--no-build", "--no-fail-fast", "--json-output={0}".format(jsonFileName), WithProperties("--%(configuration)s")]
602     failedTestsFormatString = "%d regression%s found."
603     logfiles = {"json": jsonFileName}
604
605     def countFailures(self, cmd):
606         logText = cmd.logs['stdio'].getText()
607         # We're looking for the line that looks like this: 0 failures found.
608         regex = re.compile(r'\s*(?P<count>\d+) failures? found.')
609         for line in logText.splitlines():
610             match = regex.match(line)
611             if not match:
612                 continue
613             return int(match.group('count'))
614         return 0
615
616
617 class RunBindingsTests(shell.Test):
618     name = "bindings-generation-tests"
619     description = ["bindings-tests running"]
620     descriptionDone = ["bindings-tests"]
621     command = ["python", "./Tools/Scripts/run-bindings-tests"]
622
623
624 class RunBuiltinsTests(shell.Test):
625     name = "builtins-generator-tests"
626     description = ["builtins-generator-tests running"]
627     descriptionDone = ["builtins-generator-tests"]
628     command = ["python", "./Tools/Scripts/run-builtins-generator-tests"]
629
630
631 class RunGLibAPITests(shell.Test):
632     name = "API tests"
633     description = ["API tests running"]
634     descriptionDone = ["API tests"]
635
636     def start(self):
637         additionalArguments = self.getProperty("additionalArguments")
638         if additionalArguments:
639             self.command += additionalArguments
640         self.setCommand(self.command)
641         return shell.Test.start(self)
642
643     def commandComplete(self, cmd):
644         shell.Test.commandComplete(self, cmd)
645
646         logText = cmd.logs['stdio'].getText()
647
648         failedTests = 0
649         crashedTests = 0
650         timedOutTests = 0
651         messages = []
652         self.statusLine = []
653
654         foundItems = re.findall("Unexpected failures \((\d+)\)", logText)
655         if foundItems:
656             failedTests = int(foundItems[0])
657             messages.append("%d failures" % failedTests)
658
659         foundItems = re.findall("Unexpected crashes \((\d+)\)", logText)
660         if foundItems:
661             crashedTests = int(foundItems[0])
662             messages.append("%d crashes" % crashedTests)
663
664         foundItems = re.findall("Unexpected timeouts \((\d+)\)", logText)
665         if foundItems:
666             timedOutTests = int(foundItems[0])
667             messages.append("%d timeouts" % timedOutTests)
668
669         foundItems = re.findall("Unexpected passes \((\d+)\)", logText)
670         if foundItems:
671             newPassTests = int(foundItems[0])
672             messages.append("%d new passes" % newPassTests)
673
674         self.totalFailedTests = failedTests + crashedTests + timedOutTests
675         if messages:
676             self.statusLine = ["API tests: %s" % ", ".join(messages)]
677
678     def evaluateCommand(self, cmd):
679         if self.totalFailedTests > 0:
680             return FAILURE
681
682         if cmd.rc != 0:
683             return FAILURE
684
685         return SUCCESS
686
687     def getText(self, cmd, results):
688         return self.getText2(cmd, results)
689
690     def getText2(self, cmd, results):
691         if results != SUCCESS and self.totalFailedTests > 0:
692             return self.statusLine
693
694         return [self.name]
695
696
697 class RunGtkAPITests(RunGLibAPITests):
698     command = ["python", "./Tools/Scripts/run-gtk-tests", WithProperties("--%(configuration)s")]
699
700
701 class RunWPEAPITests(RunGLibAPITests):
702     command = ["python", "./Tools/Scripts/run-wpe-tests", WithProperties("--%(configuration)s")]
703
704
705 class RunWebDriverTests(shell.Test):
706     name = "webdriver-test"
707     description = ["webdriver-tests running"]
708     descriptionDone = ["webdriver-tests"]
709     jsonFileName = "webdriver_tests.json"
710     command = ["python", "./Tools/Scripts/run-webdriver-tests", "--json-output={0}".format(jsonFileName), WithProperties("--%(configuration)s")]
711     logfiles = {"json": jsonFileName}
712
713     def start(self):
714         additionalArguments = self.getProperty('additionalArguments')
715         if additionalArguments:
716             self.setCommand(self.command + additionalArguments)
717
718         appendCustomBuildFlags(self, self.getProperty('platform'), self.getProperty('fullPlatform'))
719         return shell.Test.start(self)
720
721     def commandComplete(self, cmd):
722         shell.Test.commandComplete(self, cmd)
723         logText = cmd.logs['stdio'].getText()
724
725         self.failuresCount = 0
726         self.newPassesCount = 0
727         foundItems = re.findall("^Unexpected .+ \((\d+)\)", logText, re.MULTILINE)
728         if foundItems:
729             self.failuresCount = int(foundItems[0])
730         foundItems = re.findall("^Expected to .+, but passed \((\d+)\)", logText, re.MULTILINE)
731         if foundItems:
732             self.newPassesCount = int(foundItems[0])
733
734     def evaluateCommand(self, cmd):
735         if self.failuresCount:
736             return FAILURE
737
738         if self.newPassesCount:
739             return WARNINGS
740
741         if cmd.rc != 0:
742             return FAILURE
743
744         return SUCCESS
745
746     def getText(self, cmd, results):
747         return self.getText2(cmd, results)
748
749     def getText2(self, cmd, results):
750         if results != SUCCESS and (self.failuresCount or self.newPassesCount):
751             lines = []
752             if self.failuresCount:
753                 lines.append("%d failures" % self.failuresCount)
754             if self.newPassesCount:
755                 lines.append("%d new passes" % self.newPassesCount)
756             return ["%s %s" % (self.name, ", ".join(lines))]
757
758         return [self.name]
759
760
761 class RunWebKit1Tests(RunWebKitTests):
762     def start(self):
763         self.setCommand(self.command + ["--dump-render-tree"])
764
765         return RunWebKitTests.start(self)
766
767
768 class RunWebKit1LeakTests(RunWebKit1Tests):
769     want_stdout = False
770     want_stderr = False
771     warnOnWarnings = True
772
773     def start(self):
774         self.setCommand(self.command + ["--leaks"])
775         return RunWebKit1Tests.start(self)
776
777
778 class RunAndUploadPerfTests(shell.Test):
779     name = "perf-test"
780     description = ["perf-tests running"]
781     descriptionDone = ["perf-tests"]
782     command = ["python", "./Tools/Scripts/run-perf-tests",
783                "--output-json-path", "perf-test-results.json",
784                "--slave-config-json-path", "../../perf-test-config.json",
785                "--no-show-results",
786                "--reset-results",
787                "--test-results-server", "perf.webkit.org",
788                "--builder-name", WithProperties("%(buildername)s"),
789                "--build-number", WithProperties("%(buildnumber)s"),
790                "--platform", WithProperties("%(fullPlatform)s"),
791                "--no-build",
792                WithProperties("--%(configuration)s")]
793
794     def start(self):
795         additionalArguments = self.getProperty("additionalArguments")
796         if additionalArguments:
797             self.command += additionalArguments
798         self.setCommand(self.command)
799         return shell.Test.start(self)
800
801     def getText(self, cmd, results):
802         return self.getText2(cmd, results)
803
804     def getText2(self, cmd, results):
805         if results != SUCCESS:
806             if cmd.rc == -1 & 0xff:
807                 return ["build not up to date"]
808             elif cmd.rc == -2 & 0xff:
809                 return ["slave config JSON error"]
810             elif cmd.rc == -3 & 0xff:
811                 return ["output JSON merge error"]
812             elif cmd.rc == -4 & 0xff:
813                 return ["upload error"]
814             elif cmd.rc == -5 & 0xff:
815                 return ["system dependency error"]
816             elif cmd.rc == -1:
817                 return ["timeout"]
818             else:
819                 return ["%d perf tests failed" % cmd.rc]
820
821         return [self.name]
822
823
824 class RunBenchmarkTests(shell.Test):
825     name = "benchmark-test"
826     description = ["benchmark tests running"]
827     descriptionDone = ["benchmark tests"]
828     command = ["python", "./Tools/Scripts/browserperfdash-benchmark", "--allplans",
829                "--config-file", "../../browserperfdash-benchmark-config.txt",
830                "--browser-version", WithProperties("r%(got_revision)s")]
831
832     def start(self):
833         platform = self.getProperty("platform")
834         if platform == "gtk":
835             self.command += ["--browser", "minibrowser-gtk"]
836         self.setCommand(self.command)
837         return shell.Test.start(self)
838
839     def getText(self, cmd, results):
840         return self.getText2(cmd, results)
841
842     def getText2(self, cmd, results):
843         if results != SUCCESS:
844             return ["%d benchmark tests failed" % cmd.rc]
845         return [self.name]
846
847
848 class ArchiveTestResults(shell.ShellCommand):
849     command = ["python", "./Tools/BuildSlaveSupport/test-result-archive",
850                WithProperties("--platform=%(platform)s"), WithProperties("--%(configuration)s"), "archive"]
851     name = "archive-test-results"
852     description = ["archiving test results"]
853     descriptionDone = ["archived test results"]
854     haltOnFailure = True
855
856
857 class UploadTestResults(transfer.FileUpload):
858     slavesrc = "layout-test-results.zip"
859     masterdest = WithProperties("public_html/results/%(buildername)s/r%(got_revision)s (%(buildnumber)s).zip")
860
861     def __init__(self, **kwargs):
862         kwargs['slavesrc'] = self.slavesrc
863         kwargs['masterdest'] = self.masterdest
864         kwargs['mode'] = 0644
865         transfer.FileUpload.__init__(self, **kwargs)
866
867
868 class TransferToS3(master.MasterShellCommand):
869     name = "transfer-to-s3"
870     description = ["transferring to s3"]
871     descriptionDone = ["transferred to s3"]
872     archive = WithProperties("archives/%(fullPlatform)s-%(architecture)s-%(configuration)s/%(got_revision)s.zip")
873     minifiedArchive = WithProperties("archives/%(fullPlatform)s-%(architecture)s-%(configuration)s/minified-%(got_revision)s.zip")
874     identifier = WithProperties("%(fullPlatform)s-%(architecture)s-%(configuration)s")
875     revision = WithProperties("%(got_revision)s")
876     command = ["python", "./transfer-archive-to-s3", "--revision", revision, "--identifier", identifier, "--archive", archive]
877     haltOnFailure = True
878
879     def __init__(self, **kwargs):
880         kwargs['command'] = self.command
881         master.MasterShellCommand.__init__(self, **kwargs)
882
883     def start(self):
884         return master.MasterShellCommand.start(self)
885
886     def finished(self, result):
887         return master.MasterShellCommand.finished(self, result)
888
889
890 class ExtractTestResults(master.MasterShellCommand):
891     zipFile = WithProperties("public_html/results/%(buildername)s/r%(got_revision)s (%(buildnumber)s).zip")
892     resultDirectory = WithProperties("public_html/results/%(buildername)s/r%(got_revision)s (%(buildnumber)s)")
893     descriptionDone = ["uploaded results"]
894
895     def __init__(self, **kwargs):
896         kwargs['command'] = ""
897         master.MasterShellCommand.__init__(self, **kwargs)
898
899     def resultDirectoryURL(self):
900         return self.build.getProperties().render(self.resultDirectory).replace("public_html/", "/") + "/"
901
902     def start(self):
903         self.command = ["unzip", self.build.getProperties().render(self.zipFile), "-d", self.build.getProperties().render(self.resultDirectory)]
904         return master.MasterShellCommand.start(self)
905
906     def addCustomURLs(self):
907         self.addURL("view layout test results", self.resultDirectoryURL() + "results.html")
908         self.addURL("view dashboard test results", self.resultDirectoryURL() + "dashboard-layout-test-results/results.html")
909
910     def finished(self, result):
911         self.addCustomURLs()
912         return master.MasterShellCommand.finished(self, result)
913
914
915 class ExtractTestResultsAndLeaks(ExtractTestResults):
916     def addCustomURLs(self):
917         ExtractTestResults.addCustomURLs(self)
918         url = "/LeaksViewer/?url=" + urllib.quote(self.resultDirectoryURL(), safe="")
919         self.addURL("view leaks", url)