bots should pass --clobber-old-results to run-webkit-tests
[WebKit-https.git] / Tools / BuildSlaveSupport / build.webkit.org-config / steps.py
1 # Copyright (C) 2017 Apple Inc. All rights reserved.
2 #
3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions
5 # are met:
6 # 1.  Redistributions of source code must retain the above copyright
7 #     notice, this list of conditions and the following disclaimer.
8 # 2.  Redistributions in binary form must reproduce the above copyright
9 #     notice, this list of conditions and the following disclaimer in the
10 #     documentation and/or other materials provided with the distribution.
11 #
12 # THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND
13 # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
14 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
15 # DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR
16 # ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
17 # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
18 # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
19 # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
20 # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
21 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
22
23 from buildbot.process import buildstep, factory, properties
24 from buildbot.steps import master, shell, source, transfer, trigger
25 from buildbot.status.builder import SUCCESS, FAILURE, WARNINGS, SKIPPED, EXCEPTION
26
27 from twisted.internet import defer
28
29 import os
30 import re
31 import json
32 import cStringIO
33 import urllib
34
35 APPLE_WEBKIT_AWS_PROXY = "http://proxy01.webkit.org:3128"
36 S3URL = "https://s3-us-west-2.amazonaws.com/"
37 WithProperties = properties.WithProperties
38
39
40 class TestWithFailureCount(shell.Test):
41     failedTestsFormatString = "%d test%s failed"
42
43     def countFailures(self, cmd):
44         return 0
45
46     def commandComplete(self, cmd):
47         shell.Test.commandComplete(self, cmd)
48         self.failedTestCount = self.countFailures(cmd)
49         self.failedTestPluralSuffix = "" if self.failedTestCount == 1 else "s"
50
51     def evaluateCommand(self, cmd):
52         if self.failedTestCount:
53             return FAILURE
54
55         if cmd.rc != 0:
56             return FAILURE
57
58         return SUCCESS
59
60     def getText(self, cmd, results):
61         return self.getText2(cmd, results)
62
63     def getText2(self, cmd, results):
64         if results != SUCCESS and self.failedTestCount:
65             return [self.failedTestsFormatString % (self.failedTestCount, self.failedTestPluralSuffix)]
66
67         return [self.name]
68
69
70 class ConfigureBuild(buildstep.BuildStep):
71     name = "configure build"
72     description = ["configuring build"]
73     descriptionDone = ["configured build"]
74
75     def __init__(self, platform, configuration, architecture, buildOnly, additionalArguments, SVNMirror, *args, **kwargs):
76         buildstep.BuildStep.__init__(self, *args, **kwargs)
77         self.platform = platform
78         if platform != 'jsc-only':
79             self.platform = platform.split('-', 1)[0]
80         self.fullPlatform = platform
81         self.configuration = configuration
82         self.architecture = architecture
83         self.buildOnly = buildOnly
84         self.additionalArguments = additionalArguments
85         self.SVNMirror = SVNMirror
86         self.addFactoryArguments(platform=platform, configuration=configuration, architecture=architecture, buildOnly=buildOnly, additionalArguments=additionalArguments, SVNMirror=SVNMirror)
87
88     def start(self):
89         self.setProperty("platform", self.platform)
90         self.setProperty("fullPlatform", self.fullPlatform)
91         self.setProperty("configuration", self.configuration)
92         self.setProperty("architecture", self.architecture)
93         self.setProperty("buildOnly", self.buildOnly)
94         self.setProperty("additionalArguments", self.additionalArguments)
95         self.setProperty("SVNMirror", self.SVNMirror)
96         self.finished(SUCCESS)
97         return defer.succeed(None)
98
99
100 class CheckOutSource(source.SVN):
101     mode = "update"
102
103     def __init__(self, SVNMirror, **kwargs):
104         kwargs['baseURL'] = SVNMirror or "https://svn.webkit.org/repository/webkit/"
105         kwargs['defaultBranch'] = "trunk"
106         kwargs['mode'] = self.mode
107         source.SVN.__init__(self, **kwargs)
108         self.addFactoryArguments(SVNMirror=SVNMirror)
109
110
111 class WaitForSVNServer(shell.ShellCommand):
112     name = "wait-for-svn-server"
113     command = ["python", "./Tools/BuildSlaveSupport/wait-for-SVN-server.py", "-r", WithProperties("%(revision)s"), "-s", WithProperties("%(SVNMirror)s")]
114     description = ["waiting for SVN server"]
115     descriptionDone = ["SVN server is ready"]
116     warnOnFailure = True
117
118     def evaluateCommand(self, cmd):
119         if cmd.rc != 0:
120             return WARNINGS
121         return SUCCESS
122
123
124 class InstallWin32Dependencies(shell.Compile):
125     description = ["installing dependencies"]
126     descriptionDone = ["installed dependencies"]
127     command = ["perl", "./Tools/Scripts/update-webkit-auxiliary-libs"]
128
129
130 class KillOldProcesses(shell.Compile):
131     name = "kill old processes"
132     description = ["killing old processes"]
133     descriptionDone = ["killed old processes"]
134     command = ["python", "./Tools/BuildSlaveSupport/kill-old-processes", "buildbot"]
135
136
137 class CleanBuildIfScheduled(shell.Compile):
138     name = "delete WebKitBuild directory"
139     description = ["deleting WebKitBuild directory"]
140     descriptionDone = ["deleted WebKitBuild directory"]
141     command = ["python", "./Tools/BuildSlaveSupport/clean-build", WithProperties("--platform=%(fullPlatform)s"), WithProperties("--%(configuration)s")]
142
143     def start(self):
144         if not self.getProperty('is_clean'):
145             self.hideStepIf = True
146             return SKIPPED
147         return shell.Compile.start(self)
148
149
150 class DeleteStaleBuildFiles(shell.Compile):
151     name = "delete stale build files"
152     description = ["deleting stale build files"]
153     descriptionDone = ["deleted stale build files"]
154     command = ["python", "./Tools/BuildSlaveSupport/delete-stale-build-files", WithProperties("--platform=%(fullPlatform)s"), WithProperties("--%(configuration)s")]
155
156     def start(self):
157         if self.getProperty('is_clean'):  # Nothing to be done if WebKitBuild had been removed.
158             self.hideStepIf = True
159             return SKIPPED
160         return shell.Compile.start(self)
161
162
163 class InstallWinCairoDependencies(shell.ShellCommand):
164     name = 'wincairo-requirements'
165     description = ['updating wincairo dependencies']
166     descriptionDone = ['updated wincairo dependencies']
167     command = ['python', './Tools/Scripts/update-webkit-wincairo-libs.py']
168     haltOnFailure = True
169
170
171 class InstallGtkDependencies(shell.ShellCommand):
172     name = "jhbuild"
173     description = ["updating gtk dependencies"]
174     descriptionDone = ["updated gtk dependencies"]
175     command = ["perl", "./Tools/Scripts/update-webkitgtk-libs"]
176     haltOnFailure = True
177
178
179 class InstallWpeDependencies(shell.ShellCommand):
180     name = "jhbuild"
181     description = ["updating wpe dependencies"]
182     descriptionDone = ["updated wpe dependencies"]
183     command = ["perl", "./Tools/Scripts/update-webkitwpe-libs"]
184     haltOnFailure = True
185
186
187 def appendCustomBuildFlags(step, platform, fullPlatform):
188     if platform not in ('gtk', 'wincairo', 'ios', 'jsc-only', 'wpe'):
189         return
190     if fullPlatform.startswith('ios-simulator'):
191         platform = 'ios-simulator'
192     elif platform == 'ios':
193         platform = 'device'
194     step.setCommand(step.command + ['--' + platform])
195
196
197 class CompileWebKit(shell.Compile):
198     command = ["perl", "./Tools/Scripts/build-webkit", WithProperties("--%(configuration)s")]
199     env = {'MFLAGS': ''}
200     name = "compile-webkit"
201     description = ["compiling"]
202     descriptionDone = ["compiled"]
203     warningPattern = ".*arning: .*"
204
205     def start(self):
206         platform = self.getProperty('platform')
207         buildOnly = self.getProperty('buildOnly')
208         architecture = self.getProperty('architecture')
209         additionalArguments = self.getProperty('additionalArguments')
210
211         if additionalArguments:
212             self.setCommand(self.command + additionalArguments)
213         if platform in ('mac', 'ios') and architecture:
214             self.setCommand(self.command + ['ARCHS=' + architecture])
215             if platform == 'ios':
216                 self.setCommand(self.command + ['ONLY_ACTIVE_ARCH=NO'])
217         if platform in ('mac', 'ios') and buildOnly:
218             # For build-only bots, the expectation is that tests will be run on separate machines,
219             # so we need to package debug info as dSYMs. Only generating line tables makes
220             # this much faster than full debug info, and crash logs still have line numbers.
221             self.setCommand(self.command + ['DEBUG_INFORMATION_FORMAT=dwarf-with-dsym'])
222             self.setCommand(self.command + ['CLANG_DEBUG_INFORMATION_LEVEL=line-tables-only'])
223
224         appendCustomBuildFlags(self, platform, self.getProperty('fullPlatform'))
225
226         return shell.Compile.start(self)
227
228     def createSummary(self, log):
229         platform = self.getProperty('platform')
230         if platform.startswith('mac'):
231             warnings = []
232             errors = []
233             sio = cStringIO.StringIO(log.getText())
234             for line in sio.readlines():
235                 if "arning:" in line:
236                     warnings.append(line)
237                 if "rror:" in line:
238                     errors.append(line)
239             if warnings:
240                 self.addCompleteLog('warnings', "".join(warnings))
241             if errors:
242                 self.addCompleteLog('errors', "".join(errors))
243
244
245 class CompileLLINTCLoop(CompileWebKit):
246     command = ["perl", "./Tools/Scripts/build-jsc", "--cloop", WithProperties("--%(configuration)s")]
247
248
249 class Compile32bitJSC(CompileWebKit):
250     command = ["perl", "./Tools/Scripts/build-jsc", "--32-bit", WithProperties("--%(configuration)s")]
251
252
253 class CompileJSCOnly(CompileWebKit):
254     command = ["perl", "./Tools/Scripts/build-jsc", WithProperties("--%(configuration)s")]
255
256
257 class ArchiveBuiltProduct(shell.ShellCommand):
258     command = ["python", "./Tools/BuildSlaveSupport/built-product-archive",
259                WithProperties("--platform=%(fullPlatform)s"), WithProperties("--%(configuration)s"), "archive"]
260     name = "archive-built-product"
261     description = ["archiving built product"]
262     descriptionDone = ["archived built product"]
263     haltOnFailure = True
264
265
266 class ArchiveMinifiedBuiltProduct(ArchiveBuiltProduct):
267     command = ["python", "./Tools/BuildSlaveSupport/built-product-archive",
268                WithProperties("--platform=%(fullPlatform)s"), WithProperties("--%(configuration)s"), "archive", "--minify"]
269
270
271 class GenerateJSCBundle(shell.ShellCommand):
272     command = ["python", "./Tools/Scripts/generate-jsc-bundle", "--builder-name", WithProperties("%(buildername)s"),
273                WithProperties("--platform=%(fullPlatform)s"), WithProperties("--%(configuration)s"),
274                WithProperties("--revision=%(got_revision)s"), "--remote-config-file", "../../remote-jsc-bundle-upload-config.json"]
275     name = "generate-jsc-bundle"
276     description = ["generating jsc bundle"]
277     descriptionDone = ["generated jsc bundle"]
278     haltOnFailure = False
279
280 class ExtractBuiltProduct(shell.ShellCommand):
281     command = ["python", "./Tools/BuildSlaveSupport/built-product-archive",
282                WithProperties("--platform=%(fullPlatform)s"), WithProperties("--%(configuration)s"), "extract"]
283     name = "extract-built-product"
284     description = ["extracting built product"]
285     descriptionDone = ["extracted built product"]
286     haltOnFailure = True
287
288
289 class UploadBuiltProduct(transfer.FileUpload):
290     slavesrc = WithProperties("WebKitBuild/%(configuration)s.zip")
291     masterdest = WithProperties("archives/%(fullPlatform)s-%(architecture)s-%(configuration)s/%(got_revision)s.zip")
292     haltOnFailure = True
293
294     def __init__(self, **kwargs):
295         kwargs['slavesrc'] = self.slavesrc
296         kwargs['masterdest'] = self.masterdest
297         kwargs['mode'] = 0644
298         kwargs['blocksize'] = 1024 * 256
299         transfer.FileUpload.__init__(self, **kwargs)
300
301
302 class UploadMinifiedBuiltProduct(UploadBuiltProduct):
303     slavesrc = WithProperties("WebKitBuild/minified-%(configuration)s.zip")
304     masterdest = WithProperties("archives/%(fullPlatform)s-%(architecture)s-%(configuration)s/minified-%(got_revision)s.zip")
305
306
307 class DownloadBuiltProduct(shell.ShellCommand):
308     command = ["python", "./Tools/BuildSlaveSupport/download-built-product",
309         WithProperties("--platform=%(platform)s"), WithProperties("--%(configuration)s"),
310         WithProperties(S3URL + "archives.webkit.org/%(fullPlatform)s-%(architecture)s-%(configuration)s/%(got_revision)s.zip")]
311     name = "download-built-product"
312     description = ["downloading built product"]
313     descriptionDone = ["downloaded built product"]
314     haltOnFailure = True
315     flunkOnFailure = True
316
317     def start(self):
318         if 'apple' in self.getProperty('buildername').lower():
319             self.slaveEnvironment['HTTPS_PROXY'] = APPLE_WEBKIT_AWS_PROXY  # curl env var to use a proxy
320         return shell.ShellCommand.start(self)
321
322
323 class RunJavaScriptCoreTests(TestWithFailureCount):
324     name = "jscore-test"
325     description = ["jscore-tests running"]
326     descriptionDone = ["jscore-tests"]
327     jsonFileName = "jsc_results.json"
328     command = ["perl", "./Tools/Scripts/run-javascriptcore-tests", "--no-build", "--no-fail-fast", "--json-output={0}".format(jsonFileName), WithProperties("--%(configuration)s")]
329     failedTestsFormatString = "%d JSC test%s failed"
330     logfiles = {"json": jsonFileName}
331
332     def start(self):
333         platform = self.getProperty('platform')
334         # Linux bots have currently problems with JSC tests that try to use large amounts of memory.
335         # Check: https://bugs.webkit.org/show_bug.cgi?id=175140
336         if platform in ('gtk', 'wpe'):
337             self.setCommand(self.command + ['--memory-limited'])
338         # WinCairo uses the Windows command prompt, not Cygwin.
339         elif platform == 'wincairo':
340             self.setCommand(self.command + ['--test-writer=ruby'])
341
342         appendCustomBuildFlags(self, platform, self.getProperty('fullPlatform'))
343         return shell.Test.start(self)
344
345     def countFailures(self, cmd):
346         logText = cmd.logs['stdio'].getText()
347
348         match = re.search(r'^Results for JSC stress tests:\r?\n\s+(\d+) failure', logText, re.MULTILINE)
349         if match:
350             return int(match.group(1))
351
352         match = re.search(r'^Results for Mozilla tests:\r?\n\s+(\d+) regression', logText, re.MULTILINE)
353         if match:
354             return int(match.group(1))
355
356         return 0
357
358
359 class RunRemoteJavaScriptCoreTests(RunJavaScriptCoreTests):
360     def start(self):
361         self.setCommand(self.command + ["--memory-limited", "--remote-config-file", "../../remote-jsc-tests-config.json"])
362         return RunJavaScriptCoreTests.start(self)
363
364
365 class RunTest262Tests(TestWithFailureCount):
366     name = "test262-test"
367     description = ["test262-tests running"]
368     descriptionDone = ["test262-tests"]
369     failedTestsFormatString = "%d Test262 test%s failed"
370     command = ["perl", "./Tools/Scripts/test262-runner", "--verbose", WithProperties("--%(configuration)s")]
371
372     def start(self):
373         appendCustomBuildFlags(self, self.getProperty('platform'), self.getProperty('fullPlatform'))
374         return shell.Test.start(self)
375
376     def countFailures(self, cmd):
377         logText = cmd.logs['stdio'].getText()
378         matches = re.findall(r'^\! NEW FAIL', logText, flags=re.MULTILINE)
379         if matches:
380             return len(matches)
381         return 0
382
383
384 class RunWebKitTests(shell.Test):
385     name = "layout-test"
386     description = ["layout-tests running"]
387     descriptionDone = ["layout-tests"]
388     resultDirectory = "layout-test-results"
389     command = ["python", "./Tools/Scripts/run-webkit-tests",
390                "--no-build",
391                "--no-show-results",
392                "--no-new-test-results",
393                "--clobber-old-results",
394                "--builder-name", WithProperties("%(buildername)s"),
395                "--build-number", WithProperties("%(buildnumber)s"),
396                "--master-name", "webkit.org",
397                "--test-results-server", "webkit-test-results.webkit.org",
398                "--exit-after-n-crashes-or-timeouts", "50",
399                "--exit-after-n-failures", "500",
400                WithProperties("--%(configuration)s")]
401
402     def start(self):
403         platform = self.getProperty('platform')
404         appendCustomBuildFlags(self, platform, self.getProperty('fullPlatform'))
405         additionalArguments = self.getProperty('additionalArguments')
406
407         self.setCommand(self.command + ["--results-directory", self.resultDirectory])
408         self.setCommand(self.command + ['--debug-rwt-logging'])
409
410         if platform == "win":
411             self.setCommand(self.command + ['--batch-size', '100', '--root=' + os.path.join("WebKitBuild", self.getProperty('configuration'), "bin32")])
412
413         if additionalArguments:
414             self.setCommand(self.command + additionalArguments)
415         return shell.Test.start(self)
416
417     # FIXME: This will break if run-webkit-tests changes its default log formatter.
418     nrwt_log_message_regexp = re.compile(r'\d{2}:\d{2}:\d{2}(\.\d+)?\s+\d+\s+(?P<message>.*)')
419
420     def _strip_python_logging_prefix(self, line):
421         match_object = self.nrwt_log_message_regexp.match(line)
422         if match_object:
423             return match_object.group('message')
424         return line
425
426     def _parseRunWebKitTestsOutput(self, logText):
427         incorrectLayoutLines = []
428         expressions = [
429             ('flakes', re.compile(r'Unexpected flakiness.+\((\d+)\)')),
430             ('new passes', re.compile(r'Expected to .+, but passed:\s+\((\d+)\)')),
431             ('missing results', re.compile(r'Regressions: Unexpected missing results\s+\((\d+)\)')),
432             ('failures', re.compile(r'Regressions: Unexpected.+\((\d+)\)')),
433         ]
434         testFailures = {}
435
436         for line in logText.splitlines():
437             if line.find('Exiting early') >= 0 or line.find('leaks found') >= 0:
438                 incorrectLayoutLines.append(self._strip_python_logging_prefix(line))
439                 continue
440             for name, expression in expressions:
441                 match = expression.search(line)
442
443                 if match:
444                     testFailures[name] = testFailures.get(name, 0) + int(match.group(1))
445                     break
446
447                 # FIXME: Parse file names and put them in results
448
449         for name in testFailures:
450             incorrectLayoutLines.append(str(testFailures[name]) + ' ' + name)
451
452         self.incorrectLayoutLines = incorrectLayoutLines
453
454     def commandComplete(self, cmd):
455         shell.Test.commandComplete(self, cmd)
456
457         logText = cmd.logs['stdio'].getText()
458         self._parseRunWebKitTestsOutput(logText)
459
460     def evaluateCommand(self, cmd):
461         result = SUCCESS
462
463         if self.incorrectLayoutLines:
464             if len(self.incorrectLayoutLines) == 1:
465                 line = self.incorrectLayoutLines[0]
466                 if line.find('were new') >= 0 or line.find('was new') >= 0 or line.find(' leak') >= 0:
467                     return WARNINGS
468
469             for line in self.incorrectLayoutLines:
470                 if line.find('flakes') >= 0 or line.find('new passes') >= 0 or line.find('missing results') >= 0:
471                     result = WARNINGS
472                 else:
473                     return FAILURE
474
475         # Return code from Tools/Scripts/layout_tests/run_webkit_tests.py.
476         # This means that an exception was raised when running run-webkit-tests and
477         # was never handled.
478         if cmd.rc == 254:
479             return EXCEPTION
480         if cmd.rc != 0:
481             return FAILURE
482
483         return result
484
485     def getText(self, cmd, results):
486         return self.getText2(cmd, results)
487
488     def getText2(self, cmd, results):
489         if results != SUCCESS and self.incorrectLayoutLines:
490             return self.incorrectLayoutLines
491
492         return [self.name]
493
494
495 class RunDashboardTests(RunWebKitTests):
496     name = "dashboard-tests"
497     description = ["dashboard-tests running"]
498     descriptionDone = ["dashboard-tests"]
499     resultDirectory = os.path.join(RunWebKitTests.resultDirectory, "dashboard-layout-test-results")
500
501     def start(self):
502         self.setCommand(self.command + ["--layout-tests-directory", "./Tools/BuildSlaveSupport/build.webkit.org-config/public_html/dashboard/Scripts/tests"])
503         return RunWebKitTests.start(self)
504
505
506 class RunUnitTests(TestWithFailureCount):
507     name = "run-api-tests"
508     description = ["unit tests running"]
509     descriptionDone = ["unit-tests"]
510     command = ["python", "./Tools/Scripts/run-api-tests", "--no-build", WithProperties("--%(configuration)s"), "--verbose"]
511     failedTestsFormatString = "%d unit test%s failed or timed out"
512
513     def start(self):
514         appendCustomBuildFlags(self, self.getProperty('platform'), self.getProperty('fullPlatform'))
515         return shell.Test.start(self)
516
517     def countFailures(self, cmd):
518         log_text = cmd.logs['stdio'].getText()
519
520         match = re.search(r'Ran (?P<ran>\d+) tests of (?P<total>\d+) with (?P<passed>\d+) successful', log_text)
521         if not match:
522             return -1
523         return int(match.group('ran')) - int(match.group('passed'))
524
525
526 class RunPythonTests(TestWithFailureCount):
527     name = "webkitpy-test"
528     description = ["python-tests running"]
529     descriptionDone = ["python-tests"]
530     command = ["python", "./Tools/Scripts/test-webkitpy", "--verbose", WithProperties("--%(configuration)s")]
531     failedTestsFormatString = "%d python test%s failed"
532
533     def start(self):
534         platform = self.getProperty('platform')
535         # Python tests are flaky on the GTK builders, running them serially
536         # helps and does not significantly prolong the cycle time.
537         if platform == 'gtk':
538             self.setCommand(self.command + ['--child-processes', '1'])
539         # Python tests fail on windows bots when running more than one child process
540         # https://bugs.webkit.org/show_bug.cgi?id=97465
541         if platform == 'win':
542             self.setCommand(self.command + ['--child-processes', '1'])
543         return shell.Test.start(self)
544
545     def countFailures(self, cmd):
546         logText = cmd.logs['stdio'].getText()
547         # We're looking for the line that looks like this: FAILED (failures=2, errors=1)
548         regex = re.compile(r'^FAILED \((?P<counts>[^)]+)\)')
549         for line in logText.splitlines():
550             match = regex.match(line)
551             if not match:
552                 continue
553             return sum(int(component.split('=')[1]) for component in match.group('counts').split(', '))
554         return 0
555
556
557 class RunPerlTests(TestWithFailureCount):
558     name = "webkitperl-test"
559     description = ["perl-tests running"]
560     descriptionDone = ["perl-tests"]
561     command = ["perl", "./Tools/Scripts/test-webkitperl"]
562     failedTestsFormatString = "%d perl test%s failed"
563
564     def countFailures(self, cmd):
565         logText = cmd.logs['stdio'].getText()
566         # We're looking for the line that looks like this: Failed 2/19 test programs. 5/363 subtests failed.
567         regex = re.compile(r'^Failed \d+/\d+ test programs\. (?P<count>\d+)/\d+ subtests failed\.')
568         for line in logText.splitlines():
569             match = regex.match(line)
570             if not match:
571                 continue
572             return int(match.group('count'))
573         return 0
574
575
576 class RunLLINTCLoopTests(TestWithFailureCount):
577     name = "webkit-jsc-cloop-test"
578     description = ["cloop-tests running"]
579     descriptionDone = ["cloop-tests"]
580     jsonFileName = "jsc_cloop.json"
581     command = ["perl", "./Tools/Scripts/run-javascriptcore-tests", "--cloop", "--no-build", "--no-jsc-stress", "--no-fail-fast", "--json-output={0}".format(jsonFileName), WithProperties("--%(configuration)s")]
582     failedTestsFormatString = "%d regression%s found."
583     logfiles = {"json": jsonFileName}
584
585     def countFailures(self, cmd):
586         logText = cmd.logs['stdio'].getText()
587         # We're looking for the line that looks like this: 0 regressions found.
588         regex = re.compile(r'\s*(?P<count>\d+) regressions? found.')
589         for line in logText.splitlines():
590             match = regex.match(line)
591             if not match:
592                 continue
593             return int(match.group('count'))
594         return 0
595
596
597 class Run32bitJSCTests(TestWithFailureCount):
598     name = "webkit-32bit-jsc-test"
599     description = ["32bit-jsc-tests running"]
600     descriptionDone = ["32bit-jsc-tests"]
601     jsonFileName = "jsc_32bit.json"
602     command = ["perl", "./Tools/Scripts/run-javascriptcore-tests", "--32-bit", "--no-build", "--no-fail-fast", "--json-output={0}".format(jsonFileName), WithProperties("--%(configuration)s")]
603     failedTestsFormatString = "%d regression%s found."
604     logfiles = {"json": jsonFileName}
605
606     def countFailures(self, cmd):
607         logText = cmd.logs['stdio'].getText()
608         # We're looking for the line that looks like this: 0 failures found.
609         regex = re.compile(r'\s*(?P<count>\d+) failures? found.')
610         for line in logText.splitlines():
611             match = regex.match(line)
612             if not match:
613                 continue
614             return int(match.group('count'))
615         return 0
616
617
618 class RunBindingsTests(shell.Test):
619     name = "bindings-generation-tests"
620     description = ["bindings-tests running"]
621     descriptionDone = ["bindings-tests"]
622     command = ["python", "./Tools/Scripts/run-bindings-tests"]
623
624
625 class RunBuiltinsTests(shell.Test):
626     name = "builtins-generator-tests"
627     description = ["builtins-generator-tests running"]
628     descriptionDone = ["builtins-generator-tests"]
629     command = ["python", "./Tools/Scripts/run-builtins-generator-tests"]
630
631
632 class RunGLibAPITests(shell.Test):
633     name = "API tests"
634     description = ["API tests running"]
635     descriptionDone = ["API tests"]
636
637     def start(self):
638         additionalArguments = self.getProperty("additionalArguments")
639         if additionalArguments:
640             self.command += additionalArguments
641         self.setCommand(self.command)
642         return shell.Test.start(self)
643
644     def commandComplete(self, cmd):
645         shell.Test.commandComplete(self, cmd)
646
647         logText = cmd.logs['stdio'].getText()
648
649         failedTests = 0
650         crashedTests = 0
651         timedOutTests = 0
652         messages = []
653         self.statusLine = []
654
655         foundItems = re.findall("Unexpected failures \((\d+)\)", logText)
656         if foundItems:
657             failedTests = int(foundItems[0])
658             messages.append("%d failures" % failedTests)
659
660         foundItems = re.findall("Unexpected crashes \((\d+)\)", logText)
661         if foundItems:
662             crashedTests = int(foundItems[0])
663             messages.append("%d crashes" % crashedTests)
664
665         foundItems = re.findall("Unexpected timeouts \((\d+)\)", logText)
666         if foundItems:
667             timedOutTests = int(foundItems[0])
668             messages.append("%d timeouts" % timedOutTests)
669
670         foundItems = re.findall("Unexpected passes \((\d+)\)", logText)
671         if foundItems:
672             newPassTests = int(foundItems[0])
673             messages.append("%d new passes" % newPassTests)
674
675         self.totalFailedTests = failedTests + crashedTests + timedOutTests
676         if messages:
677             self.statusLine = ["API tests: %s" % ", ".join(messages)]
678
679     def evaluateCommand(self, cmd):
680         if self.totalFailedTests > 0:
681             return FAILURE
682
683         if cmd.rc != 0:
684             return FAILURE
685
686         return SUCCESS
687
688     def getText(self, cmd, results):
689         return self.getText2(cmd, results)
690
691     def getText2(self, cmd, results):
692         if results != SUCCESS and self.totalFailedTests > 0:
693             return self.statusLine
694
695         return [self.name]
696
697
698 class RunGtkAPITests(RunGLibAPITests):
699     command = ["python", "./Tools/Scripts/run-gtk-tests", WithProperties("--%(configuration)s")]
700
701
702 class RunWPEAPITests(RunGLibAPITests):
703     command = ["python", "./Tools/Scripts/run-wpe-tests", WithProperties("--%(configuration)s")]
704
705
706 class RunWebDriverTests(shell.Test):
707     name = "webdriver-test"
708     description = ["webdriver-tests running"]
709     descriptionDone = ["webdriver-tests"]
710     jsonFileName = "webdriver_tests.json"
711     command = ["python", "./Tools/Scripts/run-webdriver-tests", "--json-output={0}".format(jsonFileName), WithProperties("--%(configuration)s")]
712     logfiles = {"json": jsonFileName}
713
714     def start(self):
715         additionalArguments = self.getProperty('additionalArguments')
716         if additionalArguments:
717             self.setCommand(self.command + additionalArguments)
718
719         appendCustomBuildFlags(self, self.getProperty('platform'), self.getProperty('fullPlatform'))
720         return shell.Test.start(self)
721
722     def commandComplete(self, cmd):
723         shell.Test.commandComplete(self, cmd)
724         logText = cmd.logs['stdio'].getText()
725
726         self.failuresCount = 0
727         self.newPassesCount = 0
728         foundItems = re.findall("^Unexpected .+ \((\d+)\)", logText, re.MULTILINE)
729         if foundItems:
730             self.failuresCount = int(foundItems[0])
731         foundItems = re.findall("^Expected to .+, but passed \((\d+)\)", logText, re.MULTILINE)
732         if foundItems:
733             self.newPassesCount = int(foundItems[0])
734
735     def evaluateCommand(self, cmd):
736         if self.failuresCount:
737             return FAILURE
738
739         if self.newPassesCount:
740             return WARNINGS
741
742         if cmd.rc != 0:
743             return FAILURE
744
745         return SUCCESS
746
747     def getText(self, cmd, results):
748         return self.getText2(cmd, results)
749
750     def getText2(self, cmd, results):
751         if results != SUCCESS and (self.failuresCount or self.newPassesCount):
752             lines = []
753             if self.failuresCount:
754                 lines.append("%d failures" % self.failuresCount)
755             if self.newPassesCount:
756                 lines.append("%d new passes" % self.newPassesCount)
757             return ["%s %s" % (self.name, ", ".join(lines))]
758
759         return [self.name]
760
761
762 class RunWebKit1Tests(RunWebKitTests):
763     def start(self):
764         self.setCommand(self.command + ["--dump-render-tree"])
765
766         return RunWebKitTests.start(self)
767
768
769 class RunWebKit1LeakTests(RunWebKit1Tests):
770     want_stdout = False
771     want_stderr = False
772     warnOnWarnings = True
773
774     def start(self):
775         self.setCommand(self.command + ["--leaks"])
776         return RunWebKit1Tests.start(self)
777
778
779 class RunAndUploadPerfTests(shell.Test):
780     name = "perf-test"
781     description = ["perf-tests running"]
782     descriptionDone = ["perf-tests"]
783     command = ["python", "./Tools/Scripts/run-perf-tests",
784                "--output-json-path", "perf-test-results.json",
785                "--slave-config-json-path", "../../perf-test-config.json",
786                "--no-show-results",
787                "--reset-results",
788                "--test-results-server", "perf.webkit.org",
789                "--builder-name", WithProperties("%(buildername)s"),
790                "--build-number", WithProperties("%(buildnumber)s"),
791                "--platform", WithProperties("%(fullPlatform)s"),
792                "--no-build",
793                WithProperties("--%(configuration)s")]
794
795     def start(self):
796         additionalArguments = self.getProperty("additionalArguments")
797         if additionalArguments:
798             self.command += additionalArguments
799         self.setCommand(self.command)
800         return shell.Test.start(self)
801
802     def getText(self, cmd, results):
803         return self.getText2(cmd, results)
804
805     def getText2(self, cmd, results):
806         if results != SUCCESS:
807             if cmd.rc == -1 & 0xff:
808                 return ["build not up to date"]
809             elif cmd.rc == -2 & 0xff:
810                 return ["slave config JSON error"]
811             elif cmd.rc == -3 & 0xff:
812                 return ["output JSON merge error"]
813             elif cmd.rc == -4 & 0xff:
814                 return ["upload error"]
815             elif cmd.rc == -5 & 0xff:
816                 return ["system dependency error"]
817             elif cmd.rc == -1:
818                 return ["timeout"]
819             else:
820                 return ["%d perf tests failed" % cmd.rc]
821
822         return [self.name]
823
824
825 class RunBenchmarkTests(shell.Test):
826     name = "benchmark-test"
827     description = ["benchmark tests running"]
828     descriptionDone = ["benchmark tests"]
829     command = ["python", "./Tools/Scripts/browserperfdash-benchmark", "--allplans",
830                "--config-file", "../../browserperfdash-benchmark-config.txt",
831                "--browser-version", WithProperties("r%(got_revision)s")]
832
833     def start(self):
834         platform = self.getProperty("platform")
835         if platform == "gtk":
836             self.command += ["--browser", "minibrowser-gtk"]
837         self.setCommand(self.command)
838         return shell.Test.start(self)
839
840     def getText(self, cmd, results):
841         return self.getText2(cmd, results)
842
843     def getText2(self, cmd, results):
844         if results != SUCCESS:
845             return ["%d benchmark tests failed" % cmd.rc]
846         return [self.name]
847
848
849 class ArchiveTestResults(shell.ShellCommand):
850     command = ["python", "./Tools/BuildSlaveSupport/test-result-archive",
851                WithProperties("--platform=%(platform)s"), WithProperties("--%(configuration)s"), "archive"]
852     name = "archive-test-results"
853     description = ["archiving test results"]
854     descriptionDone = ["archived test results"]
855     haltOnFailure = True
856
857
858 class UploadTestResults(transfer.FileUpload):
859     slavesrc = "layout-test-results.zip"
860     masterdest = WithProperties("public_html/results/%(buildername)s/r%(got_revision)s (%(buildnumber)s).zip")
861
862     def __init__(self, **kwargs):
863         kwargs['slavesrc'] = self.slavesrc
864         kwargs['masterdest'] = self.masterdest
865         kwargs['mode'] = 0644
866         transfer.FileUpload.__init__(self, **kwargs)
867
868
869 class TransferToS3(master.MasterShellCommand):
870     name = "transfer-to-s3"
871     description = ["transferring to s3"]
872     descriptionDone = ["transferred to s3"]
873     archive = WithProperties("archives/%(fullPlatform)s-%(architecture)s-%(configuration)s/%(got_revision)s.zip")
874     minifiedArchive = WithProperties("archives/%(fullPlatform)s-%(architecture)s-%(configuration)s/minified-%(got_revision)s.zip")
875     identifier = WithProperties("%(fullPlatform)s-%(architecture)s-%(configuration)s")
876     revision = WithProperties("%(got_revision)s")
877     command = ["python", "./transfer-archive-to-s3", "--revision", revision, "--identifier", identifier, "--archive", archive]
878     haltOnFailure = True
879
880     def __init__(self, **kwargs):
881         kwargs['command'] = self.command
882         master.MasterShellCommand.__init__(self, **kwargs)
883
884     def start(self):
885         return master.MasterShellCommand.start(self)
886
887     def finished(self, result):
888         return master.MasterShellCommand.finished(self, result)
889
890
891 class ExtractTestResults(master.MasterShellCommand):
892     zipFile = WithProperties("public_html/results/%(buildername)s/r%(got_revision)s (%(buildnumber)s).zip")
893     resultDirectory = WithProperties("public_html/results/%(buildername)s/r%(got_revision)s (%(buildnumber)s)")
894     descriptionDone = ["uploaded results"]
895
896     def __init__(self, **kwargs):
897         kwargs['command'] = ""
898         master.MasterShellCommand.__init__(self, **kwargs)
899
900     def resultDirectoryURL(self):
901         return self.build.getProperties().render(self.resultDirectory).replace("public_html/", "/") + "/"
902
903     def start(self):
904         self.command = ["unzip", self.build.getProperties().render(self.zipFile), "-d", self.build.getProperties().render(self.resultDirectory)]
905         return master.MasterShellCommand.start(self)
906
907     def addCustomURLs(self):
908         self.addURL("view layout test results", self.resultDirectoryURL() + "results.html")
909         self.addURL("view dashboard test results", self.resultDirectoryURL() + "dashboard-layout-test-results/results.html")
910
911     def finished(self, result):
912         self.addCustomURLs()
913         return master.MasterShellCommand.finished(self, result)
914
915
916 class ExtractTestResultsAndLeaks(ExtractTestResults):
917     def addCustomURLs(self):
918         ExtractTestResults.addCustomURLs(self)
919         url = "/LeaksViewer/?url=" + urllib.quote(self.resultDirectoryURL(), safe="")
920         self.addURL("view leaks", url)