[ews-build] Add support for API tests in OpenSource EWS
[WebKit-https.git] / Tools / BuildSlaveSupport / ews-build / steps.py
1 # Copyright (C) 2018 Apple Inc. All rights reserved.
2 #
3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions
5 # are met:
6 # 1.  Redistributions of source code must retain the above copyright
7 #     notice, this list of conditions and the following disclaimer.
8 # 2.  Redistributions in binary form must reproduce the above copyright
9 #     notice, this list of conditions and the following disclaimer in the
10 #     documentation and/or other materials provided with the distribution.
11 #
12 # THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND
13 # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
14 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
15 # DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR
16 # ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
17 # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
18 # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
19 # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
20 # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
21 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
22
23 from buildbot.process import buildstep, logobserver, properties
24 from buildbot.process.results import Results, SUCCESS, FAILURE, WARNINGS, SKIPPED, EXCEPTION, RETRY
25 from buildbot.steps import shell, transfer
26 from buildbot.steps.source import svn
27 from twisted.internet import defer
28
29 import re
30
31 EWS_URL = 'http://ews-build.webkit-uat.org/'
32 WithProperties = properties.WithProperties
33
34
35 class ConfigureBuild(buildstep.BuildStep):
36     name = "configure-build"
37     description = ["configuring build"]
38     descriptionDone = ["configured build"]
39
40     def __init__(self, platform, configuration, architectures, buildOnly, additionalArguments):
41         super(ConfigureBuild, self).__init__()
42         self.platform = platform
43         if platform != 'jsc-only':
44             self.platform = platform.split('-', 1)[0]
45         self.fullPlatform = platform
46         self.configuration = configuration
47         self.architecture = " ".join(architectures) if architectures else None
48         self.buildOnly = buildOnly
49         self.additionalArguments = additionalArguments
50
51     def start(self):
52         if self.platform and self.platform != '*':
53             self.setProperty('platform', self.platform, 'config.json')
54         if self.fullPlatform and self.fullPlatform != '*':
55             self.setProperty('fullPlatform', self.fullPlatform, 'ConfigureBuild')
56         if self.configuration:
57             self.setProperty('configuration', self.configuration, 'config.json')
58         if self.architecture:
59             self.setProperty('architecture', self.architecture, 'config.json')
60         if self.buildOnly:
61             self.setProperty("buildOnly", self.buildOnly, 'config.json')
62         if self.additionalArguments:
63             self.setProperty("additionalArguments", self.additionalArguments, 'config.json')
64         self.finished(SUCCESS)
65         return defer.succeed(None)
66
67
68 class CheckOutSource(svn.SVN):
69     CHECKOUT_DELAY_AND_MAX_RETRIES_PAIR = (0, 2)
70
71     def __init__(self, **kwargs):
72         self.repourl = 'https://svn.webkit.org/repository/webkit/trunk'
73         super(CheckOutSource, self).__init__(repourl=self.repourl,
74                                                 retry=self.CHECKOUT_DELAY_AND_MAX_RETRIES_PAIR,
75                                                 preferLastChangedRev=True,
76                                                 **kwargs)
77
78
79 class UnApplyPatchIfRequired(CheckOutSource):
80     name = 'unapply-patch'
81
82     def __init__(self, **kwargs):
83         super(UnApplyPatchIfRequired, self).__init__(alwaysUseLatest=True, **kwargs)
84
85     def doStepIf(self, step):
86         return self.getProperty('patchFailedToBuild') or self.getProperty('patchFailedJSCTests')
87
88     def hideStepIf(self, results, step):
89         return not self.doStepIf(step)
90
91
92 class CheckStyle(shell.ShellCommand):
93     name = 'check-webkit-style'
94     description = ['check-webkit-style running']
95     descriptionDone = ['check-webkit-style']
96     flunkOnFailure = True
97     command = ['Tools/Scripts/check-webkit-style']
98
99
100 class TestWithFailureCount(shell.Test):
101     failedTestsFormatString = "%d test%s failed"
102     failedTestCount = 0
103
104     def start(self):
105         self.log_observer = logobserver.BufferLogObserver(wantStderr=True)
106         self.addLogObserver('stdio', self.log_observer)
107         return shell.Test.start(self)
108
109     def countFailures(self, cmd):
110         raise NotImplementedError
111
112     def commandComplete(self, cmd):
113         shell.Test.commandComplete(self, cmd)
114         self.failedTestCount = self.countFailures(cmd)
115         self.failedTestPluralSuffix = "" if self.failedTestCount == 1 else "s"
116
117     def evaluateCommand(self, cmd):
118         if self.failedTestCount:
119             return FAILURE
120
121         if cmd.rc != 0:
122             return FAILURE
123
124         return SUCCESS
125
126     def getResultSummary(self):
127         status = self.name
128
129         if self.results != SUCCESS and self.failedTestCount:
130             status = self.failedTestsFormatString % (self.failedTestCount, self.failedTestPluralSuffix)
131
132         if self.results != SUCCESS:
133             status += u' ({})'.format(Results[self.results])
134
135         return {u'step': status}
136
137
138 class RunBindingsTests(shell.ShellCommand):
139     name = 'bindings-tests'
140     description = ['bindings-tests running']
141     descriptionDone = ['bindings-tests']
142     flunkOnFailure = True
143     jsonFileName = 'bindings_test_results.json'
144     logfiles = {'json': jsonFileName}
145     command = ['Tools/Scripts/run-bindings-tests', '--json-output={0}'.format(jsonFileName)]
146
147
148 class RunWebKitPerlTests(shell.ShellCommand):
149     name = 'webkitperl-tests'
150     description = ['webkitperl-tests running']
151     descriptionDone = ['webkitperl-tests']
152     flunkOnFailure = True
153     command = ['Tools/Scripts/test-webkitperl']
154
155     def __init__(self, **kwargs):
156         super(RunWebKitPerlTests, self).__init__(timeout=2 * 60, **kwargs)
157
158
159 class RunWebKitPyTests(shell.ShellCommand):
160     name = 'webkitpy-tests'
161     description = ['webkitpy-tests running']
162     descriptionDone = ['webkitpy-tests']
163     flunkOnFailure = True
164     jsonFileName = 'webkitpy_test_results.json'
165     logfiles = {'json': jsonFileName}
166     command = ['Tools/Scripts/test-webkitpy', '--json-output={0}'.format(jsonFileName)]
167
168     def __init__(self, **kwargs):
169         super(RunWebKitPyTests, self).__init__(timeout=2 * 60, **kwargs)
170
171
172 def appendCustomBuildFlags(step, platform, fullPlatform):
173     # FIXME: Make a common 'supported platforms' list.
174     if platform not in ('gtk', 'wincairo', 'ios', 'jsc-only', 'wpe'):
175         return
176     if fullPlatform.startswith('ios-simulator'):
177         platform = 'ios-simulator'
178     elif platform == 'ios':
179         platform = 'device'
180     step.setCommand(step.command + ['--' + platform])
181
182
183 class CompileWebKit(shell.Compile):
184     name = "compile-webkit"
185     description = ["compiling"]
186     descriptionDone = ["compiled"]
187     env = {'MFLAGS': ''}
188     warningPattern = ".*arning: .*"
189     haltOnFailure = False
190     command = ["perl", "Tools/Scripts/build-webkit", WithProperties("--%(configuration)s")]
191
192     def start(self):
193         platform = self.getProperty('platform')
194         buildOnly = self.getProperty('buildOnly')
195         architecture = self.getProperty('architecture')
196         additionalArguments = self.getProperty('additionalArguments')
197
198         if additionalArguments:
199             self.setCommand(self.command + additionalArguments)
200         if platform in ('mac', 'ios') and architecture:
201             self.setCommand(self.command + ['ARCHS=' + architecture])
202             if platform == 'ios':
203                 self.setCommand(self.command + ['ONLY_ACTIVE_ARCH=NO'])
204         if platform in ('mac', 'ios') and buildOnly:
205             # For build-only bots, the expectation is that tests will be run on separate machines,
206             # so we need to package debug info as dSYMs. Only generating line tables makes
207             # this much faster than full debug info, and crash logs still have line numbers.
208             self.setCommand(self.command + ['DEBUG_INFORMATION_FORMAT=dwarf-with-dsym'])
209             self.setCommand(self.command + ['CLANG_DEBUG_INFORMATION_LEVEL=line-tables-only'])
210
211         appendCustomBuildFlags(self, platform, self.getProperty('fullPlatform'))
212
213         return shell.Compile.start(self)
214
215     def evaluateCommand(self, cmd):
216         if cmd.didFail():
217             self.setProperty('patchFailedToBuild', True)
218
219         return super(CompileWebKit, self).evaluateCommand(cmd)
220
221
222 class CompileWebKitToT(CompileWebKit):
223     name = 'compile-webkit-tot'
224     haltOnFailure = True
225
226     def doStepIf(self, step):
227         return self.getProperty('patchFailedToBuild')
228
229     def hideStepIf(self, results, step):
230         return not self.doStepIf(step)
231
232
233 class CompileJSCOnly(CompileWebKit):
234     name = "build-jsc"
235     command = ["perl", "Tools/Scripts/build-jsc", WithProperties("--%(configuration)s")]
236
237
238 class CompileJSCOnlyToT(CompileJSCOnly):
239     name = 'build-jsc-tot'
240
241     def doStepIf(self, step):
242         return self.getProperty('patchFailedToBuild')
243
244     def hideStepIf(self, results, step):
245         return not self.doStepIf(step)
246
247
248 class RunJavaScriptCoreTests(shell.Test):
249     name = 'jscore-test'
250     description = ['jscore-tests running']
251     descriptionDone = ['jscore-tests']
252     flunkOnFailure = True
253     jsonFileName = 'jsc_results.json'
254     logfiles = {"json": jsonFileName}
255     command = ['perl', 'Tools/Scripts/run-javascriptcore-tests', '--no-build', '--no-fail-fast', '--json-output={0}'.format(jsonFileName), WithProperties('--%(configuration)s')]
256
257     def start(self):
258         appendCustomBuildFlags(self, self.getProperty('platform'), self.getProperty('fullPlatform'))
259         return shell.Test.start(self)
260
261     def evaluateCommand(self, cmd):
262         if cmd.didFail():
263             self.setProperty('patchFailedJSCTests', True)
264
265         return super(RunJavaScriptCoreTests, self).evaluateCommand(cmd)
266
267
268 class ReRunJavaScriptCoreTests(RunJavaScriptCoreTests):
269     name = 'jscore-test-rerun'
270
271     def doStepIf(self, step):
272         return self.getProperty('patchFailedJSCTests')
273
274     def hideStepIf(self, results, step):
275         return not self.doStepIf(step)
276
277     def evaluateCommand(self, cmd):
278         self.setProperty('patchFailedJSCTests', cmd.didFail())
279         return super(RunJavaScriptCoreTests, self).evaluateCommand(cmd)
280
281
282 class RunJavaScriptCoreTestsToT(RunJavaScriptCoreTests):
283     name = 'jscore-test-tot'
284     jsonFileName = 'jsc_results.json'
285     command = ['perl', 'Tools/Scripts/run-javascriptcore-tests', '--no-fail-fast', '--json-output={0}'.format(jsonFileName), WithProperties('--%(configuration)s')]
286
287     def doStepIf(self, step):
288         return self.getProperty('patchFailedJSCTests')
289
290     def hideStepIf(self, results, step):
291         return not self.doStepIf(step)
292
293
294 class CleanBuild(shell.Compile):
295     name = "delete-WebKitBuild-directory"
296     description = ["deleting WebKitBuild directory"]
297     descriptionDone = ["deleted WebKitBuild directory"]
298     command = ["python", "Tools/BuildSlaveSupport/clean-build", WithProperties("--platform=%(fullPlatform)s"), WithProperties("--%(configuration)s")]
299
300
301 class KillOldProcesses(shell.Compile):
302     name = "kill-old-processes"
303     description = ["killing old processes"]
304     descriptionDone = ["killed old processes"]
305     command = ["python", "Tools/BuildSlaveSupport/kill-old-processes", "buildbot"]
306
307     def __init__(self, **kwargs):
308         super(KillOldProcesses, self).__init__(timeout=60, **kwargs)
309
310
311 class RunWebKitTests(shell.Test):
312     name = 'layout-tests'
313     description = ['layout-tests running']
314     descriptionDone = ['layout-tests']
315     resultDirectory = 'layout-test-results'
316     command = ['python', 'Tools/Scripts/run-webkit-tests',
317                '--no-build',
318                '--no-new-test-results',
319                '--no-show-results',
320                '--exit-after-n-failures', '30',
321                '--skip-failing-tests',
322                WithProperties('--%(configuration)s')]
323
324     def start(self):
325         platform = self.getProperty('platform')
326         appendCustomBuildFlags(self, platform, self.getProperty('fullPlatform'))
327         additionalArguments = self.getProperty('additionalArguments')
328
329         self.setCommand(self.command + ['--results-directory', self.resultDirectory])
330         self.setCommand(self.command + ['--debug-rwt-logging'])
331
332         if additionalArguments:
333             self.setCommand(self.command + additionalArguments)
334         return shell.Test.start(self)
335
336
337 class ArchiveBuiltProduct(shell.ShellCommand):
338     command = ['python', 'Tools/BuildSlaveSupport/built-product-archive',
339                WithProperties('--platform=%(fullPlatform)s'), WithProperties('--%(configuration)s'), 'archive']
340     name = 'archive-built-product'
341     description = ['archiving built product']
342     descriptionDone = ['archived built product']
343     haltOnFailure = True
344
345
346 class UploadBuiltProduct(transfer.FileUpload):
347     name = 'upload-built-product'
348     workersrc = WithProperties('WebKitBuild/%(configuration)s.zip')
349     masterdest = WithProperties('public_html/archives/%(fullPlatform)s-%(architecture)s-%(configuration)s/%(ewspatchid)s.zip')
350     haltOnFailure = True
351
352     def __init__(self, **kwargs):
353         kwargs['workersrc'] = self.workersrc
354         kwargs['masterdest'] = self.masterdest
355         kwargs['mode'] = 0644
356         kwargs['blocksize'] = 1024 * 256
357         transfer.FileUpload.__init__(self, **kwargs)
358
359
360 class DownloadBuiltProduct(shell.ShellCommand):
361     command = ['python', 'Tools/BuildSlaveSupport/download-built-product',
362         WithProperties('--platform=%(platform)s'), WithProperties('--%(configuration)s'),
363         WithProperties(EWS_URL + 'archives/%(fullPlatform)s-%(architecture)s-%(configuration)s/%(ewspatchid)s.zip')]
364     name = 'download-built-product'
365     description = ['downloading built product']
366     descriptionDone = ['downloaded built product']
367     haltOnFailure = True
368     flunkOnFailure = True
369
370
371 class ExtractBuiltProduct(shell.ShellCommand):
372     command = ['python', 'Tools/BuildSlaveSupport/built-product-archive',
373                WithProperties('--platform=%(fullPlatform)s'), WithProperties('--%(configuration)s'), 'extract']
374     name = 'extract-built-product'
375     description = ['extracting built product']
376     descriptionDone = ['extracted built product']
377     haltOnFailure = True
378     flunkOnFailure = True
379
380
381 class RunAPITests(TestWithFailureCount):
382     name = 'run-api-tests'
383     description = ['api tests running']
384     descriptionDone = ['api-tests']
385     command = ['python', 'Tools/Scripts/run-api-tests', '--no-build', WithProperties('--%(configuration)s'), '--verbose']
386     failedTestsFormatString = '%d api test%s failed or timed out'
387
388     def start(self):
389         appendCustomBuildFlags(self, self.getProperty('platform'), self.getProperty('fullPlatform'))
390         return TestWithFailureCount.start(self)
391
392     def countFailures(self, cmd):
393         log_text = self.log_observer.getStdout() + self.log_observer.getStderr()
394
395         match = re.search(r'Ran (?P<ran>\d+) tests of (?P<total>\d+) with (?P<passed>\d+) successful', log_text)
396         if not match:
397             return 0
398         return int(match.group('ran')) - int(match.group('passed'))