ba2209a2f02745f6e4464324fcbe8e541701c2af
[WebKit-https.git] / Tools / BuildSlaveSupport / ews-build / steps.py
1 # Copyright (C) 2018-2019 Apple Inc. All rights reserved.
2 #
3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions
5 # are met:
6 # 1.  Redistributions of source code must retain the above copyright
7 #     notice, this list of conditions and the following disclaimer.
8 # 2.  Redistributions in binary form must reproduce the above copyright
9 #     notice, this list of conditions and the following disclaimer in the
10 #     documentation and/or other materials provided with the distribution.
11 #
12 # THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND
13 # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
14 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
15 # DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR
16 # ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
17 # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
18 # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
19 # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
20 # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
21 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
22
23 from buildbot.plugins import steps, util
24 from buildbot.process import buildstep, logobserver, properties
25 from buildbot.process.results import Results, SUCCESS, FAILURE, WARNINGS, SKIPPED, EXCEPTION, RETRY
26 from buildbot.steps import master, shell, transfer
27 from buildbot.steps.source import git
28 from buildbot.steps.worker import CompositeStepMixin
29 from twisted.internet import defer
30
31 import json
32 import re
33 import requests
34
35 BUG_SERVER_URL = 'https://bugs.webkit.org/'
36 EWS_URL = 'https://ews-build.webkit.org/'
37 WithProperties = properties.WithProperties
38 Interpolate = properties.Interpolate
39
40
41 class ConfigureBuild(buildstep.BuildStep):
42     name = "configure-build"
43     description = ["configuring build"]
44     descriptionDone = ["Configured build"]
45
46     def __init__(self, platform, configuration, architectures, buildOnly, additionalArguments):
47         super(ConfigureBuild, self).__init__()
48         self.platform = platform
49         if platform != 'jsc-only':
50             self.platform = platform.split('-', 1)[0]
51         self.fullPlatform = platform
52         self.configuration = configuration
53         self.architecture = " ".join(architectures) if architectures else None
54         self.buildOnly = buildOnly
55         self.additionalArguments = additionalArguments
56
57     def start(self):
58         if self.platform and self.platform != '*':
59             self.setProperty('platform', self.platform, 'config.json')
60         if self.fullPlatform and self.fullPlatform != '*':
61             self.setProperty('fullPlatform', self.fullPlatform, 'ConfigureBuild')
62         if self.configuration:
63             self.setProperty('configuration', self.configuration, 'config.json')
64         if self.architecture:
65             self.setProperty('architecture', self.architecture, 'config.json')
66         if self.buildOnly:
67             self.setProperty("buildOnly", self.buildOnly, 'config.json')
68         if self.additionalArguments:
69             self.setProperty("additionalArguments", self.additionalArguments, 'config.json')
70
71         self.add_patch_id_url()
72         self.finished(SUCCESS)
73         return defer.succeed(None)
74
75     def add_patch_id_url(self):
76         patch_id = self.getProperty('patch_id', '')
77         if patch_id:
78             self.addURL('Patch {}'.format(patch_id), self.getPatchURL(patch_id))
79
80     def getPatchURL(self, patch_id):
81         if not patch_id:
82             return None
83         return '{}attachment.cgi?id={}'.format(BUG_SERVER_URL, patch_id)
84
85
86 class CheckOutSource(git.Git):
87     name = 'clean-and-update-working-directory'
88     CHECKOUT_DELAY_AND_MAX_RETRIES_PAIR = (0, 2)
89
90     def __init__(self, **kwargs):
91         self.repourl = 'https://git.webkit.org/git/WebKit.git'
92         super(CheckOutSource, self).__init__(repourl=self.repourl,
93                                                 retry=self.CHECKOUT_DELAY_AND_MAX_RETRIES_PAIR,
94                                                 timeout=2 * 60 * 60,
95                                                 alwaysUseLatest=True,
96                                                 progress=True,
97                                                 **kwargs)
98
99     def getResultSummary(self):
100         if self.results != SUCCESS:
101             return {u'step': u'Failed to updated working directory'}
102         else:
103             return {u'step': u'Cleaned and updated working directory'}
104
105
106 class CleanWorkingDirectory(shell.ShellCommand):
107     name = 'clean-working-directory'
108     description = ['clean-working-directory running']
109     descriptionDone = ['Cleaned working directory']
110     flunkOnFailure = True
111     haltOnFailure = True
112     command = ['Tools/Scripts/clean-webkit']
113
114
115 class ApplyPatch(shell.ShellCommand, CompositeStepMixin):
116     name = 'apply-patch'
117     description = ['applying-patch']
118     descriptionDone = ['Applied patch']
119     flunkOnFailure = True
120     haltOnFailure = True
121     command = ['Tools/Scripts/svn-apply', '--force', '.buildbot-diff']
122
123     def _get_patch(self):
124         sourcestamp = self.build.getSourceStamp(self.getProperty('codebase', ''))
125         if not sourcestamp or not sourcestamp.patch:
126             return None
127         return sourcestamp.patch[1]
128
129     def start(self):
130         patch = self._get_patch()
131         if not patch:
132             self.finished(FAILURE)
133             return None
134
135         d = self.downloadFileContentToWorker('.buildbot-diff', patch)
136         d.addCallback(lambda _: self.downloadFileContentToWorker('.buildbot-patched', 'patched\n'))
137         d.addCallback(lambda res: shell.ShellCommand.start(self))
138
139     def getResultSummary(self):
140         if self.results != SUCCESS:
141             return {u'step': u'Patch does not apply'}
142         return super(ApplyPatch, self).getResultSummary()
143
144
145 class CheckPatchRelevance(buildstep.BuildStep):
146     name = 'check-patch-relevance'
147     description = ['check-patch-relevance running']
148     descriptionDone = ['Checked patch relevance']
149     flunkOnFailure = True
150     haltOnFailure = True
151
152     bindings_paths = [
153         "Source/WebCore",
154         "Tools",
155     ]
156
157     jsc_paths = [
158         "JSTests/",
159         "Source/JavaScriptCore/",
160         "Source/WTF/",
161         "Source/bmalloc/",
162         "Makefile",
163         "Makefile.shared",
164         "Source/Makefile",
165         "Source/Makefile.shared",
166         "Tools/Scripts/build-webkit",
167         "Tools/Scripts/build-jsc",
168         "Tools/Scripts/jsc-stress-test-helpers/",
169         "Tools/Scripts/run-jsc",
170         "Tools/Scripts/run-jsc-benchmarks",
171         "Tools/Scripts/run-jsc-stress-tests",
172         "Tools/Scripts/run-javascriptcore-tests",
173         "Tools/Scripts/run-layout-jsc",
174         "Tools/Scripts/update-javascriptcore-test-results",
175         "Tools/Scripts/webkitdirs.pm",
176     ]
177
178     webkitpy_paths = [
179         "Tools/Scripts/webkitpy/",
180         "Tools/QueueStatusServer/",
181     ]
182
183     group_to_paths_mapping = {
184         'bindings': bindings_paths,
185         'jsc': jsc_paths,
186         'webkitpy': webkitpy_paths,
187     }
188
189     def _patch_is_relevant(self, patch, builderName):
190         group = [group for group in self.group_to_paths_mapping.keys() if group in builderName.lower()]
191         if not group:
192             # This builder doesn't have paths defined, all patches are relevant.
193             return True
194
195         relevant_paths = self.group_to_paths_mapping[group[0]]
196
197         for change in patch.splitlines():
198             for path in relevant_paths:
199                 if re.search(path, change, re.IGNORECASE):
200                     return True
201         return False
202
203     def _get_patch(self):
204         sourcestamp = self.build.getSourceStamp(self.getProperty('codebase', ''))
205         if not sourcestamp or not sourcestamp.patch:
206             return None
207         return sourcestamp.patch[1]
208
209     @defer.inlineCallbacks
210     def _addToLog(self, logName, message):
211         try:
212             log = self.getLog(logName)
213         except KeyError:
214             log = yield self.addLog(logName)
215         log.addStdout(message)
216
217     def start(self):
218         patch = self._get_patch()
219         if not patch:
220             # This build doesn't have a patch, it might be a force build.
221             self.finished(SUCCESS)
222             return None
223
224         if self._patch_is_relevant(patch, self.getProperty('buildername', '')):
225             self._addToLog('stdio', 'This patch contains relevant changes.')
226             self.finished(SUCCESS)
227             return None
228
229         self._addToLog('stdio', 'This patch does not have relevant changes.')
230         self.finished(FAILURE)
231         self.build.results = SKIPPED
232         self.build.buildFinished(['Patch {} doesn\'t have relevant changes'.format(self.getProperty('patch_id', ''))], SKIPPED)
233         return None
234
235
236 class ValidatePatch(buildstep.BuildStep):
237     name = 'validate-patch'
238     description = ['validate-patch running']
239     descriptionDone = ['Validated patch']
240     flunkOnFailure = True
241     haltOnFailure = True
242     bug_open_statuses = ["UNCONFIRMED", "NEW", "ASSIGNED", "REOPENED"]
243     bug_closed_statuses = ["RESOLVED", "VERIFIED", "CLOSED"]
244
245     @defer.inlineCallbacks
246     def _addToLog(self, logName, message):
247         try:
248             log = self.getLog(logName)
249         except KeyError:
250             log = yield self.addLog(logName)
251         log.addStdout(message)
252
253     def fetch_data_from_url(self, url):
254         response = None
255         try:
256             response = requests.get(url)
257         except Exception as e:
258             if response:
259                 self._addToLog('stdio', 'Failed to access {url} with status code {status_code}.\n'.format(url=url, status_code=response.status_code))
260             else:
261                 self._addToLog('stdio', 'Failed to access {url} with exception: {exception}\n'.format(url=url, exception=e))
262             return None
263         if response.status_code != 200:
264             self._addToLog('stdio', 'Accessed {url} with unexpected status code {status_code}.\n'.format(url=url, status_code=response.status_code))
265             return None
266         return response
267
268     def get_patch_json(self, patch_id):
269         patch_url = '{}rest/bug/attachment/{}'.format(BUG_SERVER_URL, patch_id)
270         patch = self.fetch_data_from_url(patch_url)
271         if not patch:
272             return None
273         patch_json = patch.json().get('attachments')
274         if not patch_json or len(patch_json) == 0:
275             return None
276         return patch_json.get(str(patch_id))
277
278     def get_bug_json(self, bug_id):
279         bug_url = '{}rest/bug/{}'.format(BUG_SERVER_URL, bug_id)
280         bug = self.fetch_data_from_url(bug_url)
281         if not bug:
282             return None
283         bugs_json = bug.json().get('bugs')
284         if not bugs_json or len(bugs_json) == 0:
285             return None
286         return bugs_json[0]
287
288     def get_bug_id_from_patch(self, patch_id):
289         patch_json = self.get_patch_json(patch_id)
290         if not patch_json:
291             self._addToLog('stdio', 'Unable to fetch patch {}.\n'.format(patch_id))
292             return -1
293         return patch_json.get('bug_id')
294
295     def _is_patch_obsolete(self, patch_id):
296         patch_json = self.get_patch_json(patch_id)
297         if not patch_json:
298             self._addToLog('stdio', 'Unable to fetch patch {}.\n'.format(patch_id))
299             return -1
300
301         if str(patch_json.get('id')) != self.getProperty('patch_id', ''):
302             self._addToLog('stdio', 'Fetched patch id {} does not match with requested patch id {}. Unable to validate.\n'.format(patch_json.get('id'), self.getProperty('patch_id', '')))
303             return -1
304
305         patch_author = patch_json.get('creator')
306         self.addURL('Patch by: {}'.format(patch_author), 'mailto:{}'.format(patch_author))
307         return patch_json.get('is_obsolete')
308
309     def _is_patch_review_denied(self, patch_id):
310         patch_json = self.get_patch_json(patch_id)
311         if not patch_json:
312             self._addToLog('stdio', 'Unable to fetch patch {}.\n'.format(patch_id))
313             return -1
314
315         for flag in patch_json.get('flags', []):
316             if flag.get('name') == 'review' and flag.get('status') == '-':
317                 return 1
318         return 0
319
320     def _is_bug_closed(self, bug_id):
321         if not bug_id:
322             self._addToLog('stdio', 'Skipping bug status validation since bug id is None.\n')
323             return -1
324
325         bug_json = self.get_bug_json(bug_id)
326         if not bug_json or not bug_json.get('status'):
327             self._addToLog('stdio', 'Unable to fetch bug {}.\n'.format(bug_id))
328             return -1
329
330         bug_title = bug_json.get('summary')
331         self.addURL('Bug {} {}'.format(bug_id, bug_title), '{}show_bug.cgi?id={}'.format(BUG_SERVER_URL, bug_id))
332         if bug_json.get('status') in self.bug_closed_statuses:
333             return 1
334         return 0
335
336     def skip_build(self, reason):
337         self._addToLog('stdio', reason)
338         self.finished(FAILURE)
339         self.build.results = SKIPPED
340         self.build.buildFinished([reason], SKIPPED)
341
342     def start(self):
343         patch_id = self.getProperty('patch_id', '')
344         if not patch_id:
345             self._addToLog('stdio', 'No patch_id found. Unable to proceed without patch_id.\n')
346             self.finished(FAILURE)
347             return None
348
349         bug_id = self.getProperty('bug_id', '') or self.get_bug_id_from_patch(patch_id)
350
351         bug_closed = self._is_bug_closed(bug_id)
352         if bug_closed == 1:
353             self.skip_build('Bug {} is already closed'.format(bug_id))
354             return None
355
356         obsolete = self._is_patch_obsolete(patch_id)
357         if obsolete == 1:
358             self.skip_build('Patch {} is obsolete'.format(patch_id))
359             return None
360
361         review_denied = self._is_patch_review_denied(patch_id)
362         if review_denied == 1:
363             self.skip_build('Patch {} is marked r-'.format(patch_id))
364             return None
365
366         if obsolete == -1 or review_denied == -1 or bug_closed == -1:
367             self.finished(WARNINGS)
368             return None
369
370         self._addToLog('stdio', 'Bug is open.\nPatch is not obsolete.\nPatch is not marked r-.\n')
371         self.finished(SUCCESS)
372         return None
373
374
375 class UnApplyPatchIfRequired(CleanWorkingDirectory):
376     name = 'unapply-patch'
377     descriptionDone = ['Unapplied patch']
378
379     def doStepIf(self, step):
380         return self.getProperty('patchFailedToBuild') or self.getProperty('patchFailedJSCTests') or self.getProperty('patchFailedAPITests')
381
382     def hideStepIf(self, results, step):
383         return not self.doStepIf(step)
384
385
386 class TestWithFailureCount(shell.Test):
387     failedTestsFormatString = "%d test%s failed"
388     failedTestCount = 0
389
390     def start(self):
391         self.log_observer = logobserver.BufferLogObserver(wantStderr=True)
392         self.addLogObserver('stdio', self.log_observer)
393         return shell.Test.start(self)
394
395     def countFailures(self, cmd):
396         raise NotImplementedError
397
398     def commandComplete(self, cmd):
399         shell.Test.commandComplete(self, cmd)
400         self.failedTestCount = self.countFailures(cmd)
401         self.failedTestPluralSuffix = "" if self.failedTestCount == 1 else "s"
402
403     def evaluateCommand(self, cmd):
404         if self.failedTestCount:
405             return FAILURE
406
407         if cmd.rc != 0:
408             return FAILURE
409
410         return SUCCESS
411
412     def getResultSummary(self):
413         status = self.name
414
415         if self.results != SUCCESS and self.failedTestCount:
416             status = self.failedTestsFormatString % (self.failedTestCount, self.failedTestPluralSuffix)
417
418         if self.results != SUCCESS:
419             status += u' ({})'.format(Results[self.results])
420
421         return {u'step': status}
422
423
424 class CheckStyle(TestWithFailureCount):
425     name = 'check-webkit-style'
426     description = ['check-webkit-style running']
427     descriptionDone = ['check-webkit-style']
428     flunkOnFailure = True
429     failedTestsFormatString = '%d style error%s'
430     command = ['Tools/Scripts/check-webkit-style']
431
432     def countFailures(self, cmd):
433         log_text = self.log_observer.getStdout() + self.log_observer.getStderr()
434
435         match = re.search(r'Total errors found: (?P<errors>\d+) in (?P<files>\d+) files', log_text)
436         if not match:
437             return 0
438         return int(match.group('errors'))
439
440
441 class RunBindingsTests(shell.ShellCommand):
442     name = 'bindings-tests'
443     description = ['bindings-tests running']
444     descriptionDone = ['bindings-tests']
445     flunkOnFailure = True
446     jsonFileName = 'bindings_test_results.json'
447     logfiles = {'json': jsonFileName}
448     command = ['Tools/Scripts/run-bindings-tests', '--json-output={0}'.format(jsonFileName)]
449
450
451 class RunWebKitPerlTests(shell.ShellCommand):
452     name = 'webkitperl-tests'
453     description = ['webkitperl-tests running']
454     descriptionDone = ['webkitperl-tests']
455     flunkOnFailure = True
456     command = ['Tools/Scripts/test-webkitperl']
457
458     def __init__(self, **kwargs):
459         super(RunWebKitPerlTests, self).__init__(timeout=2 * 60, **kwargs)
460
461
462 class RunWebKitPyTests(shell.ShellCommand):
463     name = 'webkitpy-tests'
464     description = ['webkitpy-tests running']
465     descriptionDone = ['webkitpy-tests']
466     flunkOnFailure = True
467     jsonFileName = 'webkitpy_test_results.json'
468     logfiles = {'json': jsonFileName}
469     command = ['Tools/Scripts/test-webkitpy', '--json-output={0}'.format(jsonFileName)]
470
471     def __init__(self, **kwargs):
472         super(RunWebKitPyTests, self).__init__(timeout=2 * 60, **kwargs)
473
474
475 def appendCustomBuildFlags(step, platform, fullPlatform):
476     # FIXME: Make a common 'supported platforms' list.
477     if platform not in ('gtk', 'wincairo', 'ios', 'jsc-only', 'wpe'):
478         return
479     if fullPlatform.startswith('ios-simulator'):
480         platform = 'ios-simulator'
481     elif platform == 'ios':
482         platform = 'device'
483     step.setCommand(step.command + ['--' + platform])
484
485
486 class CompileWebKit(shell.Compile):
487     name = "compile-webkit"
488     description = ["compiling"]
489     descriptionDone = ["Compiled WebKit"]
490     env = {'MFLAGS': ''}
491     warningPattern = ".*arning: .*"
492     haltOnFailure = False
493     command = ["perl", "Tools/Scripts/build-webkit", WithProperties("--%(configuration)s")]
494
495     def start(self):
496         platform = self.getProperty('platform')
497         buildOnly = self.getProperty('buildOnly')
498         architecture = self.getProperty('architecture')
499         additionalArguments = self.getProperty('additionalArguments')
500
501         if additionalArguments:
502             self.setCommand(self.command + additionalArguments)
503         if platform in ('mac', 'ios') and architecture:
504             self.setCommand(self.command + ['ARCHS=' + architecture])
505             if platform == 'ios':
506                 self.setCommand(self.command + ['ONLY_ACTIVE_ARCH=NO'])
507         if platform in ('mac', 'ios') and buildOnly:
508             # For build-only bots, the expectation is that tests will be run on separate machines,
509             # so we need to package debug info as dSYMs. Only generating line tables makes
510             # this much faster than full debug info, and crash logs still have line numbers.
511             self.setCommand(self.command + ['DEBUG_INFORMATION_FORMAT=dwarf-with-dsym'])
512             self.setCommand(self.command + ['CLANG_DEBUG_INFORMATION_LEVEL=line-tables-only'])
513
514         appendCustomBuildFlags(self, platform, self.getProperty('fullPlatform'))
515
516         return shell.Compile.start(self)
517
518     def evaluateCommand(self, cmd):
519         if cmd.didFail():
520             self.setProperty('patchFailedToBuild', True)
521             self.build.addStepsAfterCurrentStep([UnApplyPatchIfRequired(), CompileWebKitToT()])
522         else:
523             self.build.addStepsAfterCurrentStep([ArchiveBuiltProduct(), UploadBuiltProduct()])
524
525
526         return super(CompileWebKit, self).evaluateCommand(cmd)
527
528
529 class CompileWebKitToT(CompileWebKit):
530     name = 'compile-webkit-tot'
531     haltOnFailure = True
532
533     def doStepIf(self, step):
534         return self.getProperty('patchFailedToBuild') or self.getProperty('patchFailedAPITests')
535
536     def hideStepIf(self, results, step):
537         return not self.doStepIf(step)
538
539     def evaluateCommand(self, cmd):
540         return shell.Compile.evaluateCommand(self, cmd)
541
542
543 class CompileJSCOnly(CompileWebKit):
544     name = "build-jsc"
545     descriptionDone = ["Compiled JSC"]
546     command = ["perl", "Tools/Scripts/build-jsc", WithProperties("--%(configuration)s")]
547
548
549 class CompileJSCOnlyToT(CompileJSCOnly):
550     name = 'build-jsc-tot'
551
552     def doStepIf(self, step):
553         return self.getProperty('patchFailedToBuild')
554
555     def hideStepIf(self, results, step):
556         return not self.doStepIf(step)
557
558
559 class RunJavaScriptCoreTests(shell.Test):
560     name = 'jscore-test'
561     description = ['jscore-tests running']
562     descriptionDone = ['jscore-tests']
563     flunkOnFailure = True
564     jsonFileName = 'jsc_results.json'
565     logfiles = {"json": jsonFileName}
566     command = ['perl', 'Tools/Scripts/run-javascriptcore-tests', '--no-build', '--no-fail-fast', '--json-output={0}'.format(jsonFileName), WithProperties('--%(configuration)s')]
567
568     def start(self):
569         appendCustomBuildFlags(self, self.getProperty('platform'), self.getProperty('fullPlatform'))
570         return shell.Test.start(self)
571
572     def evaluateCommand(self, cmd):
573         if cmd.didFail():
574             self.setProperty('patchFailedJSCTests', True)
575
576         return super(RunJavaScriptCoreTests, self).evaluateCommand(cmd)
577
578
579 class ReRunJavaScriptCoreTests(RunJavaScriptCoreTests):
580     name = 'jscore-test-rerun'
581
582     def doStepIf(self, step):
583         return self.getProperty('patchFailedJSCTests')
584
585     def hideStepIf(self, results, step):
586         return not self.doStepIf(step)
587
588     def evaluateCommand(self, cmd):
589         self.setProperty('patchFailedJSCTests', cmd.didFail())
590         return super(RunJavaScriptCoreTests, self).evaluateCommand(cmd)
591
592
593 class RunJavaScriptCoreTestsToT(RunJavaScriptCoreTests):
594     name = 'jscore-test-tot'
595     jsonFileName = 'jsc_results.json'
596     command = ['perl', 'Tools/Scripts/run-javascriptcore-tests', '--no-fail-fast', '--json-output={0}'.format(jsonFileName), WithProperties('--%(configuration)s')]
597
598     def doStepIf(self, step):
599         return self.getProperty('patchFailedJSCTests')
600
601     def hideStepIf(self, results, step):
602         return not self.doStepIf(step)
603
604
605 class CleanBuild(shell.Compile):
606     name = "delete-WebKitBuild-directory"
607     description = ["deleting WebKitBuild directory"]
608     descriptionDone = ["Deleted WebKitBuild directory"]
609     command = ["python", "Tools/BuildSlaveSupport/clean-build", WithProperties("--platform=%(fullPlatform)s"), WithProperties("--%(configuration)s")]
610
611
612 class KillOldProcesses(shell.Compile):
613     name = "kill-old-processes"
614     description = ["killing old processes"]
615     descriptionDone = ["Killed old processes"]
616     command = ["python", "Tools/BuildSlaveSupport/kill-old-processes", "buildbot"]
617
618     def __init__(self, **kwargs):
619         super(KillOldProcesses, self).__init__(timeout=60, **kwargs)
620
621
622 class RunWebKitTests(shell.Test):
623     name = 'layout-tests'
624     description = ['layout-tests running']
625     descriptionDone = ['layout-tests']
626     resultDirectory = 'layout-test-results'
627     command = ['python', 'Tools/Scripts/run-webkit-tests',
628                '--no-build',
629                '--no-new-test-results',
630                '--no-show-results',
631                '--exit-after-n-failures', '30',
632                '--skip-failing-tests',
633                WithProperties('--%(configuration)s')]
634
635     def start(self):
636         platform = self.getProperty('platform')
637         appendCustomBuildFlags(self, platform, self.getProperty('fullPlatform'))
638         additionalArguments = self.getProperty('additionalArguments')
639
640         self.setCommand(self.command + ['--results-directory', self.resultDirectory])
641         self.setCommand(self.command + ['--debug-rwt-logging'])
642
643         if additionalArguments:
644             self.setCommand(self.command + additionalArguments)
645         return shell.Test.start(self)
646
647
648 class RunWebKit1Tests(RunWebKitTests):
649     def start(self):
650         self.setCommand(self.command + ['--dump-render-tree'])
651
652         return RunWebKitTests.start(self)
653
654
655 class ArchiveBuiltProduct(shell.ShellCommand):
656     command = ['python', 'Tools/BuildSlaveSupport/built-product-archive',
657                WithProperties('--platform=%(fullPlatform)s'), WithProperties('--%(configuration)s'), 'archive']
658     name = 'archive-built-product'
659     description = ['archiving built product']
660     descriptionDone = ['Archived built product']
661     haltOnFailure = True
662
663
664 class UploadBuiltProduct(transfer.FileUpload):
665     name = 'upload-built-product'
666     workersrc = WithProperties('WebKitBuild/%(configuration)s.zip')
667     masterdest = WithProperties('public_html/archives/%(fullPlatform)s-%(architecture)s-%(configuration)s/%(patch_id)s.zip')
668     haltOnFailure = True
669
670     def __init__(self, **kwargs):
671         kwargs['workersrc'] = self.workersrc
672         kwargs['masterdest'] = self.masterdest
673         kwargs['mode'] = 0644
674         kwargs['blocksize'] = 1024 * 256
675         transfer.FileUpload.__init__(self, **kwargs)
676
677
678 class DownloadBuiltProduct(shell.ShellCommand):
679     command = ['python', 'Tools/BuildSlaveSupport/download-built-product',
680         WithProperties('--platform=%(platform)s'), WithProperties('--%(configuration)s'),
681         WithProperties(EWS_URL + 'archives/%(fullPlatform)s-%(architecture)s-%(configuration)s/%(patch_id)s.zip')]
682     name = 'download-built-product'
683     description = ['downloading built product']
684     descriptionDone = ['Downloaded built product']
685     haltOnFailure = True
686     flunkOnFailure = True
687
688
689 class ExtractBuiltProduct(shell.ShellCommand):
690     command = ['python', 'Tools/BuildSlaveSupport/built-product-archive',
691                WithProperties('--platform=%(fullPlatform)s'), WithProperties('--%(configuration)s'), 'extract']
692     name = 'extract-built-product'
693     description = ['extracting built product']
694     descriptionDone = ['Extracted built product']
695     haltOnFailure = True
696     flunkOnFailure = True
697
698
699 class RunAPITests(TestWithFailureCount):
700     name = 'run-api-tests'
701     description = ['api tests running']
702     descriptionDone = ['api-tests']
703     jsonFileName = 'api_test_results.json'
704     logfiles = {'json': jsonFileName}
705     command = ['python', 'Tools/Scripts/run-api-tests', '--no-build',
706                WithProperties('--%(configuration)s'), '--verbose', '--json-output={0}'.format(jsonFileName)]
707     failedTestsFormatString = '%d api test%s failed or timed out'
708
709     def start(self):
710         appendCustomBuildFlags(self, self.getProperty('platform'), self.getProperty('fullPlatform'))
711         return TestWithFailureCount.start(self)
712
713     def countFailures(self, cmd):
714         log_text = self.log_observer.getStdout() + self.log_observer.getStderr()
715
716         match = re.search(r'Ran (?P<ran>\d+) tests of (?P<total>\d+) with (?P<passed>\d+) successful', log_text)
717         if not match:
718             return 0
719         return int(match.group('ran')) - int(match.group('passed'))
720
721     def evaluateCommand(self, cmd):
722         rc = super(RunAPITests, self).evaluateCommand(cmd)
723         if rc == SUCCESS:
724             message = 'Passed API tests'
725             self.descriptionDone = message
726             self.build.results = SUCCESS
727             self.build.buildFinished([message], SUCCESS)
728         else:
729             self.build.addStepsAfterCurrentStep([ReRunAPITests()])
730         return rc
731
732
733 class ReRunAPITests(RunAPITests):
734     name = 're-run-api-tests'
735
736     def evaluateCommand(self, cmd):
737         rc = TestWithFailureCount.evaluateCommand(self, cmd)
738         if rc == SUCCESS:
739             message = 'Passed API tests'
740             self.descriptionDone = message
741             self.build.results = SUCCESS
742             self.build.buildFinished([message], SUCCESS)
743         else:
744             self.setProperty('patchFailedAPITests', True)
745             self.build.addStepsAfterCurrentStep([UnApplyPatchIfRequired(), CompileWebKitToT(), RunAPITestsWithoutPatch(), AnalyzeAPITestsResults()])
746         return rc
747
748
749 class RunAPITestsWithoutPatch(RunAPITests):
750     name = 'run-api-tests-without-patch'
751
752     def evaluateCommand(self, cmd):
753         return TestWithFailureCount.evaluateCommand(self, cmd)
754
755
756 class AnalyzeAPITestsResults(buildstep.BuildStep):
757     name = 'analyze-api-tests-results'
758     description = ['analyze-api-test-results']
759     descriptionDone = ['analyze-api-tests-results']
760
761     def start(self):
762         self.results = {}
763         d = self.getTestsResults(RunAPITests.name)
764         d.addCallback(lambda res: self.getTestsResults(ReRunAPITests.name))
765         d.addCallback(lambda res: self.getTestsResults(RunAPITestsWithoutPatch.name))
766         d.addCallback(lambda res: self.analyzeResults())
767         return defer.succeed(None)
768
769     def analyzeResults(self):
770         if not self.results or len(self.results) == 0:
771             self._addToLog('stderr', 'Unable to parse API test results: {}'.format(self.results))
772             self.finished(RETRY)
773             self.build.buildFinished(['Unable to parse API test results'], RETRY)
774             return -1
775
776         first_run_results = self.results.get(RunAPITests.name)
777         second_run_results = self.results.get(ReRunAPITests.name)
778         clean_tree_results = self.results.get(RunAPITestsWithoutPatch.name)
779
780         if not (first_run_results and second_run_results and clean_tree_results):
781             self.finished(RETRY)
782             self.build.buildFinished(['Unable to parse API test results'], RETRY)
783             return -1
784
785         def getAPITestFailures(result):
786             # TODO: Analyze Time-out, Crash and Failure independently
787             return set([failure.get('name') for failure in result.get('Timedout', [])] +
788                 [failure.get('name') for failure in result.get('Crashed', [])] +
789                 [failure.get('name') for failure in result.get('Failed', [])])
790
791         first_run_failures = getAPITestFailures(first_run_results)
792         second_run_failures = getAPITestFailures(second_run_results)
793         clean_tree_failures = getAPITestFailures(clean_tree_results)
794
795         self._addToLog('stderr', '\nFailures in API Test first run: {}'.format(first_run_failures))
796         self._addToLog('stderr', '\nFailures in API Test second run: {}'.format(first_run_failures))
797         self._addToLog('stderr', '\nFailures in API Test on clean tree: {}'.format(clean_tree_failures))
798         failures_with_patch = first_run_failures.intersection(second_run_failures)
799         new_failures = failures_with_patch - clean_tree_failures
800         new_failures_string = ', '.join([failure_name.replace('TestWebKitAPI.', '') for failure_name in new_failures])
801
802         if new_failures:
803             self._addToLog('stderr', '\nNew failures: {}\n'.format(new_failures))
804             self.finished(FAILURE)
805             self.build.results = FAILURE
806             message = 'Found {} new API Tests failures: {}'.format(len(new_failures), new_failures_string)
807             self.descriptionDone = message
808             self.build.buildFinished([message], FAILURE)
809         else:
810             self._addToLog('stderr', '\nNo new failures\n')
811             self.finished(SUCCESS)
812             self.build.results = SUCCESS
813             self.descriptionDone = 'Passed API tests'
814             message = 'Found {} pre-existing API tests failures'.format(len(clean_tree_failures))
815             self.build.buildFinished([message], SUCCESS)
816
817     @defer.inlineCallbacks
818     def _addToLog(self, logName, message):
819         try:
820             log = self.getLog(logName)
821         except KeyError:
822             log = yield self.addLog(logName)
823         log.addStdout(message)
824
825     def getBuildStepByName(self, name):
826         for step in self.build.executedSteps:
827             if step.name == name:
828                 return step
829         return None
830
831     @defer.inlineCallbacks
832     def getTestsResults(self, name):
833         step = self.getBuildStepByName(name)
834         if not step:
835             self._addToLog('stderr', 'ERROR: step not found: {}'.format(step))
836             defer.returnValue(None)
837
838         logs = yield self.master.db.logs.getLogs(step.stepid)
839         log = next((log for log in logs if log['name'] == u'json'), None)
840         if not log:
841             self._addToLog('stderr', 'ERROR: log for step not found: {}'.format(step))
842             defer.returnValue(None)
843
844         lastline = int(max(0, log['num_lines'] - 1))
845         logLines = yield self.master.db.logs.getLogLines(log['id'], 0, lastline)
846         if log['type'] == 's':
847             logLines = ''.join([line[1:] for line in logLines.splitlines()])
848
849         try:
850             self.results[name] = json.loads(logLines)
851         except Exception as ex:
852             self._addToLog('stderr', 'ERROR: unable to parse data, exception: {}'.format(ex))
853
854
855 class ArchiveTestResults(shell.ShellCommand):
856     command = ['python', 'Tools/BuildSlaveSupport/test-result-archive',
857                Interpolate('--platform=%(prop:platform)s'), Interpolate('--%(prop:configuration)s'), 'archive']
858     name = 'archive-test-results'
859     description = ['archiving test results']
860     descriptionDone = ['Archived test results']
861     haltOnFailure = True
862
863
864 class UploadTestResults(transfer.FileUpload):
865     name = 'upload-test-results'
866     descriptionDone = ['Uploaded test results']
867     workersrc = 'layout-test-results.zip'
868     masterdest = Interpolate('public_html/results/%(prop:buildername)s/r%(prop:patch_id)s-%(prop:buildnumber)s.zip')
869     haltOnFailure = True
870
871     def __init__(self, **kwargs):
872         kwargs['workersrc'] = self.workersrc
873         kwargs['masterdest'] = self.masterdest
874         kwargs['mode'] = 0644
875         kwargs['blocksize'] = 1024 * 256
876         transfer.FileUpload.__init__(self, **kwargs)
877
878
879 class ExtractTestResults(master.MasterShellCommand):
880     name = 'extract-test-results'
881     zipFile = Interpolate('public_html/results/%(prop:buildername)s/r%(prop:patch_id)s-%(prop:buildnumber)s.zip')
882     resultDirectory = Interpolate('public_html/results/%(prop:buildername)s/r%(prop:patch_id)s-%(prop:buildnumber)s')
883
884     descriptionDone = ['Extracted test results']
885     command = ['unzip', zipFile, '-d', resultDirectory]
886     renderables = ['resultDirectory']
887
888     def __init__(self):
889         super(ExtractTestResults, self).__init__(self.command)
890
891     def resultDirectoryURL(self):
892         return self.resultDirectory.replace('public_html/', '/') + '/'
893
894     def addCustomURLs(self):
895         self.addURL('view layout test results', self.resultDirectoryURL() + 'results.html')
896
897     def finished(self, result):
898         self.addCustomURLs()
899         return master.MasterShellCommand.finished(self, result)
900
901
902 class PrintConfiguration(steps.ShellSequence):
903     name = 'configuration'
904     description = ['configuration']
905     haltOnFailure = False
906     flunkOnFailure = False
907     warnOnFailure = False
908     logEnviron = False
909     command_list = [['hostname'],
910                     ['df', '-hl'],
911                     ['date'],
912                     ['sw_vers'],
913                     ['xcodebuild', '-sdk', '-version']]
914
915     def __init__(self, **kwargs):
916         super(PrintConfiguration, self).__init__(timeout=60, **kwargs)
917         self.commands = []
918         self.log_observer = logobserver.BufferLogObserver(wantStderr=True)
919         self.addLogObserver('stdio', self.log_observer)
920         # FIXME: Check platform before running platform specific commands.
921         for command in self.command_list:
922             self.commands.append(util.ShellArg(command=command, logfile='stdio'))
923
924     def convert_build_to_os_name(self, build):
925         if not build:
926             return 'Unknown'
927
928         build_to_name_mapping = {
929             '10.14': 'Mojave',
930             '10.13': 'High Sierra',
931             '10.12': 'Sierra',
932             '10.11': 'El Capitan',
933             '10.10': 'Yosemite',
934             '10.9': 'Maverick',
935             '10.8': 'Mountain Lion',
936             '10.7': 'Lion',
937             '10.6': 'Snow Leopard',
938             '10.5': 'Leopard',
939         }
940
941         for key, value in build_to_name_mapping.iteritems():
942             if build.startswith(key):
943                 return value
944         return 'Unknown'
945
946     def getResultSummary(self):
947         if self.results != SUCCESS:
948             return {u'step': u'Failed to print configuration'}
949         logText = self.log_observer.getStdout() + self.log_observer.getStderr()
950         configuration = u''
951         match = re.search('ProductVersion:[ \t]*(.+?)\n', logText)
952         if match:
953             os_version = match.group(1).strip()
954             os_name = self.convert_build_to_os_name(os_version)
955             configuration = u'OS: {} ({})'.format(os_name, os_version)
956
957         sdk_re = 'MacOSX[\s\S]*?SDKVersion:[ \t]*(.+?)\n'
958         is_ios_builder = 'iOS' in self.getProperty('buildername', '')
959         if is_ios_builder:
960             sdk_re = 'iPhoneSimulator[\s\S]*?SDKVersion:[ \t]*(.+?)\n'
961         match = re.search(sdk_re, logText)
962         if match:
963             xcode_version = match.group(1).strip()
964             configuration += u', Xcode: {}'.format(xcode_version)
965         return {u'step': configuration}