4d34b7c1df74de93200853127c774ef6de1c077d
[WebKit-https.git] / Tools / BuildSlaveSupport / ews-build / steps_unittest.py
1 # Copyright (C) 2018 Apple Inc. All rights reserved.
2 #
3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions
5 # are met:
6 # 1.  Redistributions of source code must retain the above copyright
7 #     notice, this list of conditions and the following disclaimer.
8 # 2.  Redistributions in binary form must reproduce the above copyright
9 #     notice, this list of conditions and the following disclaimer in the
10 #     documentation and/or other materials provided with the distribution.
11 #
12 # THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND
13 # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
14 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
15 # DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR
16 # ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
17 # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
18 # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
19 # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
20 # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
21 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
22
23 import operator
24 import os
25 import shutil
26 import tempfile
27
28 from buildbot.process import remotetransfer
29 from buildbot.process.results import Results, SUCCESS, FAILURE, WARNINGS, SKIPPED, EXCEPTION, RETRY
30 from buildbot.test.fake.remotecommand import Expect, ExpectRemoteRef, ExpectShell
31 from buildbot.test.util.steps import BuildStepMixin
32 from mock import call
33 from twisted.internet import error, reactor
34 from twisted.python import failure, log
35 from twisted.trial import unittest
36
37 from steps import *
38
39 # Workaround for https://github.com/buildbot/buildbot/issues/4669
40 from buildbot.test.fake.fakebuild import FakeBuild
41 FakeBuild.addStepsAfterCurrentStep = lambda FakeBuild, step_factories: None
42
43
44 class ExpectMasterShellCommand(object):
45     def __init__(self, command, workdir=None, env=None, usePTY=0):
46         self.args = command
47         self.usePTY = usePTY
48         self.rc = None
49         self.path = None
50         self.logs = []
51
52         if env is not None:
53             self.env = env
54         else:
55             self.env = os.environ
56         if workdir:
57             self.path = os.path.join(os.getcwd(), workdir)
58
59     @classmethod
60     def log(self, name, value):
61         return ('log', name, value)
62
63     def __add__(self, other):
64         if isinstance(other, int):
65             self.rc = other
66         elif isinstance(other, tuple) and other[0] == 'log':
67             self.logs.append((other[1], other[2]))
68         return self
69
70     def __repr__(self):
71         return 'ExpectMasterShellCommand({0})'.format(repr(self.args))
72
73
74 class BuildStepMixinAdditions(BuildStepMixin):
75     def setUpBuildStep(self):
76         self.patch(reactor, 'spawnProcess', lambda *args, **kwargs: self._checkSpawnProcess(*args, **kwargs))
77         self._expected_local_commands = []
78
79         self._temp_directory = tempfile.mkdtemp()
80         os.chdir(self._temp_directory)
81         self._expected_uploaded_files = []
82
83         super(BuildStepMixinAdditions, self).setUpBuildStep()
84
85     def tearDownBuildStep(self):
86         shutil.rmtree(self._temp_directory)
87         super(BuildStepMixinAdditions, self).tearDownBuildStep()
88
89     def fakeBuildFinished(self, text, results):
90         self.build.text = text
91         self.build.results = results
92
93     def setupStep(self, step, *args, **kwargs):
94         self.previous_steps = kwargs.get('previous_steps') or []
95         if self.previous_steps:
96             del kwargs['previous_steps']
97
98         super(BuildStepMixinAdditions, self).setupStep(step, *args, **kwargs)
99         self.build.terminate = False
100         self.build.stopped = False
101         self.build.executedSteps = self.executedSteps
102         self.build.buildFinished = self.fakeBuildFinished
103         self._expected_added_urls = []
104         self._expected_sources = None
105
106     @property
107     def executedSteps(self):
108         return filter(lambda step: not step.stopped, self.previous_steps)
109
110     def setProperty(self, name, value, source='Unknown'):
111         self.properties.setProperty(name, value, source)
112
113     def expectAddedURLs(self, added_urls):
114         self._expected_added_urls = added_urls
115
116     def expectUploadedFile(self, path):
117         self._expected_uploaded_files.append(path)
118
119     def expectLocalCommands(self, *expected_commands):
120         self._expected_local_commands.extend(expected_commands)
121
122     def expectRemoteCommands(self, *expected_commands):
123         self.expectCommands(*expected_commands)
124
125     def expectSources(self, expected_sources):
126         self._expected_sources = expected_sources
127
128     def _checkSpawnProcess(self, processProtocol, executable, args, env, path, usePTY, **kwargs):
129         got = (executable, args, env, path, usePTY)
130         if not self._expected_local_commands:
131             self.fail('got local command {0} when no further commands were expected'.format(got))
132         local_command = self._expected_local_commands.pop(0)
133         try:
134             self.assertEqual(got, (local_command.args[0], local_command.args, local_command.env, local_command.path, local_command.usePTY))
135         except AssertionError:
136             log.err()
137             raise
138         for name, value in local_command.logs:
139             if name == 'stdout':
140                 processProtocol.outReceived(value)
141             elif name == 'stderr':
142                 processProtocol.errReceived(value)
143         if local_command.rc != 0:
144             value = error.ProcessTerminated(exitCode=local_command.rc)
145         else:
146             value = error.ProcessDone(None)
147         processProtocol.processEnded(failure.Failure(value))
148
149     def _added_files(self):
150         results = []
151         for dirpath, dirnames, filenames in os.walk(self._temp_directory):
152             relative_root_path = os.path.relpath(dirpath, start=self._temp_directory)
153             if relative_root_path == '.':
154                 relative_root_path = ''
155             for name in filenames:
156                 results.append(os.path.join(relative_root_path, name))
157         return results
158
159     def runStep(self):
160         def check(result):
161             self.assertEqual(self._expected_local_commands, [], 'assert all expected local commands were run')
162             self.expectAddedURLs(self._expected_added_urls)
163             self.assertEqual(self._added_files(), self._expected_uploaded_files)
164             if self._expected_sources is not None:
165                 # Convert to dictionaries because assertEqual() only knows how to diff Python built-in types.
166                 actual_sources = sorted([source.asDict() for source in self.build.sources], key=operator.itemgetter('codebase'))
167                 expected_sources = sorted([source.asDict() for source in self._expected_sources], key=operator.itemgetter('codebase'))
168                 self.assertEqual(actual_sources, expected_sources)
169         deferred_result = super(BuildStepMixinAdditions, self).runStep()
170         deferred_result.addCallback(check)
171         return deferred_result
172
173
174 def uploadFileWithContentsOfString(string, timestamp=None):
175     def behavior(command):
176         writer = command.args['writer']
177         writer.remote_write(string + '\n')
178         writer.remote_close()
179         if timestamp:
180             writer.remote_utime(timestamp)
181     return behavior
182
183
184 class TestCheckStyle(BuildStepMixinAdditions, unittest.TestCase):
185     def setUp(self):
186         self.longMessage = True
187         return self.setUpBuildStep()
188
189     def tearDown(self):
190         return self.tearDownBuildStep()
191
192     def test_success_internal(self):
193         self.setupStep(CheckStyle())
194         self.setProperty('try-codebase', 'internal')
195         self.setProperty('platform', 'mac')
196         self.setProperty('configuration', 'debug')
197
198         self.expectRemoteCommands(
199             ExpectShell(workdir='wkdir',
200                         command=['Tools/Scripts/check-webkit-style'],
201                         )
202             + 0,
203         )
204         self.expectOutcome(result=SUCCESS, state_string='check-webkit-style')
205         return self.runStep()
206
207     def test_failure_unknown_try_codebase(self):
208         self.setupStep(CheckStyle())
209         self.setProperty('try-codebase', 'foo')
210         self.setProperty('platform', 'mac')
211         self.setProperty('configuration', 'debug')
212
213         self.expectRemoteCommands(
214             ExpectShell(workdir='wkdir',
215                         command=['Tools/Scripts/check-webkit-style'],
216                         )
217             + 2,
218         )
219         self.expectOutcome(result=FAILURE, state_string='check-webkit-style (failure)')
220         return self.runStep()
221
222     def test_failures_with_style_issues(self):
223         self.setupStep(CheckStyle())
224         self.setProperty('try-codebase', 'internal')
225         self.setProperty('platform', 'mac')
226         self.setProperty('configuration', 'debug')
227
228         self.expectRemoteCommands(
229             ExpectShell(workdir='wkdir',
230                         command=['Tools/Scripts/check-webkit-style'],
231                         )
232             + ExpectShell.log('stdio', stdout='''ERROR: Source/WebCore/layout/FloatingContext.cpp:36:  Code inside a namespace should not be indented.  [whitespace/indent] [4]
233 ERROR: Source/WebCore/layout/FormattingContext.h:94:  Weird number of spaces at line-start.  Are you using a 4-space indent?  [whitespace/indent] [3]
234 ERROR: Source/WebCore/layout/LayoutContext.cpp:52:  Place brace on its own line for function definitions.  [whitespace/braces] [4]
235 ERROR: Source/WebCore/layout/LayoutContext.cpp:55:  Extra space before last semicolon. If this should be an empty statement, use { } instead.  [whitespace/semicolon] [5]
236 ERROR: Source/WebCore/layout/LayoutContext.cpp:60:  Tab found; better to use spaces  [whitespace/tab] [1]
237 ERROR: Source/WebCore/layout/Verification.cpp:88:  Missing space before ( in while(  [whitespace/parens] [5]
238 Total errors found: 8 in 48 files''')
239             + 2,
240         )
241         self.expectOutcome(result=FAILURE, state_string='8 style errors (failure)')
242         return self.runStep()
243
244     def test_failures_no_style_issues(self):
245         self.setupStep(CheckStyle())
246         self.setProperty('try-codebase', 'internal')
247         self.setProperty('platform', 'mac')
248         self.setProperty('configuration', 'debug')
249
250         self.expectRemoteCommands(
251             ExpectShell(workdir='wkdir',
252                         command=['Tools/Scripts/check-webkit-style'],
253                         )
254             + ExpectShell.log('stdio', stdout='Total errors found: 0 in 6 files')
255             + 0,
256         )
257         self.expectOutcome(result=SUCCESS, state_string='check-webkit-style')
258         return self.runStep()
259
260     def test_failures_no_changes(self):
261         self.setupStep(CheckStyle())
262         self.setProperty('try-codebase', 'internal')
263         self.setProperty('platform', 'mac')
264         self.setProperty('configuration', 'debug')
265
266         self.expectRemoteCommands(
267             ExpectShell(workdir='wkdir',
268                         command=['Tools/Scripts/check-webkit-style'],
269                         )
270             + ExpectShell.log('stdio', stdout='Total errors found: 0 in 0 files')
271             + 2,
272         )
273         self.expectOutcome(result=FAILURE, state_string='check-webkit-style (failure)')
274         return self.runStep()
275
276
277 class TestRunBindingsTests(BuildStepMixinAdditions, unittest.TestCase):
278     def setUp(self):
279         self.longMessage = True
280         self.jsonFileName = 'bindings_test_results.json'
281         return self.setUpBuildStep()
282
283     def tearDown(self):
284         return self.tearDownBuildStep()
285
286     def test_success(self):
287         self.setupStep(RunBindingsTests())
288         self.expectRemoteCommands(
289             ExpectShell(workdir='wkdir',
290                         command=['Tools/Scripts/run-bindings-tests', '--json-output={0}'.format(self.jsonFileName)],
291                         logfiles={'json': self.jsonFileName},
292                         )
293             + 0,
294         )
295         self.expectOutcome(result=SUCCESS, state_string='bindings-tests')
296         return self.runStep()
297
298     def test_failure(self):
299         self.setupStep(RunBindingsTests())
300         self.expectRemoteCommands(
301             ExpectShell(workdir='wkdir',
302                         command=['Tools/Scripts/run-bindings-tests', '--json-output={0}'.format(self.jsonFileName)],
303                         logfiles={'json': self.jsonFileName},
304                         )
305             + ExpectShell.log('stdio', stdout='FAIL: (JS) JSTestInterface.cpp')
306             + 2,
307         )
308         self.expectOutcome(result=FAILURE, state_string='bindings-tests (failure)')
309         return self.runStep()
310
311
312 class TestunWebKitPerlTests(BuildStepMixinAdditions, unittest.TestCase):
313     def setUp(self):
314         self.longMessage = True
315         return self.setUpBuildStep()
316
317     def tearDown(self):
318         return self.tearDownBuildStep()
319
320     def test_success(self):
321         self.setupStep(RunWebKitPerlTests())
322         self.expectRemoteCommands(
323             ExpectShell(workdir='wkdir',
324                         command=['Tools/Scripts/test-webkitperl'],
325                         timeout=120,
326                         )
327             + 0,
328         )
329         self.expectOutcome(result=SUCCESS, state_string='webkitperl-tests')
330         return self.runStep()
331
332     def test_failure(self):
333         self.setupStep(RunWebKitPerlTests())
334         self.expectRemoteCommands(
335             ExpectShell(workdir='wkdir',
336                         command=['Tools/Scripts/test-webkitperl'],
337                         timeout=120,
338                         )
339             + ExpectShell.log('stdio', stdout='''Failed tests:  1-3, 5-7, 9, 11-13
340 Files=40, Tests=630,  4 wallclock secs ( 0.16 usr  0.09 sys +  2.78 cusr  0.64 csys =  3.67 CPU)
341 Result: FAIL
342 Failed 1/40 test programs. 10/630 subtests failed.''')
343             + 2,
344         )
345         self.expectOutcome(result=FAILURE, state_string='webkitperl-tests (failure)')
346         return self.runStep()
347
348
349 class TestWebKitPyTests(BuildStepMixinAdditions, unittest.TestCase):
350     def setUp(self):
351         self.longMessage = True
352         self.jsonFileName = 'webkitpy_test_results.json'
353         return self.setUpBuildStep()
354
355     def tearDown(self):
356         return self.tearDownBuildStep()
357
358     def test_success(self):
359         self.setupStep(RunWebKitPyTests())
360         self.expectRemoteCommands(
361             ExpectShell(workdir='wkdir',
362                         command=['Tools/Scripts/test-webkitpy', '--json-output={0}'.format(self.jsonFileName)],
363                         logfiles={'json': self.jsonFileName},
364                         timeout=120,
365                         )
366             + 0,
367         )
368         self.expectOutcome(result=SUCCESS, state_string='webkitpy-tests')
369         return self.runStep()
370
371     def test_failure(self):
372         self.setupStep(RunWebKitPyTests())
373         self.expectRemoteCommands(
374             ExpectShell(workdir='wkdir',
375                         command=['Tools/Scripts/test-webkitpy', '--json-output={0}'.format(self.jsonFileName)],
376                         logfiles={'json': self.jsonFileName},
377                         timeout=120,
378                         )
379             + ExpectShell.log('stdio', stdout='''Ran 1744 tests in 5.913s
380 FAILED (failures=1, errors=0)''')
381             + 2,
382         )
383         self.expectOutcome(result=FAILURE, state_string='webkitpy-tests (failure)')
384         return self.runStep()
385
386
387 class TestKillOldProcesses(BuildStepMixinAdditions, unittest.TestCase):
388     def setUp(self):
389         self.longMessage = True
390         return self.setUpBuildStep()
391
392     def tearDown(self):
393         return self.tearDownBuildStep()
394
395     def test_success(self):
396         self.setupStep(KillOldProcesses())
397         self.expectRemoteCommands(
398             ExpectShell(workdir='wkdir',
399                         command=['python', 'Tools/BuildSlaveSupport/kill-old-processes', 'buildbot'],
400                         timeout=60,
401                         )
402             + 0,
403         )
404         self.expectOutcome(result=SUCCESS, state_string='Killed old processes')
405         return self.runStep()
406
407     def test_failure(self):
408         self.setupStep(KillOldProcesses())
409         self.expectRemoteCommands(
410             ExpectShell(workdir='wkdir',
411                         command=['python', 'Tools/BuildSlaveSupport/kill-old-processes', 'buildbot'],
412                         timeout=60,
413                         )
414             + ExpectShell.log('stdio', stdout='Unexpected error.')
415             + 2,
416         )
417         self.expectOutcome(result=FAILURE, state_string='Killed old processes (failure)')
418         return self.runStep()
419
420
421 class TestCleanBuild(BuildStepMixinAdditions, unittest.TestCase):
422     def setUp(self):
423         self.longMessage = True
424         return self.setUpBuildStep()
425
426     def tearDown(self):
427         return self.tearDownBuildStep()
428
429     def test_success(self):
430         self.setupStep(CleanBuild())
431         self.setProperty('fullPlatform', 'ios-11')
432         self.setProperty('configuration', 'release')
433         self.expectRemoteCommands(
434             ExpectShell(workdir='wkdir',
435                         command=['python', 'Tools/BuildSlaveSupport/clean-build', '--platform=ios-11', '--release'],
436                         )
437             + 0,
438         )
439         self.expectOutcome(result=SUCCESS, state_string='Deleted WebKitBuild directory')
440         return self.runStep()
441
442     def test_failure(self):
443         self.setupStep(CleanBuild())
444         self.setProperty('fullPlatform', 'ios-simulator-11')
445         self.setProperty('configuration', 'debug')
446         self.expectRemoteCommands(
447             ExpectShell(workdir='wkdir',
448                         command=['python', 'Tools/BuildSlaveSupport/clean-build', '--platform=ios-simulator-11', '--debug'],
449                         )
450             + ExpectShell.log('stdio', stdout='Unexpected error.')
451             + 2,
452         )
453         self.expectOutcome(result=FAILURE, state_string='Deleted WebKitBuild directory (failure)')
454         return self.runStep()
455
456
457 class TestCompileWebKit(BuildStepMixinAdditions, unittest.TestCase):
458     def setUp(self):
459         self.longMessage = True
460         return self.setUpBuildStep()
461
462     def tearDown(self):
463         return self.tearDownBuildStep()
464
465     def test_success(self):
466         self.setupStep(CompileWebKit())
467         self.setProperty('fullPlatform', 'ios-simulator-11')
468         self.setProperty('configuration', 'release')
469         self.expectRemoteCommands(
470             ExpectShell(workdir='wkdir',
471                         command=["perl", "Tools/Scripts/build-webkit", '--release'],
472                         )
473             + 0,
474         )
475         self.expectOutcome(result=SUCCESS, state_string='Compiled WebKit')
476         return self.runStep()
477
478     def test_failure(self):
479         self.setupStep(CompileWebKit())
480         self.setProperty('fullPlatform', 'mac-sierra')
481         self.setProperty('configuration', 'debug')
482         self.expectRemoteCommands(
483             ExpectShell(workdir='wkdir',
484                         command=["perl", "Tools/Scripts/build-webkit", '--debug'],
485                         )
486             + ExpectShell.log('stdio', stdout='1 error generated.')
487             + 2,
488         )
489         self.expectOutcome(result=FAILURE, state_string='Compiled WebKit (failure)')
490         return self.runStep()
491
492
493 class TestCompileWebKitToT(BuildStepMixinAdditions, unittest.TestCase):
494     def setUp(self):
495         self.longMessage = True
496         return self.setUpBuildStep()
497
498     def tearDown(self):
499         return self.tearDownBuildStep()
500
501     def test_success(self):
502         self.setupStep(CompileWebKitToT())
503         self.setProperty('fullPlatform', 'ios-simulator-11')
504         self.setProperty('configuration', 'release')
505         self.setProperty('patchFailedToBuild', True)
506         self.expectRemoteCommands(
507             ExpectShell(workdir='wkdir',
508                         command=['perl', 'Tools/Scripts/build-webkit', '--release'],
509                         )
510             + 0,
511         )
512         self.expectOutcome(result=SUCCESS, state_string='Compiled WebKit')
513         return self.runStep()
514
515     def test_failure(self):
516         self.setupStep(CompileWebKitToT())
517         self.setProperty('fullPlatform', 'mac-sierra')
518         self.setProperty('configuration', 'debug')
519         self.setProperty('patchFailedToBuild', True)
520         self.expectRemoteCommands(
521             ExpectShell(workdir='wkdir',
522                         command=['perl', 'Tools/Scripts/build-webkit', '--debug'],
523                         )
524             + ExpectShell.log('stdio', stdout='1 error generated.')
525             + 2,
526         )
527         self.expectOutcome(result=FAILURE, state_string='Compiled WebKit (failure)')
528         return self.runStep()
529
530     def test_skip(self):
531         self.setupStep(CompileWebKitToT())
532         self.setProperty('fullPlatform', 'ios-simulator-11')
533         self.setProperty('configuration', 'release')
534         self.expectHidden(True)
535         self.expectOutcome(result=SKIPPED, state_string='Compiled WebKit (skipped)')
536         return self.runStep()
537
538
539 class TestCompileJSCOnly(BuildStepMixinAdditions, unittest.TestCase):
540     def setUp(self):
541         self.longMessage = True
542         return self.setUpBuildStep()
543
544     def tearDown(self):
545         return self.tearDownBuildStep()
546
547     def test_success(self):
548         self.setupStep(CompileJSCOnly())
549         self.setProperty('fullPlatform', 'jsc-only')
550         self.setProperty('configuration', 'release')
551         self.expectRemoteCommands(
552             ExpectShell(workdir='wkdir',
553                         command=["perl", "Tools/Scripts/build-jsc", '--release'],
554                         )
555             + 0,
556         )
557         self.expectOutcome(result=SUCCESS, state_string='Compiled JSC')
558         return self.runStep()
559
560     def test_failure(self):
561         self.setupStep(CompileJSCOnly())
562         self.setProperty('fullPlatform', 'jsc-only')
563         self.setProperty('configuration', 'debug')
564         self.expectRemoteCommands(
565             ExpectShell(workdir='wkdir',
566                         command=["perl", "Tools/Scripts/build-jsc", '--debug'],
567                         )
568             + ExpectShell.log('stdio', stdout='1 error generated.')
569             + 2,
570         )
571         self.expectOutcome(result=FAILURE, state_string='Compiled JSC (failure)')
572         return self.runStep()
573
574
575 class TestCompileJSCOnlyToT(BuildStepMixinAdditions, unittest.TestCase):
576     def setUp(self):
577         self.longMessage = True
578         return self.setUpBuildStep()
579
580     def tearDown(self):
581         return self.tearDownBuildStep()
582
583     def test_success(self):
584         self.setupStep(CompileJSCOnlyToT())
585         self.setProperty('fullPlatform', 'jsc-only')
586         self.setProperty('configuration', 'release')
587         self.setProperty('patchFailedToBuild', 'True')
588         self.expectRemoteCommands(
589             ExpectShell(workdir='wkdir',
590                         command=['perl', 'Tools/Scripts/build-jsc', '--release'],
591                         )
592             + 0,
593         )
594         self.expectOutcome(result=SUCCESS, state_string='Compiled JSC')
595         return self.runStep()
596
597     def test_failure(self):
598         self.setupStep(CompileJSCOnlyToT())
599         self.setProperty('fullPlatform', 'jsc-only')
600         self.setProperty('configuration', 'debug')
601         self.setProperty('patchFailedToBuild', 'True')
602         self.expectRemoteCommands(
603             ExpectShell(workdir='wkdir',
604                         command=['perl', 'Tools/Scripts/build-jsc', '--debug'],
605                         )
606             + ExpectShell.log('stdio', stdout='1 error generated.')
607             + 2,
608         )
609         self.expectOutcome(result=FAILURE, state_string='Compiled JSC (failure)')
610         return self.runStep()
611
612     def test_skip(self):
613         self.setupStep(CompileJSCOnlyToT())
614         self.setProperty('fullPlatform', 'jsc-only')
615         self.setProperty('configuration', 'debug')
616         self.expectHidden(True)
617         self.expectOutcome(result=SKIPPED, state_string='Compiled JSC (skipped)')
618         return self.runStep()
619
620
621 class TestRunJavaScriptCoreTests(BuildStepMixinAdditions, unittest.TestCase):
622     def setUp(self):
623         self.longMessage = True
624         self.jsonFileName = 'jsc_results.json'
625         return self.setUpBuildStep()
626
627     def tearDown(self):
628         return self.tearDownBuildStep()
629
630     def test_success(self):
631         self.setupStep(RunJavaScriptCoreTests())
632         self.setProperty('fullPlatform', 'jsc-only')
633         self.setProperty('configuration', 'release')
634         self.expectRemoteCommands(
635             ExpectShell(workdir='wkdir',
636                         command=['perl', 'Tools/Scripts/run-javascriptcore-tests', '--no-build', '--no-fail-fast', '--json-output={0}'.format(self.jsonFileName), '--release'],
637                         logfiles={'json': self.jsonFileName},
638                         )
639             + 0,
640         )
641         self.expectOutcome(result=SUCCESS, state_string='jscore-tests')
642         return self.runStep()
643
644     def test_failure(self):
645         self.setupStep(RunJavaScriptCoreTests())
646         self.setProperty('fullPlatform', 'jsc-only')
647         self.setProperty('configuration', 'debug')
648         self.expectRemoteCommands(
649             ExpectShell(workdir='wkdir',
650                         command=['perl', 'Tools/Scripts/run-javascriptcore-tests', '--no-build', '--no-fail-fast', '--json-output={0}'.format(self.jsonFileName), '--debug'],
651                         logfiles={'json': self.jsonFileName},
652                         )
653             + ExpectShell.log('stdio', stdout='9 failures found.')
654             + 2,
655         )
656         self.expectOutcome(result=FAILURE, state_string='jscore-tests (failure)')
657         return self.runStep()
658
659
660 class TestReRunJavaScriptCoreTests(BuildStepMixinAdditions, unittest.TestCase):
661     def setUp(self):
662         self.longMessage = True
663         self.jsonFileName = 'jsc_results.json'
664         return self.setUpBuildStep()
665
666     def tearDown(self):
667         return self.tearDownBuildStep()
668
669     def test_success(self):
670         self.setupStep(ReRunJavaScriptCoreTests())
671         self.setProperty('fullPlatform', 'jsc-only')
672         self.setProperty('configuration', 'release')
673         self.setProperty('patchFailedJSCTests', 'True')
674         self.expectRemoteCommands(
675             ExpectShell(workdir='wkdir',
676                         command=['perl', 'Tools/Scripts/run-javascriptcore-tests', '--no-build', '--no-fail-fast', '--json-output={0}'.format(self.jsonFileName), '--release'],
677                         logfiles={'json': self.jsonFileName},
678                         )
679             + 0,
680         )
681         self.expectOutcome(result=SUCCESS, state_string='jscore-tests')
682         return self.runStep()
683
684     def test_failure(self):
685         self.setupStep(ReRunJavaScriptCoreTests())
686         self.setProperty('fullPlatform', 'jsc-only')
687         self.setProperty('configuration', 'debug')
688         self.setProperty('patchFailedJSCTests', 'True')
689         self.expectRemoteCommands(
690             ExpectShell(workdir='wkdir',
691                         command=['perl', 'Tools/Scripts/run-javascriptcore-tests', '--no-build', '--no-fail-fast', '--json-output={0}'.format(self.jsonFileName), '--debug'],
692                         logfiles={'json': self.jsonFileName},
693                         )
694             + ExpectShell.log('stdio', stdout='9 failures found.')
695             + 2,
696         )
697         self.expectOutcome(result=FAILURE, state_string='jscore-tests (failure)')
698         return self.runStep()
699
700     def test_skip(self):
701         self.setupStep(ReRunJavaScriptCoreTests())
702         self.setProperty('fullPlatform', 'jsc-only')
703         self.setProperty('configuration', 'debug')
704         self.expectHidden(True)
705         self.expectOutcome(result=SKIPPED, state_string='jscore-tests (skipped)')
706         return self.runStep()
707
708
709 class TestRunJavaScriptCoreTestsToT(BuildStepMixinAdditions, unittest.TestCase):
710     def setUp(self):
711         self.longMessage = True
712         self.jsonFileName = 'jsc_results.json'
713         return self.setUpBuildStep()
714
715     def tearDown(self):
716         return self.tearDownBuildStep()
717
718     def test_success(self):
719         self.setupStep(RunJavaScriptCoreTestsToT())
720         self.setProperty('fullPlatform', 'jsc-only')
721         self.setProperty('configuration', 'release')
722         self.setProperty('patchFailedJSCTests', 'True')
723         self.expectRemoteCommands(
724             ExpectShell(workdir='wkdir',
725                         command=['perl', 'Tools/Scripts/run-javascriptcore-tests', '--no-fail-fast', '--json-output={0}'.format(self.jsonFileName), '--release'],
726                         logfiles={'json': self.jsonFileName},
727                         )
728             + 0,
729         )
730         self.expectOutcome(result=SUCCESS, state_string='jscore-tests')
731         return self.runStep()
732
733     def test_failure(self):
734         self.setupStep(RunJavaScriptCoreTestsToT())
735         self.setProperty('fullPlatform', 'jsc-only')
736         self.setProperty('configuration', 'debug')
737         self.setProperty('patchFailedJSCTests', 'True')
738         self.expectRemoteCommands(
739             ExpectShell(workdir='wkdir',
740                         command=['perl', 'Tools/Scripts/run-javascriptcore-tests', '--no-fail-fast', '--json-output={0}'.format(self.jsonFileName), '--debug'],
741                         logfiles={'json': self.jsonFileName},
742                         )
743             + ExpectShell.log('stdio', stdout='9 failures found.')
744             + 2,
745         )
746         self.expectOutcome(result=FAILURE, state_string='jscore-tests (failure)')
747         return self.runStep()
748
749     def test_skip(self):
750         self.setupStep(RunJavaScriptCoreTestsToT())
751         self.setProperty('fullPlatform', 'jsc-only')
752         self.setProperty('configuration', 'debug')
753         self.expectHidden(True)
754         self.expectOutcome(result=SKIPPED, state_string='jscore-tests (skipped)')
755         return self.runStep()
756
757
758 class TestRunWebKitTests(BuildStepMixinAdditions, unittest.TestCase):
759     def setUp(self):
760         self.longMessage = True
761         return self.setUpBuildStep()
762
763     def tearDown(self):
764         return self.tearDownBuildStep()
765
766     def test_success(self):
767         self.setupStep(RunWebKitTests())
768         self.setProperty('fullPlatform', 'ios-simulator')
769         self.setProperty('configuration', 'release')
770         self.expectRemoteCommands(
771             ExpectShell(workdir='wkdir',
772                         command=['python', 'Tools/Scripts/run-webkit-tests', '--no-build', '--no-new-test-results', '--no-show-results', '--exit-after-n-failures', '30', '--skip-failing-tests', '--release', '--results-directory', 'layout-test-results', '--debug-rwt-logging'],
773                         )
774             + 0,
775         )
776         self.expectOutcome(result=SUCCESS, state_string='layout-tests')
777         return self.runStep()
778
779     def test_failure(self):
780         self.setupStep(RunWebKitTests())
781         self.setProperty('fullPlatform', 'ios-simulator')
782         self.setProperty('configuration', 'release')
783         self.expectRemoteCommands(
784             ExpectShell(workdir='wkdir',
785                         command=['python', 'Tools/Scripts/run-webkit-tests', '--no-build', '--no-new-test-results', '--no-show-results', '--exit-after-n-failures', '30', '--skip-failing-tests', '--release', '--results-directory', 'layout-test-results', '--debug-rwt-logging'],
786                         )
787             + ExpectShell.log('stdio', stdout='9 failures found.')
788             + 2,
789         )
790         self.expectOutcome(result=FAILURE, state_string='layout-tests (failure)')
791         return self.runStep()
792
793
794 class TestRunWebKit1Tests(BuildStepMixinAdditions, unittest.TestCase):
795     def setUp(self):
796         self.longMessage = True
797         return self.setUpBuildStep()
798
799     def tearDown(self):
800         return self.tearDownBuildStep()
801
802     def test_success(self):
803         self.setupStep(RunWebKit1Tests())
804         self.setProperty('fullPlatform', 'ios-11')
805         self.setProperty('configuration', 'debug')
806         self.expectRemoteCommands(
807             ExpectShell(workdir='wkdir',
808                         command=['python', 'Tools/Scripts/run-webkit-tests', '--no-build', '--no-new-test-results', '--no-show-results', '--exit-after-n-failures', '30', '--skip-failing-tests', '--debug', '--dump-render-tree', '--results-directory', 'layout-test-results', '--debug-rwt-logging'],
809                         )
810             + 0,
811         )
812         self.expectOutcome(result=SUCCESS, state_string='layout-tests')
813         return self.runStep()
814
815     def test_failure(self):
816         self.setupStep(RunWebKit1Tests())
817         self.setProperty('fullPlatform', 'ios-11')
818         self.setProperty('configuration', 'release')
819         self.expectRemoteCommands(
820             ExpectShell(workdir='wkdir',
821                         command=['python', 'Tools/Scripts/run-webkit-tests', '--no-build', '--no-new-test-results', '--no-show-results', '--exit-after-n-failures', '30', '--skip-failing-tests', '--release', '--dump-render-tree', '--results-directory', 'layout-test-results', '--debug-rwt-logging'],
822                         )
823             + ExpectShell.log('stdio', stdout='9 failures found.')
824             + 2,
825         )
826         self.expectOutcome(result=FAILURE, state_string='layout-tests (failure)')
827         return self.runStep()
828
829
830 class TestArchiveBuiltProduct(BuildStepMixinAdditions, unittest.TestCase):
831     def setUp(self):
832         self.longMessage = True
833         return self.setUpBuildStep()
834
835     def tearDown(self):
836         return self.tearDownBuildStep()
837
838     def test_success(self):
839         self.setupStep(ArchiveBuiltProduct())
840         self.setProperty('fullPlatform', 'ios-simulator')
841         self.setProperty('configuration', 'release')
842         self.expectRemoteCommands(
843             ExpectShell(workdir='wkdir',
844                         command=['python', 'Tools/BuildSlaveSupport/built-product-archive', '--platform=ios-simulator',  '--release', 'archive'],
845                         )
846             + 0,
847         )
848         self.expectOutcome(result=SUCCESS, state_string='Archived built product')
849         return self.runStep()
850
851     def test_failure(self):
852         self.setupStep(ArchiveBuiltProduct())
853         self.setProperty('fullPlatform', 'mac-sierra')
854         self.setProperty('configuration', 'debug')
855         self.expectRemoteCommands(
856             ExpectShell(workdir='wkdir',
857                         command=['python', 'Tools/BuildSlaveSupport/built-product-archive', '--platform=mac-sierra',  '--debug', 'archive'],
858                         )
859             + ExpectShell.log('stdio', stdout='Unexpected failure.')
860             + 2,
861         )
862         self.expectOutcome(result=FAILURE, state_string='Archived built product (failure)')
863         return self.runStep()
864
865
866 class TestUploadBuiltProduct(BuildStepMixinAdditions, unittest.TestCase):
867     def setUp(self):
868         self.longMessage = True
869         return self.setUpBuildStep()
870
871     def tearDown(self):
872         return self.tearDownBuildStep()
873
874     def test_success(self):
875         self.setupStep(UploadBuiltProduct())
876         self.setProperty('fullPlatform', 'mac-sierra')
877         self.setProperty('configuration', 'release')
878         self.setProperty('architecture', 'x86_64')
879         self.setProperty('patch_id', '1234')
880         self.expectHidden(False)
881         self.expectRemoteCommands(
882             Expect('uploadFile', dict(
883                                         workersrc='WebKitBuild/release.zip', workdir='wkdir',
884                                         blocksize=1024 * 256, maxsize=None, keepstamp=False,
885                                         writer=ExpectRemoteRef(remotetransfer.FileWriter),
886                                      ))
887             + Expect.behavior(uploadFileWithContentsOfString('Dummy zip file content.'))
888             + 0,
889         )
890         self.expectUploadedFile('public_html/archives/mac-sierra-x86_64-release/1234.zip')
891
892         self.expectOutcome(result=SUCCESS, state_string='uploading release.zip')
893         return self.runStep()
894
895
896 class TestExtractBuiltProduct(BuildStepMixinAdditions, unittest.TestCase):
897     def setUp(self):
898         self.longMessage = True
899         return self.setUpBuildStep()
900
901     def tearDown(self):
902         return self.tearDownBuildStep()
903
904     def test_success(self):
905         self.setupStep(ExtractBuiltProduct())
906         self.setProperty('fullPlatform', 'ios-simulator')
907         self.setProperty('configuration', 'release')
908         self.expectRemoteCommands(
909             ExpectShell(workdir='wkdir',
910                         command=['python', 'Tools/BuildSlaveSupport/built-product-archive', '--platform=ios-simulator',  '--release', 'extract'],
911                         )
912             + 0,
913         )
914         self.expectOutcome(result=SUCCESS, state_string='Extracted built product')
915         return self.runStep()
916
917     def test_failure(self):
918         self.setupStep(ExtractBuiltProduct())
919         self.setProperty('fullPlatform', 'mac-sierra')
920         self.setProperty('configuration', 'debug')
921         self.expectRemoteCommands(
922             ExpectShell(workdir='wkdir',
923                         command=['python', 'Tools/BuildSlaveSupport/built-product-archive', '--platform=mac-sierra',  '--debug', 'extract'],
924                         )
925             + ExpectShell.log('stdio', stdout='Unexpected failure.')
926             + 2,
927         )
928         self.expectOutcome(result=FAILURE, state_string='Extracted built product (failure)')
929         return self.runStep()
930
931
932 class TestRunAPITests(BuildStepMixinAdditions, unittest.TestCase):
933     def setUp(self):
934         self.longMessage = True
935         self.jsonFileName = 'api_test_results.json'
936         return self.setUpBuildStep()
937
938     def tearDown(self):
939         return self.tearDownBuildStep()
940
941     def test_success_mac(self):
942         self.setupStep(RunAPITests())
943         self.setProperty('fullPlatform', 'mac-mojave')
944         self.setProperty('platform', 'mac')
945         self.setProperty('configuration', 'release')
946
947         self.expectRemoteCommands(
948             ExpectShell(workdir='wkdir',
949                         command=['python', 'Tools/Scripts/run-api-tests', '--no-build', '--release', '--verbose', '--json-output={0}'.format(self.jsonFileName)],
950                         logfiles={'json': self.jsonFileName},
951                         )
952             + ExpectShell.log('stdio', stdout='''...
953 worker/0 TestWTF.WTF_Variant.OperatorAmpersand Passed
954 worker/0 TestWTF.WTF_Variant.Ref Passed
955 worker/0 TestWTF.WTF_Variant.RefPtr Passed
956 worker/0 TestWTF.WTF_Variant.RetainPtr Passed
957 worker/0 TestWTF.WTF_Variant.VisitorUsingMakeVisitor Passed
958 worker/0 TestWTF.WTF_Variant.VisitorUsingSwitchOn Passed
959 Ran 1888 tests of 1888 with 1888 successful
960 ------------------------------
961 All tests successfully passed!
962 ''')
963             + 0,
964         )
965         self.expectOutcome(result=SUCCESS, state_string='run-api-tests')
966         return self.runStep()
967
968     def test_success_ios_simulator(self):
969         self.setupStep(RunAPITests())
970         self.setProperty('fullPlatform', 'ios-simulator-11')
971         self.setProperty('platform', 'ios')
972         self.setProperty('configuration', 'debug')
973
974         self.expectRemoteCommands(
975             ExpectShell(workdir='wkdir',
976                         command=['python', 'Tools/Scripts/run-api-tests', '--no-build', '--debug', '--verbose', '--json-output={0}'.format(self.jsonFileName), '--ios-simulator'],
977                         logfiles={'json': self.jsonFileName},
978                         )
979             + ExpectShell.log('stdio', stdout='''...
980 worker/0 TestWTF.WTF_Variant.OperatorAmpersand Passed
981 worker/0 TestWTF.WTF_Variant.Ref Passed
982 worker/0 TestWTF.WTF_Variant.RefPtr Passed
983 worker/0 TestWTF.WTF_Variant.RetainPtr Passed
984 worker/0 TestWTF.WTF_Variant.VisitorUsingMakeVisitor Passed
985 worker/0 TestWTF.WTF_Variant.VisitorUsingSwitchOn Passed
986 Ran 1888 tests of 1888 with 1888 successful
987 ------------------------------
988 All tests successfully passed!
989 ''')
990             + 0,
991         )
992         self.expectOutcome(result=SUCCESS, state_string='run-api-tests')
993         return self.runStep()
994
995     def test_one_failure(self):
996         self.setupStep(RunAPITests())
997         self.setProperty('fullPlatform', 'mac-mojave')
998         self.setProperty('platform', 'mac')
999         self.setProperty('configuration', 'debug')
1000
1001         self.expectRemoteCommands(
1002             ExpectShell(workdir='wkdir',
1003                         command=['python', 'Tools/Scripts/run-api-tests', '--no-build', '--debug', '--verbose', '--json-output={0}'.format(self.jsonFileName)],
1004                         logfiles={'json': self.jsonFileName},
1005                         )
1006             + ExpectShell.log('stdio', stdout='''
1007 worker/0 TestWTF.WTF_Variant.OperatorAmpersand Passed
1008 worker/0 TestWTF.WTF_Variant.Ref Passed
1009 worker/0 TestWTF.WTF_Variant.RefPtr Passed
1010 worker/0 TestWTF.WTF_Variant.RetainPtr Passed
1011 worker/0 TestWTF.WTF_Variant.VisitorUsingMakeVisitor Passed
1012 worker/0 TestWTF.WTF_Variant.VisitorUsingSwitchOn Passed
1013 worker/0 exiting
1014 Ran 1888 tests of 1888 with 1887 successful
1015 ------------------------------
1016 Test suite failed
1017
1018 Crashed
1019
1020     TestWTF.WTF.StringConcatenate_Unsigned
1021         **FAIL** WTF.StringConcatenate_Unsigned
1022
1023         Tools\\TestWebKitAPI\\Tests\\WTF\\StringConcatenate.cpp:84
1024         Value of: makeString('hello ', static_cast<unsigned short>(42) , ' world')
1025           Actual: hello 42 world
1026         Expected: 'hello * world'
1027         Which is: 74B00C9C
1028
1029 Testing completed, Exit status: 3
1030 ''')
1031             + 1,
1032         )
1033         self.expectOutcome(result=FAILURE, state_string='1 api test failed or timed out (failure)')
1034         return self.runStep()
1035
1036     def test_multiple_failures_and_timeouts(self):
1037         self.setupStep(RunAPITests())
1038         self.setProperty('fullPlatform', 'mac-mojave')
1039         self.setProperty('platform', 'mac')
1040         self.setProperty('configuration', 'debug')
1041
1042         self.expectRemoteCommands(
1043             ExpectShell(workdir='wkdir',
1044                         command=['python', 'Tools/Scripts/run-api-tests', '--no-build', '--debug', '--verbose', '--json-output={0}'.format(self.jsonFileName)],
1045                         logfiles={'json': self.jsonFileName},
1046                         )
1047             + ExpectShell.log('stdio', stdout='''...
1048 worker/0 TestWTF.WTF_Variant.OperatorAmpersand Passed
1049 worker/0 TestWTF.WTF_Variant.Ref Passed
1050 worker/0 TestWTF.WTF_Variant.RefPtr Passed
1051 worker/0 TestWTF.WTF_Variant.RetainPtr Passed
1052 worker/0 TestWTF.WTF_Variant.VisitorUsingMakeVisitor Passed
1053 worker/0 TestWTF.WTF_Variant.VisitorUsingSwitchOn Passed
1054 worker/0 exiting
1055 Ran 1888 tests of 1888 with 1884 successful
1056 ------------------------------
1057 Test suite failed
1058
1059 Failed
1060
1061     TestWTF.WTF.StringConcatenate_Unsigned
1062         **FAIL** WTF.StringConcatenate_Unsigned
1063
1064         Tools\\TestWebKitAPI\\Tests\\WTF\\StringConcatenate.cpp:84
1065         Value of: makeString('hello ', static_cast<unsigned short>(42) , ' world')
1066           Actual: hello 42 world
1067         Expected: 'hello * world'
1068         Which is: 74B00C9C
1069
1070     TestWTF.WTF_Expected.Unexpected
1071         **FAIL** WTF_Expected.Unexpected
1072
1073         Tools\TestWebKitAPI\Tests\WTF\Expected.cpp:96
1074         Value of: s1
1075           Actual: oops
1076         Expected: s0
1077         Which is: oops
1078
1079 Timeout
1080
1081     TestWTF.WTF_PoisonedUniquePtrForTriviallyDestructibleArrays.Assignment
1082     TestWTF.WTF_Lock.ContendedShortSection
1083
1084 Testing completed, Exit status: 3
1085 ''')
1086             + 4,
1087         )
1088         self.expectOutcome(result=FAILURE, state_string='4 api tests failed or timed out (failure)')
1089         return self.runStep()
1090
1091     def test_unexpecte_failure(self):
1092         self.setupStep(RunAPITests())
1093         self.setProperty('fullPlatform', 'mac-mojave')
1094         self.setProperty('platform', 'mac')
1095         self.setProperty('configuration', 'debug')
1096
1097         self.expectRemoteCommands(
1098             ExpectShell(workdir='wkdir',
1099                         command=['python', 'Tools/Scripts/run-api-tests', '--no-build', '--debug', '--verbose', '--json-output={0}'.format(self.jsonFileName)],
1100                         logfiles={'json': self.jsonFileName},
1101                         )
1102             + ExpectShell.log('stdio', stdout='Unexpected failure. Failed to run api tests.')
1103             + 2,
1104         )
1105         self.expectOutcome(result=FAILURE, state_string='run-api-tests (failure)')
1106         return self.runStep()
1107
1108     def test_no_failures_or_timeouts_with_disabled(self):
1109         self.setupStep(RunAPITests())
1110         self.setProperty('fullPlatform', 'mac-mojave')
1111         self.setProperty('platform', 'mac')
1112         self.setProperty('configuration', 'debug')
1113
1114         self.expectRemoteCommands(
1115             ExpectShell(workdir='wkdir',
1116                         command=['python', 'Tools/Scripts/run-api-tests', '--no-build', '--debug', '--verbose', '--json-output={0}'.format(self.jsonFileName)],
1117                         logfiles={'json': self.jsonFileName},
1118                         )
1119             + ExpectShell.log('stdio', stdout='''...
1120 worker/0 TestWTF.WTF_Variant.OperatorAmpersand Passed
1121 worker/0 TestWTF.WTF_Variant.Ref Passed
1122 worker/0 TestWTF.WTF_Variant.RefPtr Passed
1123 worker/0 TestWTF.WTF_Variant.RetainPtr Passed
1124 worker/0 TestWTF.WTF_Variant.VisitorUsingMakeVisitor Passed
1125 worker/0 TestWTF.WTF_Variant.VisitorUsingSwitchOn Passed
1126 worker/0 exiting
1127 Ran 1881 tests of 1888 with 1881 successful
1128 ------------------------------
1129 All tests successfully passed!
1130 ''')
1131             + 0,
1132         )
1133         self.expectOutcome(result=SUCCESS, state_string='run-api-tests')
1134         return self.runStep()
1135
1136
1137 class TestArchiveTestResults(BuildStepMixinAdditions, unittest.TestCase):
1138     def setUp(self):
1139         self.longMessage = True
1140         return self.setUpBuildStep()
1141
1142     def tearDown(self):
1143         return self.tearDownBuildStep()
1144
1145     def test_success(self):
1146         self.setupStep(ArchiveTestResults())
1147         self.setProperty('fullPlatform', 'ios-simulator')
1148         self.setProperty('platform', 'ios-simulator')
1149         self.setProperty('configuration', 'release')
1150         self.expectRemoteCommands(
1151             ExpectShell(workdir='wkdir',
1152                         command=['python', 'Tools/BuildSlaveSupport/test-result-archive', '--platform=ios-simulator',  '--release', 'archive'],
1153                         )
1154             + 0,
1155         )
1156         self.expectOutcome(result=SUCCESS, state_string='Archived test results')
1157         return self.runStep()
1158
1159     def test_failure(self):
1160         self.setupStep(ArchiveTestResults())
1161         self.setProperty('fullPlatform', 'mac-mojave')
1162         self.setProperty('platform', 'mac')
1163         self.setProperty('configuration', 'debug')
1164         self.expectRemoteCommands(
1165             ExpectShell(workdir='wkdir',
1166                         command=['python', 'Tools/BuildSlaveSupport/test-result-archive', '--platform=mac',  '--debug', 'archive'],
1167                         )
1168             + ExpectShell.log('stdio', stdout='Unexpected failure.')
1169             + 2,
1170         )
1171         self.expectOutcome(result=FAILURE, state_string='Archived test results (failure)')
1172         return self.runStep()
1173
1174
1175 class TestUploadTestResults(BuildStepMixinAdditions, unittest.TestCase):
1176     def setUp(self):
1177         self.longMessage = True
1178         return self.setUpBuildStep()
1179
1180     def tearDown(self):
1181         return self.tearDownBuildStep()
1182
1183     def test_success(self):
1184         self.setupStep(UploadTestResults())
1185         self.setProperty('configuration', 'release')
1186         self.setProperty('architecture', 'x86_64')
1187         self.setProperty('patch_id', '1234')
1188         self.setProperty('buildername', 'macOS-Sierra-Release-WK2-Tests-EWS')
1189         self.setProperty('buildnumber', '12')
1190         self.expectHidden(False)
1191         self.expectRemoteCommands(
1192             Expect('uploadFile', dict(
1193                                         workersrc='layout-test-results.zip', workdir='wkdir',
1194                                         blocksize=1024 * 256, maxsize=None, keepstamp=False,
1195                                         writer=ExpectRemoteRef(remotetransfer.FileWriter),
1196                                      ))
1197             + Expect.behavior(uploadFileWithContentsOfString('Dummy zip file content.'))
1198             + 0,
1199         )
1200         self.expectUploadedFile('public_html/results/macOS-Sierra-Release-WK2-Tests-EWS/r1234-12.zip')
1201
1202         self.expectOutcome(result=SUCCESS, state_string='Uploaded test results')
1203         return self.runStep()
1204
1205
1206 class TestExtractTestResults(BuildStepMixinAdditions, unittest.TestCase):
1207     def setUp(self):
1208         self.longMessage = True
1209         return self.setUpBuildStep()
1210
1211     def tearDown(self):
1212         return self.tearDownBuildStep()
1213
1214     def test_success(self):
1215         self.setupStep(ExtractTestResults())
1216         self.setProperty('configuration', 'release')
1217         self.setProperty('patch_id', '1234')
1218         self.setProperty('buildername', 'macOS-Sierra-Release-WK2-Tests-EWS')
1219         self.setProperty('buildnumber', '12')
1220         self.expectLocalCommands(
1221             ExpectMasterShellCommand(command=['unzip',
1222                                               'public_html/results/macOS-Sierra-Release-WK2-Tests-EWS/r1234-12.zip',
1223                                               '-d',
1224                                               'public_html/results/macOS-Sierra-Release-WK2-Tests-EWS/r1234-12',
1225                                              ])
1226             + 0,
1227         )
1228         self.expectOutcome(result=SUCCESS, state_string='Extracted test results')
1229         self.expectAddedURLs([call('view layout test results', '/results/test/r2468_ab1a28b4feee0d42973c7c05335b35bca927e974 (1)/results.html')])
1230         return self.runStep()
1231
1232     def test_failure(self):
1233         self.setupStep(ExtractTestResults())
1234         self.setProperty('configuration', 'debug')
1235         self.setProperty('patch_id', '1234')
1236         self.setProperty('buildername', 'macOS-Sierra-Release-WK2-Tests-EWS')
1237         self.setProperty('buildnumber', '12')
1238         self.expectLocalCommands(
1239             ExpectMasterShellCommand(command=['unzip',
1240                                               'public_html/results/macOS-Sierra-Release-WK2-Tests-EWS/r1234-12.zip',
1241                                               '-d',
1242                                               'public_html/results/macOS-Sierra-Release-WK2-Tests-EWS/r1234-12',
1243                                              ])
1244             + 2,
1245         )
1246         self.expectOutcome(result=FAILURE, state_string='failed (2) (failure)')
1247         self.expectAddedURLs([call('view layout test results', '/results/test/r2468_ab1a28b4feee0d42973c7c05335b35bca927e974 (1)/results.html')])
1248         return self.runStep()
1249
1250
1251 class TestPrintConfiguration(BuildStepMixinAdditions, unittest.TestCase):
1252     def setUp(self):
1253         self.longMessage = True
1254         return self.setUpBuildStep()
1255
1256     def tearDown(self):
1257         return self.tearDownBuildStep()
1258
1259     def test_success(self):
1260         self.setupStep(PrintConfiguration())
1261         self.setProperty('buildername', 'macOS-Sierra-Release-WK2-Tests-EWS')
1262         self.setProperty('platform', 'mac')
1263
1264         self.expectRemoteCommands(
1265             ExpectShell(command=['hostname'], workdir='wkdir', timeout=60, logEnviron=False) + 0
1266             + ExpectShell.log('stdio', stdout='ews150.apple.com'),
1267             ExpectShell(command=['df', '-hl'], workdir='wkdir', timeout=60, logEnviron=False) + 0
1268             + ExpectShell.log('stdio', stdout='''Filesystem     Size   Used  Avail Capacity iused  ifree %iused  Mounted on
1269 /dev/disk1s1  119Gi   95Gi   23Gi    81%  937959 9223372036853837848    0%   /
1270 /dev/disk1s4  119Gi   20Ki   23Gi     1%       0 9223372036854775807    0%   /private/var/vm
1271 /dev/disk0s3  119Gi   22Gi   97Gi    19%  337595          4294629684    0%   /Volumes/Data'''),
1272             ExpectShell(command=['date'], workdir='wkdir', timeout=60, logEnviron=False) + 0
1273             + ExpectShell.log('stdio', stdout='Tue Apr  9 15:30:52 PDT 2019'),
1274             ExpectShell(command=['sw_vers'], workdir='wkdir', timeout=60, logEnviron=False) + 0
1275             + ExpectShell.log('stdio', stdout='''ProductName:   Mac OS X
1276 ProductVersion: 10.13.4
1277 BuildVersion:   17E199'''),
1278             ExpectShell(command=['xcodebuild', '-sdk', '-version'], workdir='wkdir', timeout=60, logEnviron=False)
1279             + ExpectShell.log('stdio', stdout='''MacOSX10.13.sdk - macOS 10.13 (macosx10.13)
1280 SDKVersion: 10.13
1281 Path: /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX10.13.sdk
1282 PlatformVersion: 1.1
1283 PlatformPath: /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform
1284 ProductBuildVersion: 17E189
1285 ProductCopyright: 1983-2018 Apple Inc.
1286 ProductName: Mac OS X
1287 ProductUserVisibleVersion: 10.13.4
1288 ProductVersion: 10.13.4
1289
1290 Xcode 9.4.1
1291 Build version 9F2000''')
1292             + 0,
1293         )
1294         self.expectOutcome(result=SUCCESS, state_string='OS: High Sierra (10.13.4), Xcode: 10.13')
1295         return self.runStep()
1296
1297     def test_failure(self):
1298         self.setupStep(PrintConfiguration())
1299         self.setProperty('platform', 'mac')
1300         self.expectRemoteCommands(
1301             ExpectShell(command=['hostname'], workdir='wkdir', timeout=60, logEnviron=False) + 0,
1302             ExpectShell(command=['df', '-hl'], workdir='wkdir', timeout=60, logEnviron=False) + 0,
1303             ExpectShell(command=['date'], workdir='wkdir', timeout=60, logEnviron=False) + 0,
1304             ExpectShell(command=['sw_vers'], workdir='wkdir', timeout=60, logEnviron=False) + 1
1305             + ExpectShell.log('stdio', stdout='''Upon execvpe sw_vers ['sw_vers'] in environment id 7696545650400
1306 :Traceback (most recent call last):
1307   File "/usr/lib/python2.7/site-packages/twisted/internet/process.py", line 445, in _fork
1308     environment)
1309   File "/usr/lib/python2.7/site-packages/twisted/internet/process.py", line 523, in _execChild
1310     os.execvpe(executable, args, environment)
1311   File "/usr/lib/python2.7/os.py", line 355, in execvpe
1312     _execvpe(file, args, env)
1313   File "/usr/lib/python2.7/os.py", line 382, in _execvpe
1314     func(fullname, *argrest)
1315 OSError: [Errno 2] No such file or directory'''),
1316             ExpectShell(command=['xcodebuild', '-sdk', '-version'], workdir='wkdir', timeout=60, logEnviron=False)
1317             + ExpectShell.log('stdio', stdout='''Upon execvpe xcodebuild ['xcodebuild', '-sdk', '-version'] in environment id 7696545612416
1318 :Traceback (most recent call last):
1319   File "/usr/lib/python2.7/site-packages/twisted/internet/process.py", line 445, in _fork
1320     environment)
1321   File "/usr/lib/python2.7/site-packages/twisted/internet/process.py", line 523, in _execChild
1322     os.execvpe(executable, args, environment)
1323   File "/usr/lib/python2.7/os.py", line 355, in execvpe
1324     _execvpe(file, args, env)
1325   File "/usr/lib/python2.7/os.py", line 382, in _execvpe
1326     func(fullname, *argrest)
1327 OSError: [Errno 2] No such file or directory''')
1328             + 1,
1329         )
1330         self.expectOutcome(result=FAILURE, state_string='Failed to print configuration')
1331         return self.runStep()
1332
1333
1334 if __name__ == '__main__':
1335     unittest.main()