ba9254b02b32c7a11cdb419099376f87956aedd8
[WebKit-https.git] / Tools / Scripts / webkitpy / tool / bot / commitqueuetask_unittest.py
1 # Copyright (c) 2010 Google Inc. All rights reserved.
2 #
3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions are
5 # met:
6 #
7 #     * Redistributions of source code must retain the above copyright
8 # notice, this list of conditions and the following disclaimer.
9 #     * Redistributions in binary form must reproduce the above
10 # copyright notice, this list of conditions and the following disclaimer
11 # in the documentation and/or other materials provided with the
12 # distribution.
13 #     * Neither the name of Google Inc. nor the names of its
14 # contributors may be used to endorse or promote products derived from
15 # this software without specific prior written permission.
16 #
17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29 from datetime import datetime
30 import logging
31 import unittest2 as unittest
32
33 from webkitpy.common.net import bugzilla
34 from webkitpy.common.net.layouttestresults import LayoutTestResults
35 from webkitpy.common.system.executive import ScriptError
36 from webkitpy.common.system.outputcapture import OutputCapture
37 from webkitpy.layout_tests.models import test_results
38 from webkitpy.layout_tests.models import test_failures
39 from webkitpy.thirdparty.mock import Mock
40 from webkitpy.tool.bot.commitqueuetask import *
41 from webkitpy.tool.bot.expectedfailures import ExpectedFailures
42 from webkitpy.tool.mocktool import MockTool
43
44 _log = logging.getLogger(__name__)
45
46
47 class MockCommitQueue(CommitQueueTaskDelegate):
48     def __init__(self, error_plan):
49         self._error_plan = error_plan
50         self._failure_status_id = 0
51
52     def run_command(self, command):
53         _log.info("run_webkit_patch: %s" % command)
54         if self._error_plan:
55             error = self._error_plan.pop(0)
56             if error:
57                 raise error
58
59     def command_passed(self, success_message, patch):
60         _log.info("command_passed: success_message='%s' patch='%s'" % (
61             success_message, patch.id()))
62
63     def command_failed(self, failure_message, script_error, patch):
64         _log.info("command_failed: failure_message='%s' script_error='%s' patch='%s'" % (
65             failure_message, script_error, patch.id()))
66         self._failure_status_id += 1
67         return self._failure_status_id
68
69     def refetch_patch(self, patch):
70         return patch
71
72     def expected_failures(self):
73         return ExpectedFailures()
74
75     def test_results(self):
76         return None
77
78     def report_flaky_tests(self, patch, flaky_results, results_archive):
79         flaky_tests = [result.filename for result in flaky_results]
80         _log.info("report_flaky_tests: patch='%s' flaky_tests='%s' archive='%s'" % (patch.id(), flaky_tests, results_archive.filename))
81
82     def archive_last_test_results(self, patch):
83         _log.info("archive_last_test_results: patch='%s'" % patch.id())
84         archive = Mock()
85         archive.filename = "mock-archive-%s.zip" % patch.id()
86         return archive
87
88     def build_style(self):
89         return "both"
90
91     def did_pass_testing_ews(self, patch):
92         return False
93
94
95 class FailingTestCommitQueue(MockCommitQueue):
96     def __init__(self, error_plan, test_failure_plan):
97         MockCommitQueue.__init__(self, error_plan)
98         self._test_run_counter = -1  # Special value to indicate tests have never been run.
99         self._test_failure_plan = test_failure_plan
100
101     def run_command(self, command):
102         if command[0] == "build-and-test":
103             self._test_run_counter += 1
104         MockCommitQueue.run_command(self, command)
105
106     def _mock_test_result(self, testname):
107         return test_results.TestResult(testname, [test_failures.FailureTextMismatch()])
108
109     def test_results(self):
110         # Doesn't make sense to ask for the test_results until the tests have run at least once.
111         assert(self._test_run_counter >= 0)
112         failures_for_run = self._test_failure_plan[self._test_run_counter]
113         results = LayoutTestResults(map(self._mock_test_result, failures_for_run))
114         # This makes the results trustable by ExpectedFailures.
115         results.set_failure_limit_count(10)
116         return results
117
118
119 # We use GoldenScriptError to make sure that the code under test throws the
120 # correct (i.e., golden) exception.
121 class GoldenScriptError(ScriptError):
122     pass
123
124
125 class CommitQueueTaskTest(unittest.TestCase):
126     def _run_through_task(self, commit_queue, expected_logs, expected_exception=None, expect_retry=False):
127         tool = MockTool(log_executive=True)
128         patch = tool.bugs.fetch_attachment(10000)
129         task = CommitQueueTask(commit_queue, patch)
130         success = OutputCapture().assert_outputs(self, task.run, expected_logs=expected_logs, expected_exception=expected_exception)
131         if not expected_exception:
132             self.assertEqual(success, not expect_retry)
133         return task
134
135     def test_success_case(self):
136         commit_queue = MockCommitQueue([])
137         expected_logs = """run_webkit_patch: ['clean']
138 command_passed: success_message='Cleaned working directory' patch='10000'
139 run_webkit_patch: ['update']
140 command_passed: success_message='Updated working directory' patch='10000'
141 run_webkit_patch: ['apply-attachment', '--no-update', '--non-interactive', 10000]
142 command_passed: success_message='Applied patch' patch='10000'
143 run_webkit_patch: ['validate-changelog', '--non-interactive', 10000]
144 command_passed: success_message='ChangeLog validated' patch='10000'
145 run_webkit_patch: ['build', '--no-clean', '--no-update', '--build-style=both']
146 command_passed: success_message='Built patch' patch='10000'
147 run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--non-interactive']
148 command_passed: success_message='Passed tests' patch='10000'
149 run_webkit_patch: ['land-attachment', '--force-clean', '--non-interactive', '--parent-command=commit-queue', 10000]
150 command_passed: success_message='Landed patch' patch='10000'
151 """
152         self._run_through_task(commit_queue, expected_logs)
153
154     def test_fast_success_case(self):
155         commit_queue = MockCommitQueue([])
156         commit_queue.did_pass_testing_ews = lambda patch: True
157         expected_logs = """run_webkit_patch: ['clean']
158 command_passed: success_message='Cleaned working directory' patch='10000'
159 run_webkit_patch: ['update']
160 command_passed: success_message='Updated working directory' patch='10000'
161 run_webkit_patch: ['apply-attachment', '--no-update', '--non-interactive', 10000]
162 command_passed: success_message='Applied patch' patch='10000'
163 run_webkit_patch: ['validate-changelog', '--non-interactive', 10000]
164 command_passed: success_message='ChangeLog validated' patch='10000'
165 run_webkit_patch: ['build', '--no-clean', '--no-update', '--build-style=both']
166 command_passed: success_message='Built patch' patch='10000'
167 run_webkit_patch: ['land-attachment', '--force-clean', '--non-interactive', '--parent-command=commit-queue', 10000]
168 command_passed: success_message='Landed patch' patch='10000'
169 """
170         self._run_through_task(commit_queue, expected_logs)
171
172     def test_clean_failure(self):
173         commit_queue = MockCommitQueue([
174             ScriptError("MOCK clean failure"),
175         ])
176         expected_logs = """run_webkit_patch: ['clean']
177 command_failed: failure_message='Unable to clean working directory' script_error='MOCK clean failure' patch='10000'
178 """
179         self._run_through_task(commit_queue, expected_logs, expect_retry=True)
180
181     def test_update_failure(self):
182         commit_queue = MockCommitQueue([
183             None,
184             ScriptError("MOCK update failure"),
185         ])
186         expected_logs = """run_webkit_patch: ['clean']
187 command_passed: success_message='Cleaned working directory' patch='10000'
188 run_webkit_patch: ['update']
189 command_failed: failure_message='Unable to update working directory' script_error='MOCK update failure' patch='10000'
190 """
191         self._run_through_task(commit_queue, expected_logs, expect_retry=True)
192
193     def test_apply_failure(self):
194         commit_queue = MockCommitQueue([
195             None,
196             None,
197             GoldenScriptError("MOCK apply failure"),
198         ])
199         expected_logs = """run_webkit_patch: ['clean']
200 command_passed: success_message='Cleaned working directory' patch='10000'
201 run_webkit_patch: ['update']
202 command_passed: success_message='Updated working directory' patch='10000'
203 run_webkit_patch: ['apply-attachment', '--no-update', '--non-interactive', 10000]
204 command_failed: failure_message='Patch does not apply' script_error='MOCK apply failure' patch='10000'
205 """
206         self._run_through_task(commit_queue, expected_logs, GoldenScriptError)
207
208     def test_validate_changelog_failure(self):
209         commit_queue = MockCommitQueue([
210             None,
211             None,
212             None,
213             GoldenScriptError("MOCK validate failure"),
214         ])
215         expected_logs = """run_webkit_patch: ['clean']
216 command_passed: success_message='Cleaned working directory' patch='10000'
217 run_webkit_patch: ['update']
218 command_passed: success_message='Updated working directory' patch='10000'
219 run_webkit_patch: ['apply-attachment', '--no-update', '--non-interactive', 10000]
220 command_passed: success_message='Applied patch' patch='10000'
221 run_webkit_patch: ['validate-changelog', '--non-interactive', 10000]
222 command_failed: failure_message='ChangeLog did not pass validation' script_error='MOCK validate failure' patch='10000'
223 """
224         self._run_through_task(commit_queue, expected_logs, GoldenScriptError)
225
226     def test_build_failure(self):
227         commit_queue = MockCommitQueue([
228             None,
229             None,
230             None,
231             None,
232             GoldenScriptError("MOCK build failure"),
233         ])
234         expected_logs = """run_webkit_patch: ['clean']
235 command_passed: success_message='Cleaned working directory' patch='10000'
236 run_webkit_patch: ['update']
237 command_passed: success_message='Updated working directory' patch='10000'
238 run_webkit_patch: ['apply-attachment', '--no-update', '--non-interactive', 10000]
239 command_passed: success_message='Applied patch' patch='10000'
240 run_webkit_patch: ['validate-changelog', '--non-interactive', 10000]
241 command_passed: success_message='ChangeLog validated' patch='10000'
242 run_webkit_patch: ['build', '--no-clean', '--no-update', '--build-style=both']
243 command_failed: failure_message='Patch does not build' script_error='MOCK build failure' patch='10000'
244 run_webkit_patch: ['build', '--force-clean', '--no-update', '--build-style=both']
245 command_passed: success_message='Able to build without patch' patch='10000'
246 """
247         self._run_through_task(commit_queue, expected_logs, GoldenScriptError)
248
249     def test_red_build_failure(self):
250         commit_queue = MockCommitQueue([
251             None,
252             None,
253             None,
254             None,
255             ScriptError("MOCK build failure"),
256             ScriptError("MOCK clean build failure"),
257         ])
258         expected_logs = """run_webkit_patch: ['clean']
259 command_passed: success_message='Cleaned working directory' patch='10000'
260 run_webkit_patch: ['update']
261 command_passed: success_message='Updated working directory' patch='10000'
262 run_webkit_patch: ['apply-attachment', '--no-update', '--non-interactive', 10000]
263 command_passed: success_message='Applied patch' patch='10000'
264 run_webkit_patch: ['validate-changelog', '--non-interactive', 10000]
265 command_passed: success_message='ChangeLog validated' patch='10000'
266 run_webkit_patch: ['build', '--no-clean', '--no-update', '--build-style=both']
267 command_failed: failure_message='Patch does not build' script_error='MOCK build failure' patch='10000'
268 run_webkit_patch: ['build', '--force-clean', '--no-update', '--build-style=both']
269 command_failed: failure_message='Unable to build without patch' script_error='MOCK clean build failure' patch='10000'
270 """
271         self._run_through_task(commit_queue, expected_logs, expect_retry=True)
272
273     def test_flaky_test_failure(self):
274         commit_queue = MockCommitQueue([
275             None,
276             None,
277             None,
278             None,
279             None,
280             ScriptError("MOCK tests failure"),
281         ])
282         # CommitQueueTask will only report flaky tests if we successfully parsed
283         # results.json and returned a LayoutTestResults object, so we fake one.
284         commit_queue.test_results = lambda: LayoutTestResults([])
285         expected_logs = """run_webkit_patch: ['clean']
286 command_passed: success_message='Cleaned working directory' patch='10000'
287 run_webkit_patch: ['update']
288 command_passed: success_message='Updated working directory' patch='10000'
289 run_webkit_patch: ['apply-attachment', '--no-update', '--non-interactive', 10000]
290 command_passed: success_message='Applied patch' patch='10000'
291 run_webkit_patch: ['validate-changelog', '--non-interactive', 10000]
292 command_passed: success_message='ChangeLog validated' patch='10000'
293 run_webkit_patch: ['build', '--no-clean', '--no-update', '--build-style=both']
294 command_passed: success_message='Built patch' patch='10000'
295 run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--non-interactive']
296 command_failed: failure_message='Patch does not pass tests' script_error='MOCK tests failure' patch='10000'
297 archive_last_test_results: patch='10000'
298 run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--non-interactive']
299 command_passed: success_message='Passed tests' patch='10000'
300 report_flaky_tests: patch='10000' flaky_tests='[]' archive='mock-archive-10000.zip'
301 run_webkit_patch: ['land-attachment', '--force-clean', '--non-interactive', '--parent-command=commit-queue', 10000]
302 command_passed: success_message='Landed patch' patch='10000'
303 """
304         self._run_through_task(commit_queue, expected_logs)
305
306     def test_failed_archive(self):
307         commit_queue = MockCommitQueue([
308             None,
309             None,
310             None,
311             None,
312             None,
313             ScriptError("MOCK tests failure"),
314         ])
315         commit_queue.test_results = lambda: LayoutTestResults([])
316         # It's possible delegate to fail to archive layout tests, don't try to report
317         # flaky tests when that happens.
318         commit_queue.archive_last_test_results = lambda patch: None
319         expected_logs = """run_webkit_patch: ['clean']
320 command_passed: success_message='Cleaned working directory' patch='10000'
321 run_webkit_patch: ['update']
322 command_passed: success_message='Updated working directory' patch='10000'
323 run_webkit_patch: ['apply-attachment', '--no-update', '--non-interactive', 10000]
324 command_passed: success_message='Applied patch' patch='10000'
325 run_webkit_patch: ['validate-changelog', '--non-interactive', 10000]
326 command_passed: success_message='ChangeLog validated' patch='10000'
327 run_webkit_patch: ['build', '--no-clean', '--no-update', '--build-style=both']
328 command_passed: success_message='Built patch' patch='10000'
329 run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--non-interactive']
330 command_failed: failure_message='Patch does not pass tests' script_error='MOCK tests failure' patch='10000'
331 run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--non-interactive']
332 command_passed: success_message='Passed tests' patch='10000'
333 run_webkit_patch: ['land-attachment', '--force-clean', '--non-interactive', '--parent-command=commit-queue', 10000]
334 command_passed: success_message='Landed patch' patch='10000'
335 """
336         self._run_through_task(commit_queue, expected_logs)
337
338     def test_double_flaky_test_failure(self):
339         commit_queue = FailingTestCommitQueue([
340             None,
341             None,
342             None,
343             None,
344             None,
345             ScriptError("MOCK test failure"),
346             ScriptError("MOCK test failure again"),
347         ], [
348             "foo.html",
349             "bar.html",
350             "foo.html",
351         ])
352         # The (subtle) point of this test is that report_flaky_tests does not appear
353         # in the expected_logs for this run.
354         # Note also that there is no attempt to run the tests w/o the patch.
355         expected_logs = """run_webkit_patch: ['clean']
356 command_passed: success_message='Cleaned working directory' patch='10000'
357 run_webkit_patch: ['update']
358 command_passed: success_message='Updated working directory' patch='10000'
359 run_webkit_patch: ['apply-attachment', '--no-update', '--non-interactive', 10000]
360 command_passed: success_message='Applied patch' patch='10000'
361 run_webkit_patch: ['validate-changelog', '--non-interactive', 10000]
362 command_passed: success_message='ChangeLog validated' patch='10000'
363 run_webkit_patch: ['build', '--no-clean', '--no-update', '--build-style=both']
364 command_passed: success_message='Built patch' patch='10000'
365 run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--non-interactive']
366 command_failed: failure_message='Patch does not pass tests' script_error='MOCK test failure' patch='10000'
367 archive_last_test_results: patch='10000'
368 run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--non-interactive']
369 command_failed: failure_message='Patch does not pass tests' script_error='MOCK test failure again' patch='10000'
370 """
371         tool = MockTool(log_executive=True)
372         patch = tool.bugs.fetch_attachment(10000)
373         task = CommitQueueTask(commit_queue, patch)
374         success = OutputCapture().assert_outputs(self, task.run, expected_logs=expected_logs)
375         self.assertFalse(success)
376
377     def test_test_failure(self):
378         commit_queue = MockCommitQueue([
379             None,
380             None,
381             None,
382             None,
383             None,
384             GoldenScriptError("MOCK test failure"),
385             ScriptError("MOCK test failure again"),
386         ])
387         expected_logs = """run_webkit_patch: ['clean']
388 command_passed: success_message='Cleaned working directory' patch='10000'
389 run_webkit_patch: ['update']
390 command_passed: success_message='Updated working directory' patch='10000'
391 run_webkit_patch: ['apply-attachment', '--no-update', '--non-interactive', 10000]
392 command_passed: success_message='Applied patch' patch='10000'
393 run_webkit_patch: ['validate-changelog', '--non-interactive', 10000]
394 command_passed: success_message='ChangeLog validated' patch='10000'
395 run_webkit_patch: ['build', '--no-clean', '--no-update', '--build-style=both']
396 command_passed: success_message='Built patch' patch='10000'
397 run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--non-interactive']
398 command_failed: failure_message='Patch does not pass tests' script_error='MOCK test failure' patch='10000'
399 archive_last_test_results: patch='10000'
400 run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--non-interactive']
401 command_failed: failure_message='Patch does not pass tests' script_error='MOCK test failure again' patch='10000'
402 archive_last_test_results: patch='10000'
403 run_webkit_patch: ['build-and-test', '--force-clean', '--no-update', '--build', '--test', '--non-interactive']
404 command_passed: success_message='Able to pass tests without patch' patch='10000'
405 """
406         self._run_through_task(commit_queue, expected_logs, GoldenScriptError)
407
408     def test_red_test_failure(self):
409         commit_queue = FailingTestCommitQueue([
410             None,
411             None,
412             None,
413             None,
414             None,
415             ScriptError("MOCK test failure"),
416             ScriptError("MOCK test failure again"),
417             ScriptError("MOCK clean test failure"),
418         ], [
419             "foo.html",
420             "foo.html",
421             "foo.html",
422         ])
423
424         # Tests always fail, and always return the same results, but we
425         # should still be able to land in this case!
426         expected_logs = """run_webkit_patch: ['clean']
427 command_passed: success_message='Cleaned working directory' patch='10000'
428 run_webkit_patch: ['update']
429 command_passed: success_message='Updated working directory' patch='10000'
430 run_webkit_patch: ['apply-attachment', '--no-update', '--non-interactive', 10000]
431 command_passed: success_message='Applied patch' patch='10000'
432 run_webkit_patch: ['validate-changelog', '--non-interactive', 10000]
433 command_passed: success_message='ChangeLog validated' patch='10000'
434 run_webkit_patch: ['build', '--no-clean', '--no-update', '--build-style=both']
435 command_passed: success_message='Built patch' patch='10000'
436 run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--non-interactive']
437 command_failed: failure_message='Patch does not pass tests' script_error='MOCK test failure' patch='10000'
438 archive_last_test_results: patch='10000'
439 run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--non-interactive']
440 command_failed: failure_message='Patch does not pass tests' script_error='MOCK test failure again' patch='10000'
441 archive_last_test_results: patch='10000'
442 run_webkit_patch: ['build-and-test', '--force-clean', '--no-update', '--build', '--test', '--non-interactive']
443 command_failed: failure_message='Unable to pass tests without patch (tree is red?)' script_error='MOCK clean test failure' patch='10000'
444 run_webkit_patch: ['land-attachment', '--force-clean', '--non-interactive', '--parent-command=commit-queue', 10000]
445 command_passed: success_message='Landed patch' patch='10000'
446 """
447         self._run_through_task(commit_queue, expected_logs)
448
449     def test_very_red_tree_retry(self):
450         lots_of_failing_tests = map(lambda num: "test-%s.html" % num, range(0, 100))
451         commit_queue = FailingTestCommitQueue([
452             None,
453             None,
454             None,
455             None,
456             None,
457             ScriptError("MOCK test failure"),
458             ScriptError("MOCK test failure again"),
459             ScriptError("MOCK clean test failure"),
460         ], [
461             lots_of_failing_tests,
462             lots_of_failing_tests,
463             lots_of_failing_tests,
464         ])
465
466         # Tests always fail, and return so many failures that we do not
467         # trust the results (see ExpectedFailures._can_trust_results) so we
468         # just give up and retry the patch.
469         expected_logs = """run_webkit_patch: ['clean']
470 command_passed: success_message='Cleaned working directory' patch='10000'
471 run_webkit_patch: ['update']
472 command_passed: success_message='Updated working directory' patch='10000'
473 run_webkit_patch: ['apply-attachment', '--no-update', '--non-interactive', 10000]
474 command_passed: success_message='Applied patch' patch='10000'
475 run_webkit_patch: ['validate-changelog', '--non-interactive', 10000]
476 command_passed: success_message='ChangeLog validated' patch='10000'
477 run_webkit_patch: ['build', '--no-clean', '--no-update', '--build-style=both']
478 command_passed: success_message='Built patch' patch='10000'
479 run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--non-interactive']
480 command_failed: failure_message='Patch does not pass tests' script_error='MOCK test failure' patch='10000'
481 archive_last_test_results: patch='10000'
482 run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--non-interactive']
483 command_failed: failure_message='Patch does not pass tests' script_error='MOCK test failure again' patch='10000'
484 archive_last_test_results: patch='10000'
485 run_webkit_patch: ['build-and-test', '--force-clean', '--no-update', '--build', '--test', '--non-interactive']
486 command_failed: failure_message='Unable to pass tests without patch (tree is red?)' script_error='MOCK clean test failure' patch='10000'
487 """
488         self._run_through_task(commit_queue, expected_logs, expect_retry=True)
489
490     def test_red_tree_patch_rejection(self):
491         commit_queue = FailingTestCommitQueue([
492             None,
493             None,
494             None,
495             None,
496             None,
497             GoldenScriptError("MOCK test failure"),
498             ScriptError("MOCK test failure again"),
499             ScriptError("MOCK clean test failure"),
500         ], [
501             ["foo.html", "bar.html"],
502             ["foo.html", "bar.html"],
503             ["foo.html"],
504         ])
505
506         # Tests always fail, but the clean tree only fails one test
507         # while the patch fails two.  So we should reject the patch!
508         expected_logs = """run_webkit_patch: ['clean']
509 command_passed: success_message='Cleaned working directory' patch='10000'
510 run_webkit_patch: ['update']
511 command_passed: success_message='Updated working directory' patch='10000'
512 run_webkit_patch: ['apply-attachment', '--no-update', '--non-interactive', 10000]
513 command_passed: success_message='Applied patch' patch='10000'
514 run_webkit_patch: ['validate-changelog', '--non-interactive', 10000]
515 command_passed: success_message='ChangeLog validated' patch='10000'
516 run_webkit_patch: ['build', '--no-clean', '--no-update', '--build-style=both']
517 command_passed: success_message='Built patch' patch='10000'
518 run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--non-interactive']
519 command_failed: failure_message='Patch does not pass tests' script_error='MOCK test failure' patch='10000'
520 archive_last_test_results: patch='10000'
521 run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--non-interactive']
522 command_failed: failure_message='Patch does not pass tests' script_error='MOCK test failure again' patch='10000'
523 archive_last_test_results: patch='10000'
524 run_webkit_patch: ['build-and-test', '--force-clean', '--no-update', '--build', '--test', '--non-interactive']
525 command_failed: failure_message='Unable to pass tests without patch (tree is red?)' script_error='MOCK clean test failure' patch='10000'
526 """
527         task = self._run_through_task(commit_queue, expected_logs, GoldenScriptError)
528         self.assertEqual(task.results_from_patch_test_run(task._patch).failing_tests(), ["foo.html", "bar.html"])
529         # failure_status_id should be of the test with patch (1), not the test without patch (2).
530         self.assertEqual(task.failure_status_id, 1)
531
532     def test_land_failure(self):
533         commit_queue = MockCommitQueue([
534             None,
535             None,
536             None,
537             None,
538             None,
539             None,
540             GoldenScriptError("MOCK land failure"),
541         ])
542         expected_logs = """run_webkit_patch: ['clean']
543 command_passed: success_message='Cleaned working directory' patch='10000'
544 run_webkit_patch: ['update']
545 command_passed: success_message='Updated working directory' patch='10000'
546 run_webkit_patch: ['apply-attachment', '--no-update', '--non-interactive', 10000]
547 command_passed: success_message='Applied patch' patch='10000'
548 run_webkit_patch: ['validate-changelog', '--non-interactive', 10000]
549 command_passed: success_message='ChangeLog validated' patch='10000'
550 run_webkit_patch: ['build', '--no-clean', '--no-update', '--build-style=both']
551 command_passed: success_message='Built patch' patch='10000'
552 run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--non-interactive']
553 command_passed: success_message='Passed tests' patch='10000'
554 run_webkit_patch: ['land-attachment', '--force-clean', '--non-interactive', '--parent-command=commit-queue', 10000]
555 command_failed: failure_message='Unable to land patch' script_error='MOCK land failure' patch='10000'
556 """
557         # FIXME: This should really be expect_retry=True for a better user experiance.
558         self._run_through_task(commit_queue, expected_logs, GoldenScriptError)
559
560     def _expect_validate(self, patch, is_valid):
561         class MockDelegate(object):
562             def refetch_patch(self, patch):
563                 return patch
564
565             def expected_failures(self):
566                 return ExpectedFailures()
567
568         task = CommitQueueTask(MockDelegate(), patch)
569         self.assertEqual(task.validate(), is_valid)
570
571     def _mock_patch(self, attachment_dict={}, bug_dict={'bug_status': 'NEW'}, committer="fake"):
572         bug = bugzilla.Bug(bug_dict, None)
573         patch = bugzilla.Attachment(attachment_dict, bug)
574         patch._committer = committer
575         return patch
576
577     def test_validate(self):
578         self._expect_validate(self._mock_patch(), True)
579         self._expect_validate(self._mock_patch({'is_obsolete': True}), False)
580         self._expect_validate(self._mock_patch(bug_dict={'bug_status': 'CLOSED'}), False)
581         self._expect_validate(self._mock_patch(committer=None), False)
582         self._expect_validate(self._mock_patch({'review': '-'}), False)