commit-queue should check for oops in changelog entries
[WebKit-https.git] / Tools / Scripts / webkitpy / tool / bot / commitqueuetask_unittest.py
1 # Copyright (c) 2010 Google Inc. All rights reserved.
2 #
3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions are
5 # met:
6 #
7 #     * Redistributions of source code must retain the above copyright
8 # notice, this list of conditions and the following disclaimer.
9 #     * Redistributions in binary form must reproduce the above
10 # copyright notice, this list of conditions and the following disclaimer
11 # in the documentation and/or other materials provided with the
12 # distribution.
13 #     * Neither the name of Google Inc. nor the names of its
14 # contributors may be used to endorse or promote products derived from
15 # this software without specific prior written permission.
16 #
17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29 from datetime import datetime
30 import logging
31 import unittest2 as unittest
32
33 from webkitpy.common.net import bugzilla
34 from webkitpy.common.net.layouttestresults import LayoutTestResults
35 from webkitpy.common.system.executive import ScriptError
36 from webkitpy.common.system.outputcapture import OutputCapture
37 from webkitpy.layout_tests.models import test_results
38 from webkitpy.layout_tests.models import test_failures
39 from webkitpy.thirdparty.mock import Mock
40 from webkitpy.tool.bot.commitqueuetask import *
41 from webkitpy.tool.bot.expectedfailures import ExpectedFailures
42 from webkitpy.tool.mocktool import MockTool
43
44 _log = logging.getLogger(__name__)
45
46
47 class MockCommitQueue(CommitQueueTaskDelegate):
48     def __init__(self, error_plan):
49         self._error_plan = error_plan
50         self._failure_status_id = 0
51
52     def run_command(self, command):
53         _log.info("run_webkit_patch: %s" % command)
54         if self._error_plan:
55             error = self._error_plan.pop(0)
56             if error:
57                 raise error
58
59     def command_passed(self, success_message, patch):
60         _log.info("command_passed: success_message='%s' patch='%s'" % (
61             success_message, patch.id()))
62
63     def command_failed(self, failure_message, script_error, patch):
64         _log.info("command_failed: failure_message='%s' script_error='%s' patch='%s'" % (
65             failure_message, script_error, patch.id()))
66         self._failure_status_id += 1
67         return self._failure_status_id
68
69     def refetch_patch(self, patch):
70         return patch
71
72     def expected_failures(self):
73         return ExpectedFailures()
74
75     def test_results(self):
76         return None
77
78     def report_flaky_tests(self, patch, flaky_results, results_archive):
79         flaky_tests = [result.filename for result in flaky_results]
80         _log.info("report_flaky_tests: patch='%s' flaky_tests='%s' archive='%s'" % (patch.id(), flaky_tests, results_archive.filename))
81
82     def archive_last_test_results(self, patch):
83         _log.info("archive_last_test_results: patch='%s'" % patch.id())
84         archive = Mock()
85         archive.filename = "mock-archive-%s.zip" % patch.id()
86         return archive
87
88     def build_style(self):
89         return "both"
90
91     def did_pass_testing_ews(self, patch):
92         return False
93
94
95 class FailingTestCommitQueue(MockCommitQueue):
96     def __init__(self, error_plan, test_failure_plan):
97         MockCommitQueue.__init__(self, error_plan)
98         self._test_run_counter = -1  # Special value to indicate tests have never been run.
99         self._test_failure_plan = test_failure_plan
100
101     def run_command(self, command):
102         if command[0] == "build-and-test":
103             self._test_run_counter += 1
104         MockCommitQueue.run_command(self, command)
105
106     def _mock_test_result(self, testname):
107         return test_results.TestResult(testname, [test_failures.FailureTextMismatch()])
108
109     def test_results(self):
110         # Doesn't make sense to ask for the test_results until the tests have run at least once.
111         assert(self._test_run_counter >= 0)
112         failures_for_run = self._test_failure_plan[self._test_run_counter]
113         results = LayoutTestResults(map(self._mock_test_result, failures_for_run))
114         # This makes the results trustable by ExpectedFailures.
115         results.set_failure_limit_count(10)
116         return results
117
118
119 # We use GoldenScriptError to make sure that the code under test throws the
120 # correct (i.e., golden) exception.
121 class GoldenScriptError(ScriptError):
122     pass
123
124
125 class CommitQueueTaskTest(unittest.TestCase):
126     def _run_through_task(self, commit_queue, expected_logs, expected_exception=None, expect_retry=False):
127         self.maxDiff = None
128         tool = MockTool(log_executive=True)
129         patch = tool.bugs.fetch_attachment(10000)
130         task = CommitQueueTask(commit_queue, patch)
131         success = OutputCapture().assert_outputs(self, task.run, expected_logs=expected_logs, expected_exception=expected_exception)
132         if not expected_exception:
133             self.assertEqual(success, not expect_retry)
134         return task
135
136     def test_success_case(self):
137         commit_queue = MockCommitQueue([])
138         expected_logs = """run_webkit_patch: ['clean']
139 command_passed: success_message='Cleaned working directory' patch='10000'
140 run_webkit_patch: ['update']
141 command_passed: success_message='Updated working directory' patch='10000'
142 run_webkit_patch: ['apply-attachment', '--no-update', '--non-interactive', 10000]
143 command_passed: success_message='Applied patch' patch='10000'
144 run_webkit_patch: ['validate-changelog', '--check-oops', '--non-interactive', 10000]
145 command_passed: success_message='ChangeLog validated' patch='10000'
146 run_webkit_patch: ['build', '--no-clean', '--no-update', '--build-style=both']
147 command_passed: success_message='Built patch' patch='10000'
148 run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--non-interactive']
149 command_passed: success_message='Passed tests' patch='10000'
150 run_webkit_patch: ['land-attachment', '--force-clean', '--non-interactive', '--parent-command=commit-queue', 10000]
151 command_passed: success_message='Landed patch' patch='10000'
152 """
153         self._run_through_task(commit_queue, expected_logs)
154
155     def test_fast_success_case(self):
156         commit_queue = MockCommitQueue([])
157         commit_queue.did_pass_testing_ews = lambda patch: True
158         expected_logs = """run_webkit_patch: ['clean']
159 command_passed: success_message='Cleaned working directory' patch='10000'
160 run_webkit_patch: ['update']
161 command_passed: success_message='Updated working directory' patch='10000'
162 run_webkit_patch: ['apply-attachment', '--no-update', '--non-interactive', 10000]
163 command_passed: success_message='Applied patch' patch='10000'
164 run_webkit_patch: ['validate-changelog', '--check-oops', '--non-interactive', 10000]
165 command_passed: success_message='ChangeLog validated' patch='10000'
166 run_webkit_patch: ['build', '--no-clean', '--no-update', '--build-style=both']
167 command_passed: success_message='Built patch' patch='10000'
168 run_webkit_patch: ['land-attachment', '--force-clean', '--non-interactive', '--parent-command=commit-queue', 10000]
169 command_passed: success_message='Landed patch' patch='10000'
170 """
171         self._run_through_task(commit_queue, expected_logs)
172
173     def test_clean_failure(self):
174         commit_queue = MockCommitQueue([
175             ScriptError("MOCK clean failure"),
176         ])
177         expected_logs = """run_webkit_patch: ['clean']
178 command_failed: failure_message='Unable to clean working directory' script_error='MOCK clean failure' patch='10000'
179 """
180         self._run_through_task(commit_queue, expected_logs, expect_retry=True)
181
182     def test_update_failure(self):
183         commit_queue = MockCommitQueue([
184             None,
185             ScriptError("MOCK update failure"),
186         ])
187         expected_logs = """run_webkit_patch: ['clean']
188 command_passed: success_message='Cleaned working directory' patch='10000'
189 run_webkit_patch: ['update']
190 command_failed: failure_message='Unable to update working directory' script_error='MOCK update failure' patch='10000'
191 """
192         self._run_through_task(commit_queue, expected_logs, expect_retry=True)
193
194     def test_apply_failure(self):
195         commit_queue = MockCommitQueue([
196             None,
197             None,
198             GoldenScriptError("MOCK apply failure"),
199         ])
200         expected_logs = """run_webkit_patch: ['clean']
201 command_passed: success_message='Cleaned working directory' patch='10000'
202 run_webkit_patch: ['update']
203 command_passed: success_message='Updated working directory' patch='10000'
204 run_webkit_patch: ['apply-attachment', '--no-update', '--non-interactive', 10000]
205 command_failed: failure_message='Patch does not apply' script_error='MOCK apply failure' patch='10000'
206 """
207         self._run_through_task(commit_queue, expected_logs, GoldenScriptError)
208
209     def test_validate_changelog_failure(self):
210         commit_queue = MockCommitQueue([
211             None,
212             None,
213             None,
214             GoldenScriptError("MOCK validate failure"),
215         ])
216         expected_logs = """run_webkit_patch: ['clean']
217 command_passed: success_message='Cleaned working directory' patch='10000'
218 run_webkit_patch: ['update']
219 command_passed: success_message='Updated working directory' patch='10000'
220 run_webkit_patch: ['apply-attachment', '--no-update', '--non-interactive', 10000]
221 command_passed: success_message='Applied patch' patch='10000'
222 run_webkit_patch: ['validate-changelog', '--check-oops', '--non-interactive', 10000]
223 command_failed: failure_message='ChangeLog did not pass validation' script_error='MOCK validate failure' patch='10000'
224 """
225         self._run_through_task(commit_queue, expected_logs, GoldenScriptError)
226
227     def test_build_failure(self):
228         commit_queue = MockCommitQueue([
229             None,
230             None,
231             None,
232             None,
233             GoldenScriptError("MOCK build failure"),
234         ])
235         expected_logs = """run_webkit_patch: ['clean']
236 command_passed: success_message='Cleaned working directory' patch='10000'
237 run_webkit_patch: ['update']
238 command_passed: success_message='Updated working directory' patch='10000'
239 run_webkit_patch: ['apply-attachment', '--no-update', '--non-interactive', 10000]
240 command_passed: success_message='Applied patch' patch='10000'
241 run_webkit_patch: ['validate-changelog', '--check-oops', '--non-interactive', 10000]
242 command_passed: success_message='ChangeLog validated' patch='10000'
243 run_webkit_patch: ['build', '--no-clean', '--no-update', '--build-style=both']
244 command_failed: failure_message='Patch does not build' script_error='MOCK build failure' patch='10000'
245 run_webkit_patch: ['build', '--force-clean', '--no-update', '--build-style=both']
246 command_passed: success_message='Able to build without patch' patch='10000'
247 """
248         self._run_through_task(commit_queue, expected_logs, GoldenScriptError)
249
250     def test_red_build_failure(self):
251         commit_queue = MockCommitQueue([
252             None,
253             None,
254             None,
255             None,
256             ScriptError("MOCK build failure"),
257             ScriptError("MOCK clean build failure"),
258         ])
259         expected_logs = """run_webkit_patch: ['clean']
260 command_passed: success_message='Cleaned working directory' patch='10000'
261 run_webkit_patch: ['update']
262 command_passed: success_message='Updated working directory' patch='10000'
263 run_webkit_patch: ['apply-attachment', '--no-update', '--non-interactive', 10000]
264 command_passed: success_message='Applied patch' patch='10000'
265 run_webkit_patch: ['validate-changelog', '--check-oops', '--non-interactive', 10000]
266 command_passed: success_message='ChangeLog validated' patch='10000'
267 run_webkit_patch: ['build', '--no-clean', '--no-update', '--build-style=both']
268 command_failed: failure_message='Patch does not build' script_error='MOCK build failure' patch='10000'
269 run_webkit_patch: ['build', '--force-clean', '--no-update', '--build-style=both']
270 command_failed: failure_message='Unable to build without patch' script_error='MOCK clean build failure' patch='10000'
271 """
272         self._run_through_task(commit_queue, expected_logs, expect_retry=True)
273
274     def test_flaky_test_failure(self):
275         commit_queue = MockCommitQueue([
276             None,
277             None,
278             None,
279             None,
280             None,
281             ScriptError("MOCK tests failure"),
282         ])
283         # CommitQueueTask will only report flaky tests if we successfully parsed
284         # results.json and returned a LayoutTestResults object, so we fake one.
285         commit_queue.test_results = lambda: LayoutTestResults([])
286         expected_logs = """run_webkit_patch: ['clean']
287 command_passed: success_message='Cleaned working directory' patch='10000'
288 run_webkit_patch: ['update']
289 command_passed: success_message='Updated working directory' patch='10000'
290 run_webkit_patch: ['apply-attachment', '--no-update', '--non-interactive', 10000]
291 command_passed: success_message='Applied patch' patch='10000'
292 run_webkit_patch: ['validate-changelog', '--check-oops', '--non-interactive', 10000]
293 command_passed: success_message='ChangeLog validated' patch='10000'
294 run_webkit_patch: ['build', '--no-clean', '--no-update', '--build-style=both']
295 command_passed: success_message='Built patch' patch='10000'
296 run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--non-interactive']
297 command_failed: failure_message='Patch does not pass tests' script_error='MOCK tests failure' patch='10000'
298 archive_last_test_results: patch='10000'
299 run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--non-interactive']
300 command_passed: success_message='Passed tests' patch='10000'
301 report_flaky_tests: patch='10000' flaky_tests='[]' archive='mock-archive-10000.zip'
302 run_webkit_patch: ['land-attachment', '--force-clean', '--non-interactive', '--parent-command=commit-queue', 10000]
303 command_passed: success_message='Landed patch' patch='10000'
304 """
305         self._run_through_task(commit_queue, expected_logs)
306
307     def test_failed_archive(self):
308         commit_queue = MockCommitQueue([
309             None,
310             None,
311             None,
312             None,
313             None,
314             ScriptError("MOCK tests failure"),
315         ])
316         commit_queue.test_results = lambda: LayoutTestResults([])
317         # It's possible delegate to fail to archive layout tests, don't try to report
318         # flaky tests when that happens.
319         commit_queue.archive_last_test_results = lambda patch: None
320         expected_logs = """run_webkit_patch: ['clean']
321 command_passed: success_message='Cleaned working directory' patch='10000'
322 run_webkit_patch: ['update']
323 command_passed: success_message='Updated working directory' patch='10000'
324 run_webkit_patch: ['apply-attachment', '--no-update', '--non-interactive', 10000]
325 command_passed: success_message='Applied patch' patch='10000'
326 run_webkit_patch: ['validate-changelog', '--check-oops', '--non-interactive', 10000]
327 command_passed: success_message='ChangeLog validated' patch='10000'
328 run_webkit_patch: ['build', '--no-clean', '--no-update', '--build-style=both']
329 command_passed: success_message='Built patch' patch='10000'
330 run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--non-interactive']
331 command_failed: failure_message='Patch does not pass tests' script_error='MOCK tests failure' patch='10000'
332 run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--non-interactive']
333 command_passed: success_message='Passed tests' patch='10000'
334 run_webkit_patch: ['land-attachment', '--force-clean', '--non-interactive', '--parent-command=commit-queue', 10000]
335 command_passed: success_message='Landed patch' patch='10000'
336 """
337         self._run_through_task(commit_queue, expected_logs)
338
339     def test_double_flaky_test_failure(self):
340         commit_queue = FailingTestCommitQueue([
341             None,
342             None,
343             None,
344             None,
345             None,
346             ScriptError("MOCK test failure"),
347             ScriptError("MOCK test failure again"),
348         ], [
349             "foo.html",
350             "bar.html",
351             "foo.html",
352         ])
353         # The (subtle) point of this test is that report_flaky_tests does not appear
354         # in the expected_logs for this run.
355         # Note also that there is no attempt to run the tests w/o the patch.
356         expected_logs = """run_webkit_patch: ['clean']
357 command_passed: success_message='Cleaned working directory' patch='10000'
358 run_webkit_patch: ['update']
359 command_passed: success_message='Updated working directory' patch='10000'
360 run_webkit_patch: ['apply-attachment', '--no-update', '--non-interactive', 10000]
361 command_passed: success_message='Applied patch' patch='10000'
362 run_webkit_patch: ['validate-changelog', '--check-oops', '--non-interactive', 10000]
363 command_passed: success_message='ChangeLog validated' patch='10000'
364 run_webkit_patch: ['build', '--no-clean', '--no-update', '--build-style=both']
365 command_passed: success_message='Built patch' patch='10000'
366 run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--non-interactive']
367 command_failed: failure_message='Patch does not pass tests' script_error='MOCK test failure' patch='10000'
368 archive_last_test_results: patch='10000'
369 run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--non-interactive']
370 command_failed: failure_message='Patch does not pass tests' script_error='MOCK test failure again' patch='10000'
371 """
372         tool = MockTool(log_executive=True)
373         patch = tool.bugs.fetch_attachment(10000)
374         task = CommitQueueTask(commit_queue, patch)
375         success = OutputCapture().assert_outputs(self, task.run, expected_logs=expected_logs)
376         self.assertFalse(success)
377
378     def test_test_failure(self):
379         commit_queue = MockCommitQueue([
380             None,
381             None,
382             None,
383             None,
384             None,
385             GoldenScriptError("MOCK test failure"),
386             ScriptError("MOCK test failure again"),
387         ])
388         expected_logs = """run_webkit_patch: ['clean']
389 command_passed: success_message='Cleaned working directory' patch='10000'
390 run_webkit_patch: ['update']
391 command_passed: success_message='Updated working directory' patch='10000'
392 run_webkit_patch: ['apply-attachment', '--no-update', '--non-interactive', 10000]
393 command_passed: success_message='Applied patch' patch='10000'
394 run_webkit_patch: ['validate-changelog', '--check-oops', '--non-interactive', 10000]
395 command_passed: success_message='ChangeLog validated' patch='10000'
396 run_webkit_patch: ['build', '--no-clean', '--no-update', '--build-style=both']
397 command_passed: success_message='Built patch' patch='10000'
398 run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--non-interactive']
399 command_failed: failure_message='Patch does not pass tests' script_error='MOCK test failure' patch='10000'
400 archive_last_test_results: patch='10000'
401 run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--non-interactive']
402 command_failed: failure_message='Patch does not pass tests' script_error='MOCK test failure again' patch='10000'
403 archive_last_test_results: patch='10000'
404 run_webkit_patch: ['build-and-test', '--force-clean', '--no-update', '--build', '--test', '--non-interactive']
405 command_passed: success_message='Able to pass tests without patch' patch='10000'
406 """
407         self._run_through_task(commit_queue, expected_logs, GoldenScriptError)
408
409     def test_red_test_failure(self):
410         commit_queue = FailingTestCommitQueue([
411             None,
412             None,
413             None,
414             None,
415             None,
416             ScriptError("MOCK test failure"),
417             ScriptError("MOCK test failure again"),
418             ScriptError("MOCK clean test failure"),
419         ], [
420             "foo.html",
421             "foo.html",
422             "foo.html",
423         ])
424
425         # Tests always fail, and always return the same results, but we
426         # should still be able to land in this case!
427         expected_logs = """run_webkit_patch: ['clean']
428 command_passed: success_message='Cleaned working directory' patch='10000'
429 run_webkit_patch: ['update']
430 command_passed: success_message='Updated working directory' patch='10000'
431 run_webkit_patch: ['apply-attachment', '--no-update', '--non-interactive', 10000]
432 command_passed: success_message='Applied patch' patch='10000'
433 run_webkit_patch: ['validate-changelog', '--check-oops', '--non-interactive', 10000]
434 command_passed: success_message='ChangeLog validated' patch='10000'
435 run_webkit_patch: ['build', '--no-clean', '--no-update', '--build-style=both']
436 command_passed: success_message='Built patch' patch='10000'
437 run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--non-interactive']
438 command_failed: failure_message='Patch does not pass tests' script_error='MOCK test failure' patch='10000'
439 archive_last_test_results: patch='10000'
440 run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--non-interactive']
441 command_failed: failure_message='Patch does not pass tests' script_error='MOCK test failure again' patch='10000'
442 archive_last_test_results: patch='10000'
443 run_webkit_patch: ['build-and-test', '--force-clean', '--no-update', '--build', '--test', '--non-interactive']
444 command_failed: failure_message='Unable to pass tests without patch (tree is red?)' script_error='MOCK clean test failure' patch='10000'
445 run_webkit_patch: ['land-attachment', '--force-clean', '--non-interactive', '--parent-command=commit-queue', 10000]
446 command_passed: success_message='Landed patch' patch='10000'
447 """
448         self._run_through_task(commit_queue, expected_logs)
449
450     def test_very_red_tree_retry(self):
451         lots_of_failing_tests = map(lambda num: "test-%s.html" % num, range(0, 100))
452         commit_queue = FailingTestCommitQueue([
453             None,
454             None,
455             None,
456             None,
457             None,
458             ScriptError("MOCK test failure"),
459             ScriptError("MOCK test failure again"),
460             ScriptError("MOCK clean test failure"),
461         ], [
462             lots_of_failing_tests,
463             lots_of_failing_tests,
464             lots_of_failing_tests,
465         ])
466
467         # Tests always fail, and return so many failures that we do not
468         # trust the results (see ExpectedFailures._can_trust_results) so we
469         # just give up and retry the patch.
470         expected_logs = """run_webkit_patch: ['clean']
471 command_passed: success_message='Cleaned working directory' patch='10000'
472 run_webkit_patch: ['update']
473 command_passed: success_message='Updated working directory' patch='10000'
474 run_webkit_patch: ['apply-attachment', '--no-update', '--non-interactive', 10000]
475 command_passed: success_message='Applied patch' patch='10000'
476 run_webkit_patch: ['validate-changelog', '--check-oops', '--non-interactive', 10000]
477 command_passed: success_message='ChangeLog validated' patch='10000'
478 run_webkit_patch: ['build', '--no-clean', '--no-update', '--build-style=both']
479 command_passed: success_message='Built patch' patch='10000'
480 run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--non-interactive']
481 command_failed: failure_message='Patch does not pass tests' script_error='MOCK test failure' patch='10000'
482 archive_last_test_results: patch='10000'
483 run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--non-interactive']
484 command_failed: failure_message='Patch does not pass tests' script_error='MOCK test failure again' patch='10000'
485 archive_last_test_results: patch='10000'
486 run_webkit_patch: ['build-and-test', '--force-clean', '--no-update', '--build', '--test', '--non-interactive']
487 command_failed: failure_message='Unable to pass tests without patch (tree is red?)' script_error='MOCK clean test failure' patch='10000'
488 """
489         self._run_through_task(commit_queue, expected_logs, expect_retry=True)
490
491     def test_red_tree_patch_rejection(self):
492         commit_queue = FailingTestCommitQueue([
493             None,
494             None,
495             None,
496             None,
497             None,
498             GoldenScriptError("MOCK test failure"),
499             ScriptError("MOCK test failure again"),
500             ScriptError("MOCK clean test failure"),
501         ], [
502             ["foo.html", "bar.html"],
503             ["foo.html", "bar.html"],
504             ["foo.html"],
505         ])
506
507         # Tests always fail, but the clean tree only fails one test
508         # while the patch fails two.  So we should reject the patch!
509         expected_logs = """run_webkit_patch: ['clean']
510 command_passed: success_message='Cleaned working directory' patch='10000'
511 run_webkit_patch: ['update']
512 command_passed: success_message='Updated working directory' patch='10000'
513 run_webkit_patch: ['apply-attachment', '--no-update', '--non-interactive', 10000]
514 command_passed: success_message='Applied patch' patch='10000'
515 run_webkit_patch: ['validate-changelog', '--check-oops', '--non-interactive', 10000]
516 command_passed: success_message='ChangeLog validated' patch='10000'
517 run_webkit_patch: ['build', '--no-clean', '--no-update', '--build-style=both']
518 command_passed: success_message='Built patch' patch='10000'
519 run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--non-interactive']
520 command_failed: failure_message='Patch does not pass tests' script_error='MOCK test failure' patch='10000'
521 archive_last_test_results: patch='10000'
522 run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--non-interactive']
523 command_failed: failure_message='Patch does not pass tests' script_error='MOCK test failure again' patch='10000'
524 archive_last_test_results: patch='10000'
525 run_webkit_patch: ['build-and-test', '--force-clean', '--no-update', '--build', '--test', '--non-interactive']
526 command_failed: failure_message='Unable to pass tests without patch (tree is red?)' script_error='MOCK clean test failure' patch='10000'
527 """
528         task = self._run_through_task(commit_queue, expected_logs, GoldenScriptError)
529         self.assertEqual(task.results_from_patch_test_run(task._patch).failing_tests(), ["foo.html", "bar.html"])
530         # failure_status_id should be of the test with patch (1), not the test without patch (2).
531         self.assertEqual(task.failure_status_id, 1)
532
533     def test_land_failure(self):
534         commit_queue = MockCommitQueue([
535             None,
536             None,
537             None,
538             None,
539             None,
540             None,
541             GoldenScriptError("MOCK land failure"),
542         ])
543         expected_logs = """run_webkit_patch: ['clean']
544 command_passed: success_message='Cleaned working directory' patch='10000'
545 run_webkit_patch: ['update']
546 command_passed: success_message='Updated working directory' patch='10000'
547 run_webkit_patch: ['apply-attachment', '--no-update', '--non-interactive', 10000]
548 command_passed: success_message='Applied patch' patch='10000'
549 run_webkit_patch: ['validate-changelog', '--check-oops', '--non-interactive', 10000]
550 command_passed: success_message='ChangeLog validated' patch='10000'
551 run_webkit_patch: ['build', '--no-clean', '--no-update', '--build-style=both']
552 command_passed: success_message='Built patch' patch='10000'
553 run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--non-interactive']
554 command_passed: success_message='Passed tests' patch='10000'
555 run_webkit_patch: ['land-attachment', '--force-clean', '--non-interactive', '--parent-command=commit-queue', 10000]
556 command_failed: failure_message='Unable to land patch' script_error='MOCK land failure' patch='10000'
557 """
558         # FIXME: This should really be expect_retry=True for a better user experiance.
559         self._run_through_task(commit_queue, expected_logs, GoldenScriptError)
560
561     def _expect_validate(self, patch, is_valid):
562         class MockDelegate(object):
563             def refetch_patch(self, patch):
564                 return patch
565
566             def expected_failures(self):
567                 return ExpectedFailures()
568
569         task = CommitQueueTask(MockDelegate(), patch)
570         self.assertEqual(task.validate(), is_valid)
571
572     def _mock_patch(self, attachment_dict={}, bug_dict={'bug_status': 'NEW'}, committer="fake"):
573         bug = bugzilla.Bug(bug_dict, None)
574         patch = bugzilla.Attachment(attachment_dict, bug)
575         patch._committer = committer
576         return patch
577
578     def test_validate(self):
579         self._expect_validate(self._mock_patch(), True)
580         self._expect_validate(self._mock_patch({'is_obsolete': True}), False)
581         self._expect_validate(self._mock_patch(bug_dict={'bug_status': 'CLOSED'}), False)
582         self._expect_validate(self._mock_patch(committer=None), False)
583         self._expect_validate(self._mock_patch({'review': '-'}), False)