Get rid of Retry status in webkit-queues
[WebKit-https.git] / Tools / Scripts / webkitpy / tool / commands / queues.py
1 # Copyright (c) 2009 Google Inc. All rights reserved.
2 # Copyright (c) 2009 Apple Inc. All rights reserved.
3 #
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
6 # met:
7 #
8 #     * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 #     * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
13 # distribution.
14 #     * Neither the name of Google Inc. nor the names of its
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
17 #
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30 import codecs
31 import logging
32 import os
33 import re
34 import sys
35 import time
36 import traceback
37
38 from datetime import datetime
39 from optparse import make_option
40 from StringIO import StringIO
41
42 from webkitpy.common.config.committervalidator import CommitterValidator
43 from webkitpy.common.config.ports import DeprecatedPort
44 from webkitpy.common.net.bugzilla import Attachment
45 from webkitpy.common.net.statusserver import StatusServer
46 from webkitpy.common.system.executive import ScriptError
47 from webkitpy.tool.bot.botinfo import BotInfo
48 from webkitpy.tool.bot.commitqueuetask import CommitQueueTask, CommitQueueTaskDelegate
49 from webkitpy.tool.bot.expectedfailures import ExpectedFailures
50 from webkitpy.tool.bot.feeders import CommitQueueFeeder, EWSFeeder
51 from webkitpy.tool.bot.flakytestreporter import FlakyTestReporter
52 from webkitpy.tool.bot.layouttestresultsreader import LayoutTestResultsReader
53 from webkitpy.tool.bot.patchanalysistask import UnableToApplyPatch
54 from webkitpy.tool.bot.queueengine import QueueEngine, QueueEngineDelegate
55 from webkitpy.tool.bot.stylequeuetask import StyleQueueTask, StyleQueueTaskDelegate
56 from webkitpy.tool.commands.stepsequence import StepSequenceErrorHandler
57 from webkitpy.tool.multicommandtool import Command, TryAgain
58
59 _log = logging.getLogger(__name__)
60
61
62 class AbstractQueue(Command, QueueEngineDelegate):
63     watchers = [
64     ]
65
66     _pass_status = "Pass"
67     _fail_status = "Fail"
68     _error_status = "Error"
69
70     def __init__(self, options=None): # Default values should never be collections (like []) as default values are shared between invocations
71         options_list = (options or []) + [
72             make_option("--no-confirm", action="store_false", dest="confirm", default=True, help="Do not ask the user for confirmation before running the queue.  Dangerous!"),
73             make_option("--exit-after-iteration", action="store", type="int", dest="iterations", default=None, help="Stop running the queue after iterating this number of times."),
74         ]
75         self.help_text = "Run the %s" % self.name
76         Command.__init__(self, options=options_list)
77         self._iteration_count = 0
78
79     def _cc_watchers(self, bug_id):
80         try:
81             self._tool.bugs.add_cc_to_bug(bug_id, self.watchers)
82         except Exception, e:
83             traceback.print_exc()
84             _log.error("Failed to CC watchers.")
85
86     def run_webkit_patch(self, args):
87         webkit_patch_args = [self._tool.path()]
88         # FIXME: This is a hack, we should have a more general way to pass global options.
89         # FIXME: We must always pass global options and their value in one argument
90         # because our global option code looks for the first argument which does
91         # not begin with "-" and assumes that is the command name.
92         webkit_patch_args += ["--status-host=%s" % self._tool.status_server.host]
93         if self._tool.status_server.bot_id:
94             webkit_patch_args += ["--bot-id=%s" % self._tool.status_server.bot_id]
95         if self._options.port:
96             webkit_patch_args += ["--port=%s" % self._options.port]
97         webkit_patch_args.extend(args)
98
99         try:
100             args_for_printing = list(webkit_patch_args)
101             args_for_printing[0] = 'webkit-patch'  # Printing our path for each log is redundant.
102             _log.info("Running: %s" % self._tool.executive.command_for_printing(args_for_printing))
103             command_output = self._tool.executive.run_command(webkit_patch_args, cwd=self._tool.scm().checkout_root)
104         except ScriptError, e:
105             # Make sure the whole output gets printed if the command failed.
106             _log.error(e.message_with_output(output_limit=None))
107             raise
108         return command_output
109
110     def _log_directory(self):
111         return os.path.join("..", "%s-logs" % self.name)
112
113     # QueueEngineDelegate methods
114
115     def queue_log_path(self):
116         return os.path.join(self._log_directory(), "%s.log" % self.name)
117
118     def work_item_log_path(self, work_item):
119         raise NotImplementedError, "subclasses must implement"
120
121     def begin_work_queue(self):
122         _log.info("CAUTION: %s will discard all local changes in \"%s\"" % (self.name, self._tool.scm().checkout_root))
123         if self._options.confirm:
124             response = self._tool.user.prompt("Are you sure?  Type \"yes\" to continue: ")
125             if (response != "yes"):
126                 _log.error("User declined.")
127                 sys.exit(1)
128         _log.info("Running WebKit %s." % self.name)
129         self._tool.status_server.update_status(self.name, "Starting Queue")
130
131     def stop_work_queue(self, reason):
132         self._tool.status_server.update_status(self.name, "Stopping Queue, reason: %s" % reason)
133
134     def should_continue_work_queue(self):
135         self._iteration_count += 1
136         return not self._options.iterations or self._iteration_count <= self._options.iterations
137
138     def next_work_item(self):
139         raise NotImplementedError, "subclasses must implement"
140
141     def process_work_item(self, work_item):
142         raise NotImplementedError, "subclasses must implement"
143
144     def handle_unexpected_error(self, work_item, message):
145         raise NotImplementedError, "subclasses must implement"
146
147     # Command methods
148
149     def execute(self, options, args, tool, engine=QueueEngine):
150         self._options = options # FIXME: This code is wrong.  Command.options is a list, this assumes an Options element!
151         self._tool = tool  # FIXME: This code is wrong too!  Command.bind_to_tool handles this!
152         return engine(self.name, self, self._tool.wakeup_event, self._options.seconds_to_sleep).run()
153
154     @classmethod
155     def _log_from_script_error_for_upload(cls, script_error, output_limit=None):
156         # We have seen request timeouts with app engine due to large
157         # log uploads.  Trying only the last 512k.
158         if not output_limit:
159             output_limit = 512 * 1024  # 512k
160         output = script_error.message_with_output(output_limit=output_limit)
161         # We pre-encode the string to a byte array before passing it
162         # to status_server, because ClientForm (part of mechanize)
163         # wants a file-like object with pre-encoded data.
164         return StringIO(output.encode("utf-8"))
165
166     @classmethod
167     def _update_status_for_script_error(cls, tool, state, script_error, is_error=False):
168         message = str(script_error)
169         if is_error:
170             message = "Error: %s" % message
171         failure_log = cls._log_from_script_error_for_upload(script_error)
172         return tool.status_server.update_status(cls.name, message, state["patch"], failure_log)
173
174
175 class FeederQueue(AbstractQueue):
176     name = "feeder-queue"
177
178     _sleep_duration = 30  # seconds
179
180     # AbstractQueue methods
181
182     def begin_work_queue(self):
183         AbstractQueue.begin_work_queue(self)
184         self.feeders = [
185             CommitQueueFeeder(self._tool),
186             EWSFeeder(self._tool),
187         ]
188
189     def next_work_item(self):
190         # This really show inherit from some more basic class that doesn't
191         # understand work items, but the base class in the heirarchy currently
192         # understands work items.
193         return "synthetic-work-item"
194
195     def process_work_item(self, work_item):
196         for feeder in self.feeders:
197             feeder.feed()
198         time.sleep(self._sleep_duration)
199         return True
200
201     def work_item_log_path(self, work_item):
202         return None
203
204     def handle_unexpected_error(self, work_item, message):
205         _log.error(message)
206
207
208 class AbstractPatchQueue(AbstractQueue):
209     def _update_status(self, message, patch=None, results_file=None):
210         return self._tool.status_server.update_status(self.name, message, patch, results_file)
211
212     def _next_patch(self):
213         # FIXME: Bugzilla accessibility should be checked here; if it's unaccessible,
214         # it should return None.
215         patch = None
216         while not patch:
217             patch_id = self._tool.status_server.next_work_item(self.name)
218             if not patch_id:
219                 return None
220             patch = self._tool.bugs.fetch_attachment(patch_id)
221             if not patch:
222                 # FIXME: Using a fake patch because release_work_item has the wrong API.
223                 # We also don't really need to release the lock (although that's fine),
224                 # mostly we just need to remove this bogus patch from our queue.
225                 # If for some reason bugzilla is just down, then it will be re-fed later.
226                 fake_patch = Attachment({'id': patch_id}, None)
227                 self._release_work_item(fake_patch)
228         return patch
229
230     def _release_work_item(self, patch):
231         self._tool.status_server.release_work_item(self.name, patch)
232
233     def _did_pass(self, patch):
234         self._update_status(self._pass_status, patch)
235         self._release_work_item(patch)
236
237     def _did_fail(self, patch):
238         self._update_status(self._fail_status, patch)
239         self._release_work_item(patch)
240
241     def _did_error(self, patch, reason):
242         message = "%s: %s" % (self._error_status, reason)
243         self._update_status(message, patch)
244         self._release_work_item(patch)
245
246     def _unlock_patch(self, patch):
247         self._tool.status_server.release_lock(self.name, patch)
248
249     def work_item_log_path(self, patch):
250         return os.path.join(self._log_directory(), "%s.log" % patch.bug_id())
251
252
253 # Used to share code between the EWS and commit-queue.
254 class PatchProcessingQueue(AbstractPatchQueue):
255     # Subclasses must override.
256     port_name = None
257
258     def __init__(self, options=None):
259         self._port = None  # We can't instantiate port here because tool isn't avaialble.
260         AbstractPatchQueue.__init__(self, options)
261
262     # FIXME: This is a hack to map between the old port names and the new port names.
263     def _new_port_name_from_old(self, port_name, platform):
264         # ApplePort.determine_full_port_name asserts if the name doesn't include version.
265         if port_name == 'mac':
266             return 'mac-' + platform.os_version
267         if port_name == 'win':
268             return 'win-future'
269         return port_name
270
271     def begin_work_queue(self):
272         AbstractPatchQueue.begin_work_queue(self)
273         if not self.port_name:
274             return
275         # FIXME: This is only used for self._deprecated_port.flag()
276         self._deprecated_port = DeprecatedPort.port(self.port_name)
277         # FIXME: This violates abstraction
278         self._tool._deprecated_port = self._deprecated_port
279         self._port = self._tool.port_factory.get(self._new_port_name_from_old(self.port_name, self._tool.platform))
280
281     def _upload_results_archive_for_patch(self, patch, results_archive_zip):
282         if not self._port:
283             self._port = self._tool.port_factory.get(self._new_port_name_from_old(self.port_name, self._tool.platform))
284
285         bot_id = self._tool.status_server.bot_id or "bot"
286         description = "Archive of layout-test-results from %s for %s" % (bot_id, self._port.name())
287         # results_archive is a ZipFile object, grab the File object (.fp) to pass to Mechanize for uploading.
288         results_archive_file = results_archive_zip.fp
289         # Rewind the file object to start (since Mechanize won't do that automatically)
290         # See https://bugs.webkit.org/show_bug.cgi?id=54593
291         results_archive_file.seek(0)
292         # FIXME: This is a small lie to always say run-webkit-tests since Chromium uses new-run-webkit-tests.
293         # We could make this code look up the test script name off the port.
294         comment_text = "The attached test failures were seen while running run-webkit-tests on the %s.\n" % (self.name)
295         # FIXME: We could easily list the test failures from the archive here,
296         # currently callers do that separately.
297         comment_text += BotInfo(self._tool, self._port.name()).summary_text()
298         self._tool.bugs.add_attachment_to_bug(patch.bug_id(), results_archive_file, description, filename="layout-test-results.zip", comment_text=comment_text)
299
300
301 class CommitQueue(PatchProcessingQueue, StepSequenceErrorHandler, CommitQueueTaskDelegate):
302     name = "commit-queue"
303     port_name = "mac-mountainlion"
304
305     # AbstractPatchQueue methods
306
307     def begin_work_queue(self):
308         PatchProcessingQueue.begin_work_queue(self)
309         self.committer_validator = CommitterValidator(self._tool)
310         self._expected_failures = ExpectedFailures()
311         self._layout_test_results_reader = LayoutTestResultsReader(self._tool, self._port.results_directory(), self._log_directory())
312
313     def next_work_item(self):
314         return self._next_patch()
315
316     def process_work_item(self, patch):
317         self._cc_watchers(patch.bug_id())
318         task = CommitQueueTask(self, patch)
319         try:
320             if task.run():
321                 self._did_pass(patch)
322                 return True
323             self._unlock_patch(patch)
324             return False
325         except ScriptError, e:
326             validator = CommitterValidator(self._tool)
327             validator.reject_patch_from_commit_queue(patch.id(), self._error_message_for_bug(task, patch, e))
328             results_archive = task.results_archive_from_patch_test_run(patch)
329             if results_archive:
330                 self._upload_results_archive_for_patch(patch, results_archive)
331             self._did_fail(patch)
332             return False
333
334     def _failing_tests_message(self, task, patch):
335         results = task.results_from_patch_test_run(patch)
336         unexpected_failures = self._expected_failures.unexpected_failures_observed(results)
337         if not unexpected_failures:
338             return None
339         return "New failing tests:\n%s" % "\n".join(unexpected_failures)
340
341     def _error_message_for_bug(self, task, patch, script_error):
342         message = self._failing_tests_message(task, patch)
343         if not message:
344             message = script_error.message_with_output()
345         results_link = self._tool.status_server.results_url_for_status(task.failure_status_id)
346         return "%s\nFull output: %s" % (message, results_link)
347
348     def handle_unexpected_error(self, patch, message):
349         self.committer_validator.reject_patch_from_commit_queue(patch.id(), message)
350
351     # CommitQueueTaskDelegate methods
352
353     def run_command(self, command):
354         self.run_webkit_patch(command + [self._deprecated_port.flag()])
355
356     def command_passed(self, message, patch):
357         self._update_status(message, patch=patch)
358
359     def command_failed(self, message, script_error, patch):
360         failure_log = self._log_from_script_error_for_upload(script_error)
361         return self._update_status(message, patch=patch, results_file=failure_log)
362
363     def expected_failures(self):
364         return self._expected_failures
365
366     def test_results(self):
367         return self._layout_test_results_reader.results()
368
369     def archive_last_test_results(self, patch):
370         return self._layout_test_results_reader.archive(patch)
371
372     def build_style(self):
373         return "release"
374
375     def refetch_patch(self, patch):
376         return self._tool.bugs.fetch_attachment(patch.id())
377
378     def report_flaky_tests(self, patch, flaky_test_results, results_archive=None):
379         reporter = FlakyTestReporter(self._tool, self.name)
380         reporter.report_flaky_tests(patch, flaky_test_results, results_archive)
381
382     def did_pass_testing_ews(self, patch):
383         # Only Mac and Mac WK2 run tests
384         # FIXME: We shouldn't have to hard-code it here.
385         patch_status = self._tool.status_server.patch_status
386         return patch_status("mac-ews", patch.id()) == self._pass_status or patch_status("mac-wk2-ews", patch.id()) == self._pass_status
387
388     # StepSequenceErrorHandler methods
389
390     @classmethod
391     def handle_script_error(cls, tool, state, script_error):
392         # Hitting this error handler should be pretty rare.  It does occur,
393         # however, when a patch no longer applies to top-of-tree in the final
394         # land step.
395         _log.error(script_error.message_with_output())
396
397     @classmethod
398     def handle_checkout_needs_update(cls, tool, state, options, error):
399         message = "Tests passed, but commit failed (checkout out of date).  Updating, then landing without building or re-running tests."
400         tool.status_server.update_status(cls.name, message, state["patch"])
401         # The only time when we find out that out checkout needs update is
402         # when we were ready to actually pull the trigger and land the patch.
403         # Rather than spinning in the master process, we retry without
404         # building or testing, which is much faster.
405         options.build = False
406         options.test = False
407         options.update = True
408         raise TryAgain()
409
410
411 class AbstractReviewQueue(PatchProcessingQueue, StepSequenceErrorHandler):
412     """This is the base-class for the EWS queues and the style-queue."""
413     def __init__(self, options=None):
414         PatchProcessingQueue.__init__(self, options)
415
416     def review_patch(self, patch):
417         raise NotImplementedError("subclasses must implement")
418
419     # AbstractPatchQueue methods
420
421     def begin_work_queue(self):
422         PatchProcessingQueue.begin_work_queue(self)
423
424     def next_work_item(self):
425         return self._next_patch()
426
427     def process_work_item(self, patch):
428         passed = self.review_patch(patch)
429         if passed:
430             self._did_pass(patch)
431         return passed
432
433     def handle_unexpected_error(self, patch, message):
434         _log.error(message)
435
436     # StepSequenceErrorHandler methods
437
438     @classmethod
439     def handle_script_error(cls, tool, state, script_error):
440         _log.error(script_error.output)
441
442
443 class StyleQueue(AbstractReviewQueue, StyleQueueTaskDelegate):
444     name = "style-queue"
445
446     def __init__(self):
447         AbstractReviewQueue.__init__(self)
448
449     def review_patch(self, patch):
450         task = StyleQueueTask(self, patch)
451         if not task.validate():
452             self._did_error(patch, "%s did not process patch." % self.name)
453             return False
454         try:
455             style_check_succeeded = task.run()
456             if not style_check_succeeded:
457                 # Caller unlocks when review_patch returns True, so we only need to unlock on transient failure.
458                 self._unlock_patch(patch)
459             return style_check_succeeded
460         except UnableToApplyPatch, e:
461             self._did_error(patch, "%s unable to apply patch." % self.name)
462             return False
463         except ScriptError, e:
464             output = re.sub(r'Failed to run .+ exit_code: 1', '', e.output)
465             message = "Attachment %s did not pass %s:\n\n%s\n\nIf any of these errors are false positives, please file a bug against check-webkit-style." % (patch.id(), self.name, output)
466             self._tool.bugs.post_comment_to_bug(patch.bug_id(), message, cc=self.watchers)
467             self._did_fail(patch)
468             return False
469         return True
470
471     # StyleQueueTaskDelegate methods
472
473     def run_command(self, command):
474         self.run_webkit_patch(command)
475
476     def command_passed(self, message, patch):
477         self._update_status(message, patch=patch)
478
479     def command_failed(self, message, script_error, patch):
480         failure_log = self._log_from_script_error_for_upload(script_error)
481         return self._update_status(message, patch=patch, results_file=failure_log)
482
483     def expected_failures(self):
484         return None
485
486     def refetch_patch(self, patch):
487         return self._tool.bugs.fetch_attachment(patch.id())