08780fa130ac9e752a36a9cc0ff3a78199417468
[WebKit-https.git] / Tools / Scripts / webkitpy / tool / commands / queues.py
1 # Copyright (c) 2009 Google Inc. All rights reserved.
2 # Copyright (c) 2009 Apple Inc. All rights reserved.
3 #
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
6 # met:
7 #
8 #     * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 #     * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
13 # distribution.
14 #     * Neither the name of Google Inc. nor the names of its
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
17 #
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30 import codecs
31 import logging
32 import os
33 import re
34 import sys
35 import time
36 import traceback
37
38 from datetime import datetime
39 from optparse import make_option
40 from StringIO import StringIO
41
42 from webkitpy.common.config.committervalidator import CommitterValidator
43 from webkitpy.common.config.ports import DeprecatedPort
44 from webkitpy.common.net.bugzilla import Attachment
45 from webkitpy.common.net.statusserver import StatusServer
46 from webkitpy.common.system.executive import ScriptError
47 from webkitpy.tool.bot.botinfo import BotInfo
48 from webkitpy.tool.bot.commitqueuetask import CommitQueueTask, CommitQueueTaskDelegate
49 from webkitpy.tool.bot.expectedfailures import ExpectedFailures
50 from webkitpy.tool.bot.feeders import CommitQueueFeeder, EWSFeeder
51 from webkitpy.tool.bot.flakytestreporter import FlakyTestReporter
52 from webkitpy.tool.bot.layouttestresultsreader import LayoutTestResultsReader
53 from webkitpy.tool.bot.patchanalysistask import UnableToApplyPatch
54 from webkitpy.tool.bot.queueengine import QueueEngine, QueueEngineDelegate
55 from webkitpy.tool.bot.stylequeuetask import StyleQueueTask, StyleQueueTaskDelegate
56 from webkitpy.tool.commands.stepsequence import StepSequenceErrorHandler
57 from webkitpy.tool.multicommandtool import Command, TryAgain
58
59 _log = logging.getLogger(__name__)
60
61
62 class AbstractQueue(Command, QueueEngineDelegate):
63     watchers = [
64     ]
65
66     _pass_status = "Pass"
67     _fail_status = "Fail"
68     _retry_status = "Retry"
69     _error_status = "Error"
70
71     def __init__(self, options=None): # Default values should never be collections (like []) as default values are shared between invocations
72         options_list = (options or []) + [
73             make_option("--no-confirm", action="store_false", dest="confirm", default=True, help="Do not ask the user for confirmation before running the queue.  Dangerous!"),
74             make_option("--exit-after-iteration", action="store", type="int", dest="iterations", default=None, help="Stop running the queue after iterating this number of times."),
75         ]
76         self.help_text = "Run the %s" % self.name
77         Command.__init__(self, options=options_list)
78         self._iteration_count = 0
79
80     def _cc_watchers(self, bug_id):
81         try:
82             self._tool.bugs.add_cc_to_bug(bug_id, self.watchers)
83         except Exception, e:
84             traceback.print_exc()
85             _log.error("Failed to CC watchers.")
86
87     def run_webkit_patch(self, args):
88         webkit_patch_args = [self._tool.path()]
89         # FIXME: This is a hack, we should have a more general way to pass global options.
90         # FIXME: We must always pass global options and their value in one argument
91         # because our global option code looks for the first argument which does
92         # not begin with "-" and assumes that is the command name.
93         webkit_patch_args += ["--status-host=%s" % self._tool.status_server.host]
94         if self._tool.status_server.bot_id:
95             webkit_patch_args += ["--bot-id=%s" % self._tool.status_server.bot_id]
96         if self._options.port:
97             webkit_patch_args += ["--port=%s" % self._options.port]
98         webkit_patch_args.extend(args)
99
100         try:
101             args_for_printing = list(webkit_patch_args)
102             args_for_printing[0] = 'webkit-patch'  # Printing our path for each log is redundant.
103             _log.info("Running: %s" % self._tool.executive.command_for_printing(args_for_printing))
104             command_output = self._tool.executive.run_command(webkit_patch_args, cwd=self._tool.scm().checkout_root)
105         except ScriptError, e:
106             # Make sure the whole output gets printed if the command failed.
107             _log.error(e.message_with_output(output_limit=None))
108             raise
109         return command_output
110
111     def _log_directory(self):
112         return os.path.join("..", "%s-logs" % self.name)
113
114     # QueueEngineDelegate methods
115
116     def queue_log_path(self):
117         return os.path.join(self._log_directory(), "%s.log" % self.name)
118
119     def work_item_log_path(self, work_item):
120         raise NotImplementedError, "subclasses must implement"
121
122     def begin_work_queue(self):
123         _log.info("CAUTION: %s will discard all local changes in \"%s\"" % (self.name, self._tool.scm().checkout_root))
124         if self._options.confirm:
125             response = self._tool.user.prompt("Are you sure?  Type \"yes\" to continue: ")
126             if (response != "yes"):
127                 _log.error("User declined.")
128                 sys.exit(1)
129         _log.info("Running WebKit %s." % self.name)
130         self._tool.status_server.update_status(self.name, "Starting Queue")
131
132     def stop_work_queue(self, reason):
133         self._tool.status_server.update_status(self.name, "Stopping Queue, reason: %s" % reason)
134
135     def should_continue_work_queue(self):
136         self._iteration_count += 1
137         return not self._options.iterations or self._iteration_count <= self._options.iterations
138
139     def next_work_item(self):
140         raise NotImplementedError, "subclasses must implement"
141
142     def process_work_item(self, work_item):
143         raise NotImplementedError, "subclasses must implement"
144
145     def handle_unexpected_error(self, work_item, message):
146         raise NotImplementedError, "subclasses must implement"
147
148     # Command methods
149
150     def execute(self, options, args, tool, engine=QueueEngine):
151         self._options = options # FIXME: This code is wrong.  Command.options is a list, this assumes an Options element!
152         self._tool = tool  # FIXME: This code is wrong too!  Command.bind_to_tool handles this!
153         return engine(self.name, self, self._tool.wakeup_event, self._options.seconds_to_sleep).run()
154
155     @classmethod
156     def _log_from_script_error_for_upload(cls, script_error, output_limit=None):
157         # We have seen request timeouts with app engine due to large
158         # log uploads.  Trying only the last 512k.
159         if not output_limit:
160             output_limit = 512 * 1024  # 512k
161         output = script_error.message_with_output(output_limit=output_limit)
162         # We pre-encode the string to a byte array before passing it
163         # to status_server, because ClientForm (part of mechanize)
164         # wants a file-like object with pre-encoded data.
165         return StringIO(output.encode("utf-8"))
166
167     @classmethod
168     def _update_status_for_script_error(cls, tool, state, script_error, is_error=False):
169         message = str(script_error)
170         if is_error:
171             message = "Error: %s" % message
172         failure_log = cls._log_from_script_error_for_upload(script_error)
173         return tool.status_server.update_status(cls.name, message, state["patch"], failure_log)
174
175
176 class FeederQueue(AbstractQueue):
177     name = "feeder-queue"
178
179     _sleep_duration = 30  # seconds
180
181     # AbstractQueue methods
182
183     def begin_work_queue(self):
184         AbstractQueue.begin_work_queue(self)
185         self.feeders = [
186             CommitQueueFeeder(self._tool),
187             EWSFeeder(self._tool),
188         ]
189
190     def next_work_item(self):
191         # This really show inherit from some more basic class that doesn't
192         # understand work items, but the base class in the heirarchy currently
193         # understands work items.
194         return "synthetic-work-item"
195
196     def process_work_item(self, work_item):
197         for feeder in self.feeders:
198             feeder.feed()
199         time.sleep(self._sleep_duration)
200         return True
201
202     def work_item_log_path(self, work_item):
203         return None
204
205     def handle_unexpected_error(self, work_item, message):
206         _log.error(message)
207
208
209 class AbstractPatchQueue(AbstractQueue):
210     def _update_status(self, message, patch=None, results_file=None):
211         return self._tool.status_server.update_status(self.name, message, patch, results_file)
212
213     def _next_patch(self):
214         # FIXME: Bugzilla accessibility should be checked here; if it's unaccessible,
215         # it should return None.
216         patch = None
217         while not patch:
218             patch_id = self._tool.status_server.next_work_item(self.name)
219             if not patch_id:
220                 return None
221             patch = self._tool.bugs.fetch_attachment(patch_id)
222             if not patch:
223                 # FIXME: Using a fake patch because release_work_item has the wrong API.
224                 # We also don't really need to release the lock (although that's fine),
225                 # mostly we just need to remove this bogus patch from our queue.
226                 # If for some reason bugzilla is just down, then it will be re-fed later.
227                 fake_patch = Attachment({'id': patch_id}, None)
228                 self._release_work_item(fake_patch)
229         return patch
230
231     def _release_work_item(self, patch):
232         self._tool.status_server.release_work_item(self.name, patch)
233
234     def _did_pass(self, patch):
235         self._update_status(self._pass_status, patch)
236         self._release_work_item(patch)
237
238     def _did_fail(self, patch):
239         self._update_status(self._fail_status, patch)
240         self._release_work_item(patch)
241
242     def _did_retry(self, patch):
243         self._update_status(self._retry_status, patch)
244         self._release_work_item(patch)
245
246     def _did_error(self, patch, reason):
247         message = "%s: %s" % (self._error_status, reason)
248         self._update_status(message, patch)
249         self._release_work_item(patch)
250
251     def _unlock_patch(self, patch):
252         self._tool.status_server.release_lock(self.name, patch)
253
254     def work_item_log_path(self, patch):
255         return os.path.join(self._log_directory(), "%s.log" % patch.bug_id())
256
257
258 # Used to share code between the EWS and commit-queue.
259 class PatchProcessingQueue(AbstractPatchQueue):
260     # Subclasses must override.
261     port_name = None
262
263     def __init__(self, options=None):
264         self._port = None  # We can't instantiate port here because tool isn't avaialble.
265         AbstractPatchQueue.__init__(self, options)
266
267     # FIXME: This is a hack to map between the old port names and the new port names.
268     def _new_port_name_from_old(self, port_name, platform):
269         # ApplePort.determine_full_port_name asserts if the name doesn't include version.
270         if port_name == 'mac':
271             return 'mac-' + platform.os_version
272         if port_name == 'win':
273             return 'win-future'
274         return port_name
275
276     def begin_work_queue(self):
277         AbstractPatchQueue.begin_work_queue(self)
278         if not self.port_name:
279             return
280         # FIXME: This is only used for self._deprecated_port.flag()
281         self._deprecated_port = DeprecatedPort.port(self.port_name)
282         # FIXME: This violates abstraction
283         self._tool._deprecated_port = self._deprecated_port
284         self._port = self._tool.port_factory.get(self._new_port_name_from_old(self.port_name, self._tool.platform))
285
286     def _upload_results_archive_for_patch(self, patch, results_archive_zip):
287         if not self._port:
288             self._port = self._tool.port_factory.get(self._new_port_name_from_old(self.port_name, self._tool.platform))
289
290         bot_id = self._tool.status_server.bot_id or "bot"
291         description = "Archive of layout-test-results from %s for %s" % (bot_id, self._port.name())
292         # results_archive is a ZipFile object, grab the File object (.fp) to pass to Mechanize for uploading.
293         results_archive_file = results_archive_zip.fp
294         # Rewind the file object to start (since Mechanize won't do that automatically)
295         # See https://bugs.webkit.org/show_bug.cgi?id=54593
296         results_archive_file.seek(0)
297         # FIXME: This is a small lie to always say run-webkit-tests since Chromium uses new-run-webkit-tests.
298         # We could make this code look up the test script name off the port.
299         comment_text = "The attached test failures were seen while running run-webkit-tests on the %s.\n" % (self.name)
300         # FIXME: We could easily list the test failures from the archive here,
301         # currently callers do that separately.
302         comment_text += BotInfo(self._tool, self._port.name()).summary_text()
303         self._tool.bugs.add_attachment_to_bug(patch.bug_id(), results_archive_file, description, filename="layout-test-results.zip", comment_text=comment_text)
304
305
306 class CommitQueue(PatchProcessingQueue, StepSequenceErrorHandler, CommitQueueTaskDelegate):
307     name = "commit-queue"
308     port_name = "mac-mountainlion"
309
310     # AbstractPatchQueue methods
311
312     def begin_work_queue(self):
313         PatchProcessingQueue.begin_work_queue(self)
314         self.committer_validator = CommitterValidator(self._tool)
315         self._expected_failures = ExpectedFailures()
316         self._layout_test_results_reader = LayoutTestResultsReader(self._tool, self._port.results_directory(), self._log_directory())
317
318     def next_work_item(self):
319         return self._next_patch()
320
321     def process_work_item(self, patch):
322         self._cc_watchers(patch.bug_id())
323         task = CommitQueueTask(self, patch)
324         try:
325             if task.run():
326                 self._did_pass(patch)
327                 return True
328             self._did_retry(patch)
329         except ScriptError, e:
330             validator = CommitterValidator(self._tool)
331             validator.reject_patch_from_commit_queue(patch.id(), self._error_message_for_bug(task, patch, e))
332             results_archive = task.results_archive_from_patch_test_run(patch)
333             if results_archive:
334                 self._upload_results_archive_for_patch(patch, results_archive)
335             self._did_fail(patch)
336
337     def _failing_tests_message(self, task, patch):
338         results = task.results_from_patch_test_run(patch)
339         unexpected_failures = self._expected_failures.unexpected_failures_observed(results)
340         if not unexpected_failures:
341             return None
342         return "New failing tests:\n%s" % "\n".join(unexpected_failures)
343
344     def _error_message_for_bug(self, task, patch, script_error):
345         message = self._failing_tests_message(task, patch)
346         if not message:
347             message = script_error.message_with_output()
348         results_link = self._tool.status_server.results_url_for_status(task.failure_status_id)
349         return "%s\nFull output: %s" % (message, results_link)
350
351     def handle_unexpected_error(self, patch, message):
352         self.committer_validator.reject_patch_from_commit_queue(patch.id(), message)
353
354     # CommitQueueTaskDelegate methods
355
356     def run_command(self, command):
357         self.run_webkit_patch(command + [self._deprecated_port.flag()])
358
359     def command_passed(self, message, patch):
360         self._update_status(message, patch=patch)
361
362     def command_failed(self, message, script_error, patch):
363         failure_log = self._log_from_script_error_for_upload(script_error)
364         return self._update_status(message, patch=patch, results_file=failure_log)
365
366     def expected_failures(self):
367         return self._expected_failures
368
369     def test_results(self):
370         return self._layout_test_results_reader.results()
371
372     def archive_last_test_results(self, patch):
373         return self._layout_test_results_reader.archive(patch)
374
375     def build_style(self):
376         return "release"
377
378     def refetch_patch(self, patch):
379         return self._tool.bugs.fetch_attachment(patch.id())
380
381     def report_flaky_tests(self, patch, flaky_test_results, results_archive=None):
382         reporter = FlakyTestReporter(self._tool, self.name)
383         reporter.report_flaky_tests(patch, flaky_test_results, results_archive)
384
385     def did_pass_testing_ews(self, patch):
386         # Only Mac and Mac WK2 run tests
387         # FIXME: We shouldn't have to hard-code it here.
388         patch_status = self._tool.status_server.patch_status
389         return patch_status("mac-ews", patch.id()) == self._pass_status or patch_status("mac-wk2-ews", patch.id()) == self._pass_status
390
391     # StepSequenceErrorHandler methods
392
393     @classmethod
394     def handle_script_error(cls, tool, state, script_error):
395         # Hitting this error handler should be pretty rare.  It does occur,
396         # however, when a patch no longer applies to top-of-tree in the final
397         # land step.
398         _log.error(script_error.message_with_output())
399
400     @classmethod
401     def handle_checkout_needs_update(cls, tool, state, options, error):
402         message = "Tests passed, but commit failed (checkout out of date).  Updating, then landing without building or re-running tests."
403         tool.status_server.update_status(cls.name, message, state["patch"])
404         # The only time when we find out that out checkout needs update is
405         # when we were ready to actually pull the trigger and land the patch.
406         # Rather than spinning in the master process, we retry without
407         # building or testing, which is much faster.
408         options.build = False
409         options.test = False
410         options.update = True
411         raise TryAgain()
412
413
414 class AbstractReviewQueue(PatchProcessingQueue, StepSequenceErrorHandler):
415     """This is the base-class for the EWS queues and the style-queue."""
416     def __init__(self, options=None):
417         PatchProcessingQueue.__init__(self, options)
418
419     def review_patch(self, patch):
420         raise NotImplementedError("subclasses must implement")
421
422     # AbstractPatchQueue methods
423
424     def begin_work_queue(self):
425         PatchProcessingQueue.begin_work_queue(self)
426
427     def next_work_item(self):
428         return self._next_patch()
429
430     def process_work_item(self, patch):
431         passed = self.review_patch(patch)
432         if passed:
433             self._did_pass(patch)
434         return passed
435
436     def handle_unexpected_error(self, patch, message):
437         _log.error(message)
438
439     # StepSequenceErrorHandler methods
440
441     @classmethod
442     def handle_script_error(cls, tool, state, script_error):
443         _log.error(script_error.output)
444
445
446 class StyleQueue(AbstractReviewQueue, StyleQueueTaskDelegate):
447     name = "style-queue"
448
449     def __init__(self):
450         AbstractReviewQueue.__init__(self)
451
452     def review_patch(self, patch):
453         task = StyleQueueTask(self, patch)
454         if not task.validate():
455             self._did_error(patch, "%s did not process patch." % self.name)
456             return False
457         try:
458             style_check_succeeded = task.run()
459             if not style_check_succeeded:
460                 # Caller unlocks when review_patch returns True, so we only need to unlock on transient failure.
461                 self._unlock_patch(patch)
462             return style_check_succeeded
463         except UnableToApplyPatch, e:
464             self._did_error(patch, "%s unable to apply patch." % self.name)
465             return False
466         except ScriptError, e:
467             output = re.sub(r'Failed to run .+ exit_code: 1', '', e.output)
468             message = "Attachment %s did not pass %s:\n\n%s\n\nIf any of these errors are false positives, please file a bug against check-webkit-style." % (patch.id(), self.name, output)
469             self._tool.bugs.post_comment_to_bug(patch.bug_id(), message, cc=self.watchers)
470             self._did_fail(patch)
471             return False
472         return True
473
474     # StyleQueueTaskDelegate methods
475
476     def run_command(self, command):
477         self.run_webkit_patch(command)
478
479     def command_passed(self, message, patch):
480         self._update_status(message, patch=patch)
481
482     def command_failed(self, message, script_error, patch):
483         failure_log = self._log_from_script_error_for_upload(script_error)
484         return self._update_status(message, patch=patch, results_file=failure_log)
485
486     def expected_failures(self):
487         return None
488
489     def refetch_patch(self, patch):
490         return self._tool.bugs.fetch_attachment(patch.id())