We need a CIA replacement
[WebKit-https.git] / Tools / Scripts / webkitpy / tool / commands / queues.py
1 # Copyright (c) 2009 Google Inc. All rights reserved.
2 # Copyright (c) 2009 Apple Inc. All rights reserved.
3 #
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
6 # met:
7 #
8 #     * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 #     * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
13 # distribution.
14 #     * Neither the name of Google Inc. nor the names of its
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
17 #
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30 import codecs
31 import logging
32 import os
33 import sys
34 import time
35 import traceback
36
37 from datetime import datetime
38 from optparse import make_option
39 from StringIO import StringIO
40
41 from webkitpy.common.config.committervalidator import CommitterValidator
42 from webkitpy.common.config.ports import DeprecatedPort
43 from webkitpy.common.net.bugzilla import Attachment
44 from webkitpy.common.net.statusserver import StatusServer
45 from webkitpy.common.system.executive import ScriptError
46 from webkitpy.tool.bot.botinfo import BotInfo
47 from webkitpy.tool.bot.commitqueuetask import CommitQueueTask, CommitQueueTaskDelegate
48 from webkitpy.tool.bot.expectedfailures import ExpectedFailures
49 from webkitpy.tool.bot.feeders import CommitQueueFeeder, EWSFeeder
50 from webkitpy.tool.bot.flakytestreporter import FlakyTestReporter
51 from webkitpy.tool.bot.layouttestresultsreader import LayoutTestResultsReader
52 from webkitpy.tool.bot.patchanalysistask import UnableToApplyPatch
53 from webkitpy.tool.bot.queueengine import QueueEngine, QueueEngineDelegate
54 from webkitpy.tool.bot.stylequeuetask import StyleQueueTask, StyleQueueTaskDelegate
55 from webkitpy.tool.commands.stepsequence import StepSequenceErrorHandler
56 from webkitpy.tool.multicommandtool import Command, TryAgain
57
58 _log = logging.getLogger(__name__)
59
60
61 class AbstractQueue(Command, QueueEngineDelegate):
62     watchers = [
63     ]
64
65     _pass_status = "Pass"
66     _fail_status = "Fail"
67     _retry_status = "Retry"
68     _error_status = "Error"
69
70     def __init__(self, options=None): # Default values should never be collections (like []) as default values are shared between invocations
71         options_list = (options or []) + [
72             make_option("--no-confirm", action="store_false", dest="confirm", default=True, help="Do not ask the user for confirmation before running the queue.  Dangerous!"),
73             make_option("--exit-after-iteration", action="store", type="int", dest="iterations", default=None, help="Stop running the queue after iterating this number of times."),
74         ]
75         Command.__init__(self, "Run the %s" % self.name, options=options_list)
76         self._iteration_count = 0
77
78     def _cc_watchers(self, bug_id):
79         try:
80             self._tool.bugs.add_cc_to_bug(bug_id, self.watchers)
81         except Exception, e:
82             traceback.print_exc()
83             _log.error("Failed to CC watchers.")
84
85     def run_webkit_patch(self, args):
86         webkit_patch_args = [self._tool.path()]
87         # FIXME: This is a hack, we should have a more general way to pass global options.
88         # FIXME: We must always pass global options and their value in one argument
89         # because our global option code looks for the first argument which does
90         # not begin with "-" and assumes that is the command name.
91         webkit_patch_args += ["--status-host=%s" % self._tool.status_server.host]
92         if self._tool.status_server.bot_id:
93             webkit_patch_args += ["--bot-id=%s" % self._tool.status_server.bot_id]
94         if self._options.port:
95             webkit_patch_args += ["--port=%s" % self._options.port]
96         webkit_patch_args.extend(args)
97
98         try:
99             args_for_printing = list(webkit_patch_args)
100             args_for_printing[0] = 'webkit-patch'  # Printing our path for each log is redundant.
101             _log.info("Running: %s" % self._tool.executive.command_for_printing(args_for_printing))
102             command_output = self._tool.executive.run_command(webkit_patch_args, cwd=self._tool.scm().checkout_root)
103         except ScriptError, e:
104             # Make sure the whole output gets printed if the command failed.
105             _log.error(e.message_with_output(output_limit=None))
106             raise
107         return command_output
108
109     def _log_directory(self):
110         return os.path.join("..", "%s-logs" % self.name)
111
112     # QueueEngineDelegate methods
113
114     def queue_log_path(self):
115         return os.path.join(self._log_directory(), "%s.log" % self.name)
116
117     def work_item_log_path(self, work_item):
118         raise NotImplementedError, "subclasses must implement"
119
120     def begin_work_queue(self):
121         _log.info("CAUTION: %s will discard all local changes in \"%s\"" % (self.name, self._tool.scm().checkout_root))
122         if self._options.confirm:
123             response = self._tool.user.prompt("Are you sure?  Type \"yes\" to continue: ")
124             if (response != "yes"):
125                 _log.error("User declined.")
126                 sys.exit(1)
127         _log.info("Running WebKit %s." % self.name)
128         self._tool.status_server.update_status(self.name, "Starting Queue")
129
130     def stop_work_queue(self, reason):
131         self._tool.status_server.update_status(self.name, "Stopping Queue, reason: %s" % reason)
132
133     def should_continue_work_queue(self):
134         self._iteration_count += 1
135         return not self._options.iterations or self._iteration_count <= self._options.iterations
136
137     def next_work_item(self):
138         raise NotImplementedError, "subclasses must implement"
139
140     def process_work_item(self, work_item):
141         raise NotImplementedError, "subclasses must implement"
142
143     def handle_unexpected_error(self, work_item, message):
144         raise NotImplementedError, "subclasses must implement"
145
146     # Command methods
147
148     def execute(self, options, args, tool, engine=QueueEngine):
149         self._options = options # FIXME: This code is wrong.  Command.options is a list, this assumes an Options element!
150         self._tool = tool  # FIXME: This code is wrong too!  Command.bind_to_tool handles this!
151         return engine(self.name, self, self._tool.wakeup_event, self._options.seconds_to_sleep).run()
152
153     @classmethod
154     def _log_from_script_error_for_upload(cls, script_error, output_limit=None):
155         # We have seen request timeouts with app engine due to large
156         # log uploads.  Trying only the last 512k.
157         if not output_limit:
158             output_limit = 512 * 1024  # 512k
159         output = script_error.message_with_output(output_limit=output_limit)
160         # We pre-encode the string to a byte array before passing it
161         # to status_server, because ClientForm (part of mechanize)
162         # wants a file-like object with pre-encoded data.
163         return StringIO(output.encode("utf-8"))
164
165     @classmethod
166     def _update_status_for_script_error(cls, tool, state, script_error, is_error=False):
167         message = str(script_error)
168         if is_error:
169             message = "Error: %s" % message
170         failure_log = cls._log_from_script_error_for_upload(script_error)
171         return tool.status_server.update_status(cls.name, message, state["patch"], failure_log)
172
173
174 class FeederQueue(AbstractQueue):
175     name = "feeder-queue"
176
177     _sleep_duration = 30  # seconds
178
179     # AbstractQueue methods
180
181     def begin_work_queue(self):
182         AbstractQueue.begin_work_queue(self)
183         self.feeders = [
184             CommitQueueFeeder(self._tool),
185             EWSFeeder(self._tool),
186         ]
187
188     def next_work_item(self):
189         # This really show inherit from some more basic class that doesn't
190         # understand work items, but the base class in the heirarchy currently
191         # understands work items.
192         return "synthetic-work-item"
193
194     def process_work_item(self, work_item):
195         for feeder in self.feeders:
196             feeder.feed()
197         time.sleep(self._sleep_duration)
198         return True
199
200     def work_item_log_path(self, work_item):
201         return None
202
203     def handle_unexpected_error(self, work_item, message):
204         _log.error(message)
205
206
207 class AbstractPatchQueue(AbstractQueue):
208     def _update_status(self, message, patch=None, results_file=None):
209         return self._tool.status_server.update_status(self.name, message, patch, results_file)
210
211     def _next_patch(self):
212         # FIXME: Bugzilla accessibility should be checked here; if it's unaccessible,
213         # it should return None.
214         patch = None
215         while not patch:
216             patch_id = self._tool.status_server.next_work_item(self.name)
217             if not patch_id:
218                 return None
219             patch = self._tool.bugs.fetch_attachment(patch_id)
220             if not patch:
221                 # FIXME: Using a fake patch because release_work_item has the wrong API.
222                 # We also don't really need to release the lock (although that's fine),
223                 # mostly we just need to remove this bogus patch from our queue.
224                 # If for some reason bugzilla is just down, then it will be re-fed later.
225                 fake_patch = Attachment({'id': patch_id}, None)
226                 self._release_work_item(fake_patch)
227         return patch
228
229     def _release_work_item(self, patch):
230         self._tool.status_server.release_work_item(self.name, patch)
231
232     def _did_pass(self, patch):
233         self._update_status(self._pass_status, patch)
234         self._release_work_item(patch)
235
236     def _did_fail(self, patch):
237         self._update_status(self._fail_status, patch)
238         self._release_work_item(patch)
239
240     def _did_retry(self, patch):
241         self._update_status(self._retry_status, patch)
242         self._release_work_item(patch)
243
244     def _did_error(self, patch, reason):
245         message = "%s: %s" % (self._error_status, reason)
246         self._update_status(message, patch)
247         self._release_work_item(patch)
248
249     def work_item_log_path(self, patch):
250         return os.path.join(self._log_directory(), "%s.log" % patch.bug_id())
251
252
253 # Used to share code between the EWS and commit-queue.
254 class PatchProcessingQueue(AbstractPatchQueue):
255     # Subclasses must override.
256     port_name = None
257
258     # FIXME: This is a hack to map between the old port names and the new port names.
259     def _new_port_name_from_old(self, port_name):
260         # The new port system has no concept of xvfb yet.
261         if port_name == 'chromium-xvfb':
262             return 'chromium'
263         # ApplePort.determine_full_port_name asserts if the name doesn't include version.
264         if port_name == 'mac':
265             return 'mac-future'
266         if port_name == 'win':
267             return 'win-future'
268         return port_name
269
270     def begin_work_queue(self):
271         AbstractPatchQueue.begin_work_queue(self)
272         if not self.port_name:
273             return
274         # FIXME: This is only used for self._deprecated_port.flag()
275         self._deprecated_port = DeprecatedPort.port(self.port_name)
276         # FIXME: This violates abstraction
277         self._tool._deprecated_port = self._deprecated_port
278         self._port = self._tool.port_factory.get(self._new_port_name_from_old(self.port_name))
279
280     def _upload_results_archive_for_patch(self, patch, results_archive_zip):
281         bot_id = self._tool.status_server.bot_id or "bot"
282         description = "Archive of layout-test-results from %s" % bot_id
283         # results_archive is a ZipFile object, grab the File object (.fp) to pass to Mechanize for uploading.
284         results_archive_file = results_archive_zip.fp
285         # Rewind the file object to start (since Mechanize won't do that automatically)
286         # See https://bugs.webkit.org/show_bug.cgi?id=54593
287         results_archive_file.seek(0)
288         # FIXME: This is a small lie to always say run-webkit-tests since Chromium uses new-run-webkit-tests.
289         # We could make this code look up the test script name off the port.
290         comment_text = "The attached test failures were seen while running run-webkit-tests on the %s.\n" % (self.name)
291         # FIXME: We could easily list the test failures from the archive here,
292         # currently callers do that separately.
293         comment_text += BotInfo(self._tool).summary_text()
294         self._tool.bugs.add_attachment_to_bug(patch.bug_id(), results_archive_file, description, filename="layout-test-results.zip", comment_text=comment_text)
295
296
297 class CommitQueue(PatchProcessingQueue, StepSequenceErrorHandler, CommitQueueTaskDelegate):
298     name = "commit-queue"
299     port_name = "chromium-xvfb"
300
301     # AbstractPatchQueue methods
302
303     def begin_work_queue(self):
304         PatchProcessingQueue.begin_work_queue(self)
305         self.committer_validator = CommitterValidator(self._tool)
306         self._expected_failures = ExpectedFailures()
307         self._layout_test_results_reader = LayoutTestResultsReader(self._tool, self._port.results_directory(), self._log_directory())
308
309     def next_work_item(self):
310         return self._next_patch()
311
312     def process_work_item(self, patch):
313         self._cc_watchers(patch.bug_id())
314         task = CommitQueueTask(self, patch)
315         try:
316             if task.run():
317                 self._did_pass(patch)
318                 return True
319             self._did_retry(patch)
320         except ScriptError, e:
321             validator = CommitterValidator(self._tool)
322             validator.reject_patch_from_commit_queue(patch.id(), self._error_message_for_bug(task, patch, e))
323             results_archive = task.results_archive_from_patch_test_run(patch)
324             if results_archive:
325                 self._upload_results_archive_for_patch(patch, results_archive)
326             self._did_fail(patch)
327
328     def _failing_tests_message(self, task, patch):
329         results = task.results_from_patch_test_run(patch)
330         unexpected_failures = self._expected_failures.unexpected_failures_observed(results)
331         if not unexpected_failures:
332             return None
333         return "New failing tests:\n%s" % "\n".join(unexpected_failures)
334
335     def _error_message_for_bug(self, task, patch, script_error):
336         message = self._failing_tests_message(task, patch)
337         if not message:
338             message = script_error.message_with_output()
339         results_link = self._tool.status_server.results_url_for_status(task.failure_status_id)
340         return "%s\nFull output: %s" % (message, results_link)
341
342     def handle_unexpected_error(self, patch, message):
343         self.committer_validator.reject_patch_from_commit_queue(patch.id(), message)
344
345     # CommitQueueTaskDelegate methods
346
347     def run_command(self, command):
348         self.run_webkit_patch(command + [self._deprecated_port.flag()])
349
350     def command_passed(self, message, patch):
351         self._update_status(message, patch=patch)
352
353     def command_failed(self, message, script_error, patch):
354         failure_log = self._log_from_script_error_for_upload(script_error)
355         return self._update_status(message, patch=patch, results_file=failure_log)
356
357     def expected_failures(self):
358         return self._expected_failures
359
360     def test_results(self):
361         return self._layout_test_results_reader.results()
362
363     def archive_last_test_results(self, patch):
364         return self._layout_test_results_reader.archive(patch)
365
366     def build_style(self):
367         return "release"
368
369     def refetch_patch(self, patch):
370         return self._tool.bugs.fetch_attachment(patch.id())
371
372     def report_flaky_tests(self, patch, flaky_test_results, results_archive=None):
373         reporter = FlakyTestReporter(self._tool, self.name)
374         reporter.report_flaky_tests(patch, flaky_test_results, results_archive)
375
376     def did_pass_testing_ews(self, patch):
377         # Currently, chromium-ews is the only testing EWS. Once there are more,
378         # should make sure they all pass.
379         status = self._tool.status_server.patch_status("chromium-ews", patch.id())
380         return status == self._pass_status
381
382     # StepSequenceErrorHandler methods
383
384     @classmethod
385     def handle_script_error(cls, tool, state, script_error):
386         # Hitting this error handler should be pretty rare.  It does occur,
387         # however, when a patch no longer applies to top-of-tree in the final
388         # land step.
389         _log.error(script_error.message_with_output())
390
391     @classmethod
392     def handle_checkout_needs_update(cls, tool, state, options, error):
393         message = "Tests passed, but commit failed (checkout out of date).  Updating, then landing without building or re-running tests."
394         tool.status_server.update_status(cls.name, message, state["patch"])
395         # The only time when we find out that out checkout needs update is
396         # when we were ready to actually pull the trigger and land the patch.
397         # Rather than spinning in the master process, we retry without
398         # building or testing, which is much faster.
399         options.build = False
400         options.test = False
401         options.update = True
402         raise TryAgain()
403
404
405 class AbstractReviewQueue(PatchProcessingQueue, StepSequenceErrorHandler):
406     """This is the base-class for the EWS queues and the style-queue."""
407     def __init__(self, options=None):
408         PatchProcessingQueue.__init__(self, options)
409
410     def review_patch(self, patch):
411         raise NotImplementedError("subclasses must implement")
412
413     # AbstractPatchQueue methods
414
415     def begin_work_queue(self):
416         PatchProcessingQueue.begin_work_queue(self)
417
418     def next_work_item(self):
419         return self._next_patch()
420
421     def process_work_item(self, patch):
422         try:
423             if not self.review_patch(patch):
424                 return False
425             self._did_pass(patch)
426             return True
427         except ScriptError, e:
428             if e.exit_code != QueueEngine.handled_error_code:
429                 self._did_fail(patch)
430             else:
431                 # The subprocess handled the error, but won't have released the patch, so we do.
432                 # FIXME: We need to simplify the rules by which _release_work_item is called.
433                 self._release_work_item(patch)
434             raise e
435
436     def handle_unexpected_error(self, patch, message):
437         _log.error(message)
438
439     # StepSequenceErrorHandler methods
440
441     @classmethod
442     def handle_script_error(cls, tool, state, script_error):
443         _log.error(script_error.output)
444
445
446 class StyleQueue(AbstractReviewQueue, StyleQueueTaskDelegate):
447     name = "style-queue"
448
449     def __init__(self):
450         AbstractReviewQueue.__init__(self)
451
452     def review_patch(self, patch):
453         task = StyleQueueTask(self, patch)
454         if not task.validate():
455             self._did_error(patch, "%s did not process patch." % self.name)
456             return False
457         try:
458             return task.run()
459         except UnableToApplyPatch, e:
460             self._did_error(patch, "%s unable to apply patch." % self.name)
461             return False
462         except ScriptError, e:
463             message = "Attachment %s did not pass %s:\n\n%s\n\nIf any of these errors are false positives, please file a bug against check-webkit-style." % (patch.id(), self.name, e.output)
464             self._tool.bugs.post_comment_to_bug(patch.bug_id(), message, cc=self.watchers)
465             self._did_fail(patch)
466             return False
467         return True
468
469     # StyleQueueTaskDelegate methods
470
471     def run_command(self, command):
472         self.run_webkit_patch(command)
473
474     def command_passed(self, message, patch):
475         self._update_status(message, patch=patch)
476
477     def command_failed(self, message, script_error, patch):
478         failure_log = self._log_from_script_error_for_upload(script_error)
479         return self._update_status(message, patch=patch, results_file=failure_log)
480
481     def expected_failures(self):
482         return None
483
484     def refetch_patch(self, patch):
485         return self._tool.bugs.fetch_attachment(patch.id())