fe34945e387df4855a3bbd0bc90add8bf1add8e3
[WebKit-https.git] / WebKitTools / Scripts / bugzilla-tool
1 #!/usr/bin/env python
2 # Copyright (c) 2009, Google Inc. All rights reserved.
3 # Copyright (c) 2009 Apple Inc. All rights reserved.
4 #
5 # Redistribution and use in source and binary forms, with or without
6 # modification, are permitted provided that the following conditions are
7 # met:
8
9 #     * Redistributions of source code must retain the above copyright
10 # notice, this list of conditions and the following disclaimer.
11 #     * Redistributions in binary form must reproduce the above
12 # copyright notice, this list of conditions and the following disclaimer
13 # in the documentation and/or other materials provided with the
14 # distribution.
15 #     * Neither the name of Google Inc. nor the names of its
16 # contributors may be used to endorse or promote products derived from
17 # this software without specific prior written permission.
18
19 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30 #
31 # A tool for automating dealing with bugzilla, posting patches, committing patches, etc.
32
33 import os
34 import re
35 import StringIO # for add_patch_to_bug file wrappers
36 import subprocess
37 import sys
38 import time
39
40 from datetime import datetime, timedelta
41 from optparse import make_option
42
43 # Import WebKit-specific modules.
44 from modules.bugzilla import Bugzilla, parse_bug_id
45 from modules.buildbot import BuildBot
46 from modules.changelogs import ChangeLog
47 from modules.comments import bug_comment_from_commit_text
48 from modules.logging import error, log, tee
49 from modules.multicommandtool import MultiCommandTool, Command
50 from modules.patchcollection import PatchCollection
51 from modules.scm import CommitMessage, detect_scm_system, ScriptError, CheckoutNeedsUpdate
52 from modules.statusbot import StatusBot
53 from modules.webkitport import WebKitPort
54 from modules.workqueue import WorkQueue, WorkQueueDelegate
55
56
57 def plural(noun):
58     # This is a dumb plural() implementation which was just enough for our uses.
59     if re.search("h$", noun):
60         return noun + "es"
61     else:
62         return noun + "s"
63
64 def pluralize(noun, count):
65     if count != 1:
66         noun = plural(noun)
67     return "%d %s" % (count, noun)
68
69 def commit_message_for_this_commit(scm):
70     changelog_paths = scm.modified_changelogs()
71     if not len(changelog_paths):
72         raise ScriptError(message="Found no modified ChangeLogs, cannot create a commit message.\n"
73                           "All changes require a ChangeLog.  See:\n"
74                           "http://webkit.org/coding/contributing.html")
75
76     changelog_messages = []
77     for changelog_path in changelog_paths:
78         log("Parsing ChangeLog: %s" % changelog_path)
79         changelog_entry = ChangeLog(changelog_path).latest_entry()
80         if not changelog_entry:
81             raise ScriptError(message="Failed to parse ChangeLog: " + os.path.abspath(changelog_path))
82         changelog_messages.append(changelog_entry)
83
84     # FIXME: We should sort and label the ChangeLog messages like commit-log-editor does.
85     return CommitMessage("".join(changelog_messages).splitlines())
86
87
88 class BugsToCommit(Command):
89     def __init__(self):
90         Command.__init__(self, "Bugs in the commit queue")
91
92     def execute(self, options, args, tool):
93         bug_ids = tool.bugs.fetch_bug_ids_from_commit_queue()
94         for bug_id in bug_ids:
95             print "%s" % bug_id
96
97
98 class PatchesToCommit(Command):
99     def __init__(self):
100         Command.__init__(self, "Patches in the commit queue")
101
102     def execute(self, options, args, tool):
103         patches = tool.bugs.fetch_patches_from_commit_queue()
104         log("Patches in commit queue:")
105         for patch in patches:
106             print "%s" % patch["url"]
107
108
109 class ReviewedPatches(Command):
110     def __init__(self):
111         Command.__init__(self, "r+\'d patches on a bug", "BUGID")
112
113     def execute(self, options, args, tool):
114         bug_id = args[0]
115         patches_to_land = tool.bugs.fetch_reviewed_patches_from_bug(bug_id)
116         for patch in patches_to_land:
117             print "%s" % patch["url"]
118
119
120 class CheckStyle(Command):
121     def __init__(self):
122         options = WebKitLandingScripts.cleaning_options()
123         Command.__init__(self, "Runs check-webkit-style on the specified attachment", "ATTACHMENT_ID", options=options)
124
125     @classmethod
126     def check_style(cls, patch, options, tool):
127         tool.scm().update_webkit()
128         log("Checking style for patch %s from bug %s." % (patch["id"], patch["bug_id"]))
129         try:
130             # FIXME: check-webkit-style shouldn't really have to apply the patch to check the style.
131             tool.scm().apply_patch(patch)
132             WebKitLandingScripts.run_webkit_script("check-webkit-style")
133         except ScriptError, e:
134             log("Patch %s from bug %s failed to apply and check style." % (patch["id"], patch["bug_id"]))
135             log(e.output)
136
137         # This is safe because in order to get here the working directory had to be
138         # clean at the beginning.  Clean it out again before we exit.
139         tool.scm().ensure_clean_working_directory(force_clean=True)
140
141     def execute(self, options, args, tool):
142         attachment_id = args[0]
143         attachment = tool.bugs.fetch_attachment(attachment_id)
144
145         WebKitLandingScripts.prepare_clean_working_directory(tool.scm(), options)
146         self.check_style(attachment, options, tool)
147
148
149 class ApplyAttachment(Command):
150     def __init__(self):
151         options = WebKitApplyingScripts.apply_options() + WebKitLandingScripts.cleaning_options()
152         Command.__init__(self, "Applies an attachment to the local working directory.", "ATTACHMENT_ID", options=options)
153
154     def execute(self, options, args, tool):
155         WebKitApplyingScripts.setup_for_patch_apply(tool.scm(), options)
156         attachment_id = args[0]
157         attachment = tool.bugs.fetch_attachment(attachment_id)
158         WebKitApplyingScripts.apply_patches_with_options(tool.scm(), [attachment], options)
159
160
161 class ApplyPatches(Command):
162     def __init__(self):
163         options = WebKitApplyingScripts.apply_options() + WebKitLandingScripts.cleaning_options()
164         Command.__init__(self, "Applies all patches on a bug to the local working directory.", "BUGID", options=options)
165
166     def execute(self, options, args, tool):
167         WebKitApplyingScripts.setup_for_patch_apply(tool.scm(), options)
168         bug_id = args[0]
169         patches = tool.bugs.fetch_reviewed_patches_from_bug(bug_id)
170         WebKitApplyingScripts.apply_patches_with_options(tool.scm(), patches, options)
171
172
173 class WebKitApplyingScripts:
174     @staticmethod
175     def apply_options():
176         return [
177             make_option("--no-update", action="store_false", dest="update", default=True, help="Don't update the working directory before applying patches"),
178             make_option("--local-commit", action="store_true", dest="local_commit", default=False, help="Make a local commit for each applied patch"),
179         ]
180
181     @staticmethod
182     def setup_for_patch_apply(scm, options):
183         WebKitLandingScripts.prepare_clean_working_directory(scm, options, allow_local_commits=True)
184         if options.update:
185             scm.update_webkit()
186
187     @staticmethod
188     def apply_patches_with_options(scm, patches, options):
189         if options.local_commit and not scm.supports_local_commits():
190             error("--local-commit passed, but %s does not support local commits" % scm.display_name())
191
192         for patch in patches:
193             log("Applying attachment %s from bug %s" % (patch["id"], patch["bug_id"]))
194             scm.apply_patch(patch)
195             if options.local_commit:
196                 commit_message = commit_message_for_this_commit(scm)
197                 scm.commit_locally_with_message(commit_message.message() or patch["name"])
198
199
200 class WebKitLandingScripts:
201     @staticmethod
202     def cleaning_options():
203         return [
204             make_option("--force-clean", action="store_true", dest="force_clean", default=False, help="Clean working directory before applying patches (removes local changes and commits)"),
205             make_option("--no-clean", action="store_false", dest="clean", default=True, help="Don't check if the working directory is clean before applying patches"),
206         ]
207
208     @staticmethod
209     def land_options():
210         return [
211             make_option("--ignore-builders", action="store_false", dest="check_builders", default=True, help="Don't check to see if the build.webkit.org builders are green before landing."),
212             make_option("--no-close", action="store_false", dest="close_bug", default=True, help="Leave bug open after landing."),
213             make_option("--no-build", action="store_false", dest="build", default=True, help="Commit without building first, implies --no-test."),
214             make_option("--no-test", action="store_false", dest="test", default=True, help="Commit without running run-webkit-tests."),
215             make_option("--quiet", action="store_true", dest="quiet", default=False, help="Produce less console output."),
216             make_option("--non-interactive", action="store_true", dest="non_interactive", default=False, help="Never prompt the user, fail as fast as possible."),
217         ] + WebKitPort.port_options()
218
219     @staticmethod
220     def run_command_with_teed_output(args, teed_output):
221         child_process = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
222
223         # Use our own custom wait loop because Popen ignores a tee'd stderr/stdout.
224         # FIXME: This could be improved not to flatten output to stdout.
225         while True:
226             output_line = child_process.stdout.readline()
227             if output_line == "" and child_process.poll() != None:
228                 return child_process.poll()
229             teed_output.write(output_line)
230
231     @staticmethod
232     def run_and_throw_if_fail(args, quiet=False):
233         # Cache the child's output locally so it can be used for error reports.
234         child_out_file = StringIO.StringIO()
235         if quiet:
236             dev_null = open(os.devnull, "w")
237         child_stdout = tee(child_out_file, dev_null if quiet else sys.stdout)
238         exit_code = WebKitLandingScripts.run_command_with_teed_output(args, child_stdout)
239         if quiet:
240             dev_null.close()
241
242         child_output = child_out_file.getvalue()
243         child_out_file.close()
244
245         if exit_code:
246             raise ScriptError(script_args=args, exit_code=exit_code, output=child_output)
247
248     @classmethod
249     def run_webkit_script(cls, script_name, quiet=False, port=WebKitPort):
250         log("Running %s" % script_name)
251         cls.run_and_throw_if_fail(port.script_path(script_name), quiet)
252
253     @classmethod
254     def build_webkit(cls, quiet=False, port=WebKitPort):
255         log("Building WebKit")
256         cls.run_and_throw_if_fail(port.build_webkit_command(), quiet)
257
258     @staticmethod
259     def ensure_builders_are_green(buildbot, options):
260         if not options.check_builders or buildbot.core_builders_are_green():
261             return
262         error("Builders at %s are red, please do not commit.  Pass --ignore-builders to bypass this check." % (buildbot.buildbot_host))
263
264     @classmethod
265     def run_webkit_tests(cls, launch_safari, fail_fast=False, quiet=False, port=WebKitPort):
266         args = port.run_webkit_tests_command()
267         if not launch_safari:
268             args.append("--no-launch-safari")
269         if quiet:
270             args.append("--quiet")
271         if fail_fast:
272             args.append("--exit-after-n-failures=1")
273         cls.run_and_throw_if_fail(args)
274
275     @staticmethod
276     def prepare_clean_working_directory(scm, options, allow_local_commits=False):
277         os.chdir(scm.checkout_root)
278         if not allow_local_commits:
279             scm.ensure_no_local_commits(options.force_clean)
280         if options.clean:
281             scm.ensure_clean_working_directory(force_clean=options.force_clean)
282
283     @classmethod
284     def build_and_commit(cls, scm, options):
285         port = WebKitPort.get_port(options)
286         if options.build:
287             cls.build_webkit(quiet=options.quiet, port=port)
288             if options.test:
289                 # When running the commit-queue we don't want to launch Safari and we want to exit after the first failure.
290                 cls.run_webkit_tests(launch_safari=not options.non_interactive, fail_fast=options.non_interactive, quiet=options.quiet, port=port)
291         commit_message = commit_message_for_this_commit(scm)
292         commit_log = scm.commit_with_message(commit_message.message())
293         return bug_comment_from_commit_text(scm, commit_log)
294
295     @classmethod
296     def _close_bug_if_no_active_patches(cls, bugs, bug_id):
297         # Check to make sure there are no r? or r+ patches on the bug before closing.
298         # Assume that r- patches are just previous patches someone forgot to obsolete.
299         patches = bugs.fetch_patches_from_bug(bug_id)
300         for patch in patches:
301             review_flag = patch.get("review")
302             if review_flag == "?" or review_flag == "+":
303                 log("Not closing bug %s as attachment %s has review=%s.  Assuming there are more patches to land from this bug." % (patch["bug_id"], patch["id"], review_flag))
304                 return
305         bugs.close_bug_as_fixed(bug_id, "All reviewed patches have been landed.  Closing bug.")
306
307     @classmethod
308     def _land_patch(cls, patch, options, tool):
309         tool.scm().update_webkit() # Update before every patch in case the tree has changed
310         log("Applying patch %s from bug %s." % (patch["id"], patch["bug_id"]))
311         tool.scm().apply_patch(patch, force=options.non_interactive)
312
313         # Make sure the tree is still green after updating, before building this patch.
314         # The first patch ends up checking tree status twice, but that's OK.
315         WebKitLandingScripts.ensure_builders_are_green(tool.buildbot, options)
316         comment_text = WebKitLandingScripts.build_and_commit(tool.scm(), options)
317         tool.bugs.clear_attachment_flags(patch["id"], comment_text)
318
319     @classmethod
320     def land_patch_and_handle_errors(cls, patch, options, tool):
321         try:
322             cls._land_patch(patch, options, tool)
323             if options.close_bug:
324                 cls._close_bug_if_no_active_patches(tool.bugs, patch["bug_id"])
325         except CheckoutNeedsUpdate, e:
326             log("Commit failed because the checkout is out of date.  Please update and try again.")
327             log("You can pass --no-build to skip building/testing after update if you believe the new commits did not affect the results.")
328             WorkQueue.exit_after_handled_error(e)
329         except ScriptError, e:
330             # Mark the patch as commit-queue- and comment in the bug.
331             tool.bugs.reject_patch_from_commit_queue(patch["id"], e.message_with_output())
332             WorkQueue.exit_after_handled_error(e)
333
334
335 class LandDiff(Command):
336     def __init__(self):
337         options = [
338             make_option("-r", "--reviewer", action="store", type="string", dest="reviewer", help="Update ChangeLogs to say Reviewed by REVIEWER."),
339         ]
340         options += WebKitLandingScripts.land_options()
341         Command.__init__(self, "Lands the current working directory diff and updates the bug if provided.", "[BUGID]", options=options)
342
343     def guess_reviewer_from_bug(self, bugs, bug_id):
344         patches = bugs.fetch_reviewed_patches_from_bug(bug_id)
345         if len(patches) != 1:
346             log("%s on bug %s, cannot infer reviewer." % (pluralize("reviewed patch", len(patches)), bug_id))
347             return None
348         patch = patches[0]
349         reviewer = patch["reviewer"]
350         log("Guessing \"%s\" as reviewer from attachment %s on bug %s." % (reviewer, patch["id"], bug_id))
351         return reviewer
352
353     def update_changelogs_with_reviewer(self, reviewer, bug_id, tool):
354         if not reviewer:
355             if not bug_id:
356                 log("No bug id provided and --reviewer= not provided.  Not updating ChangeLogs with reviewer.")
357                 return
358             reviewer = self.guess_reviewer_from_bug(tool.bugs, bug_id)
359
360         if not reviewer:
361             log("Failed to guess reviewer from bug %s and --reviewer= not provided.  Not updating ChangeLogs with reviewer." % bug_id)
362             return
363
364         for changelog_path in tool.scm().modified_changelogs():
365             ChangeLog(changelog_path).set_reviewer(reviewer)
366
367     def execute(self, options, args, tool):
368         bug_id = (args and args[0]) or parse_bug_id(tool.scm().create_patch())
369
370         WebKitLandingScripts.ensure_builders_are_green(tool.buildbot, options)
371
372         os.chdir(tool.scm().checkout_root)
373         self.update_changelogs_with_reviewer(options.reviewer, bug_id, tool)
374
375         comment_text = WebKitLandingScripts.build_and_commit(tool.scm(), options)
376         if bug_id:
377             log("Updating bug %s" % bug_id)
378             if options.close_bug:
379                 tool.bugs.close_bug_as_fixed(bug_id, comment_text)
380             else:
381                 # FIXME: We should a smart way to figure out if the patch is attached
382                 # to the bug, and if so obsolete it.
383                 tool.bugs.post_comment_to_bug(bug_id, comment_text)
384         else:
385             log(comment_text)
386             log("No bug id provided.")
387
388
389 class AbstractPatchLandingCommand(Command):
390     def __init__(self, description, args_description):
391         options = WebKitLandingScripts.cleaning_options() + WebKitLandingScripts.land_options()
392         Command.__init__(self, description, args_description, options=options)
393
394     @staticmethod
395     def _fetch_list_of_patches_to_land(options, args, tool):
396         raise NotImplementedError, "subclasses must implement"
397
398     @staticmethod
399     def _collect_patches_by_bug(patches):
400         bugs_to_patches = {}
401         for patch in patches:
402             bug_id = patch["bug_id"]
403             bugs_to_patches[bug_id] = bugs_to_patches.get(bug_id, []).append(patch)
404         return bugs_to_patches
405
406     def execute(self, options, args, tool):
407         if not args:
408             error("%s required" % self.argument_names)
409
410         # Check the tree status first so we can fail early.
411         WebKitLandingScripts.ensure_builders_are_green(tool.buildbot, options)
412         WebKitLandingScripts.prepare_clean_working_directory(tool.scm(), options)
413
414         patches = self._fetch_list_of_patches_to_land(options, args, tool)
415
416         # It's nice to print out total statistics.
417         bugs_to_patches = self._collect_patches_by_bug(patches)
418         log("Landing %s from %s." % (pluralize("patch", len(patches)), pluralize("bug", len(bugs_to_patches))))
419
420         for patch in patches:
421             WebKitLandingScripts.land_patch_and_handle_errors(patch, options, tool)
422
423
424 class LandAttachment(AbstractPatchLandingCommand):
425     def __init__(self):
426         AbstractPatchLandingCommand.__init__(self, "Lands a patches from bugzilla, optionally building and testing them first", "ATTACHMENT_ID [ATTACHMENT_IDS]")
427
428     @staticmethod
429     def _fetch_list_of_patches_to_land(options, args, tool):
430         return map(lambda patch_id: tool.bugs.fetch_attachment(patch_id), args)
431
432
433 class LandPatches(AbstractPatchLandingCommand):
434     def __init__(self):
435         AbstractPatchLandingCommand.__init__(self, "Lands all patches on the given bugs, optionally building and testing them first", "BUGID [BUGIDS]")
436
437     @staticmethod
438     def _fetch_list_of_patches_to_land(options, args, tool):
439         all_patches = []
440         for bug_id in args:
441             patches = tool.bugs.fetch_reviewed_patches_from_bug(bug_id)
442             log("%s found on bug %s." % (pluralize("reviewed patch", len(patches)), bug_id))
443             all_patches += patches
444         return all_patches
445
446
447 class CommitMessageForCurrentDiff(Command):
448     def __init__(self):
449         Command.__init__(self, "Prints a commit message suitable for the uncommitted changes.")
450
451     def execute(self, options, args, tool):
452         os.chdir(tool.scm().checkout_root)
453         print "%s" % commit_message_for_this_commit(tool.scm()).message()
454
455
456 class ObsoleteAttachments(Command):
457     def __init__(self):
458         Command.__init__(self, "Marks all attachments on a bug as obsolete.", "BUGID")
459
460     def execute(self, options, args, tool):
461         bug_id = args[0]
462         attachments = tool.bugs.fetch_attachments_from_bug(bug_id)
463         for attachment in attachments:
464             if not attachment["is_obsolete"]:
465                 tool.bugs.obsolete_attachment(attachment["id"])
466
467
468 class PostDiff(Command):
469     def __init__(self):
470         options = [
471             make_option("-m", "--description", action="store", type="string", dest="description", help="Description string for the attachment (default: \"patch\")"),
472         ]
473         options += self.posting_options()
474         Command.__init__(self, "Attaches the current working directory diff to a bug as a patch file.", "[BUGID]", options=options)
475
476     @staticmethod
477     def posting_options():
478         return [
479             make_option("--no-obsolete", action="store_false", dest="obsolete_patches", default=True, help="Do not obsolete old patches before posting this one."),
480             make_option("--no-review", action="store_false", dest="review", default=True, help="Do not mark the patch for review."),
481             make_option("--request-commit", action="store_true", dest="request_commit", default=False, help="Mark the patch as needing auto-commit after review."),
482         ]
483
484     @staticmethod
485     def obsolete_patches_on_bug(bug_id, bugs):
486         patches = bugs.fetch_patches_from_bug(bug_id)
487         if len(patches):
488             log("Obsoleting %s on bug %s" % (pluralize("old patch", len(patches)), bug_id))
489             for patch in patches:
490                 bugs.obsolete_attachment(patch["id"])
491
492     def execute(self, options, args, tool):
493         # Perfer a bug id passed as an argument over a bug url in the diff (i.e. ChangeLogs).
494         bug_id = (args and args[0]) or parse_bug_id(tool.scm().create_patch())
495         if not bug_id:
496             error("No bug id passed and no bug url found in diff, can't post.")
497
498         if options.obsolete_patches:
499             self.obsolete_patches_on_bug(bug_id, tool.bugs)
500
501         diff = tool.scm().create_patch()
502         diff_file = StringIO.StringIO(diff) # add_patch_to_bug expects a file-like object
503
504         description = options.description or "Patch"
505         tool.bugs.add_patch_to_bug(bug_id, diff_file, description, mark_for_review=options.review, mark_for_commit_queue=options.request_commit)
506
507
508 class PostCommits(Command):
509     def __init__(self):
510         options = [
511             make_option("-b", "--bug-id", action="store", type="string", dest="bug_id", help="Specify bug id if no URL is provided in the commit log."),
512             make_option("--add-log-as-comment", action="store_true", dest="add_log_as_comment", default=False, help="Add commit log message as a comment when uploading the patch."),
513             make_option("-m", "--description", action="store", type="string", dest="description", help="Description string for the attachment (default: description from commit message)"),
514         ]
515         options += PostDiff.posting_options()
516         Command.__init__(self, "Attaches a range of local commits to bugs as patch files.", "COMMITISH", options=options, requires_local_commits=True)
517
518     def _comment_text_for_commit(self, options, commit_message, tool, commit_id):
519         comment_text = None
520         if (options.add_log_as_comment):
521             comment_text = commit_message.body(lstrip=True)
522             comment_text += "---\n"
523             comment_text += tool.scm().files_changed_summary_for_commit(commit_id)
524         return comment_text
525
526     def _diff_file_for_commit(self, tool, commit_id):
527         diff = tool.scm().create_patch_from_local_commit(commit_id)
528         return StringIO.StringIO(diff) # add_patch_to_bug expects a file-like object
529
530     def execute(self, options, args, tool):
531         if not args:
532             error("%s argument is required" % self.argument_names)
533
534         commit_ids = tool.scm().commit_ids_from_commitish_arguments(args)
535         if len(commit_ids) > 10: # We could lower this limit, 10 is too many for one bug as-is.
536             error("bugzilla-tool does not support attaching %s at once.  Are you sure you passed the right commit range?" % (pluralize("patch", len(commit_ids))))
537
538         have_obsoleted_patches = set()
539         for commit_id in commit_ids:
540             commit_message = tool.scm().commit_message_for_local_commit(commit_id)
541
542             # Prefer --bug-id=, then a bug url in the commit message, then a bug url in the entire commit diff (i.e. ChangeLogs).
543             bug_id = options.bug_id or parse_bug_id(commit_message.message()) or parse_bug_id(tool.scm().create_patch_from_local_commit(commit_id))
544             if not bug_id:
545                 log("Skipping %s: No bug id found in commit or specified with --bug-id." % commit_id)
546                 continue
547
548             if options.obsolete_patches and bug_id not in have_obsoleted_patches:
549                 PostDiff.obsolete_patches_on_bug(bug_id, tool.bugs)
550                 have_obsoleted_patches.add(bug_id)
551
552             diff_file = self._diff_file_for_commit(tool, commit_id)
553             description = options.description or commit_message.description(lstrip=True, strip_url=True)
554             comment_text = self._comment_text_for_commit(options, commit_message, tool, commit_id)
555             tool.bugs.add_patch_to_bug(bug_id, diff_file, description, comment_text, mark_for_review=options.review, mark_for_commit_queue=options.request_commit)
556
557
558 class Rollout(Command):
559     def __init__(self):
560         options = WebKitLandingScripts.land_options()
561         options += WebKitLandingScripts.cleaning_options()
562         options.append(make_option("--complete-rollout", action="store_true", dest="complete_rollout", help="Experimental support for complete unsupervised rollouts, including re-opening the bug.  Not recommended."))
563         Command.__init__(self, "Reverts the given revision and commits the revert and re-opens the original bug.", "REVISION [BUGID]", options=options)
564
565     @staticmethod
566     def _create_changelogs_for_revert(scm, revision):
567         # First, discard the ChangeLog changes from the rollout.
568         changelog_paths = scm.modified_changelogs()
569         scm.revert_files(changelog_paths)
570
571         # Second, make new ChangeLog entries for this rollout.
572         # This could move to prepare-ChangeLog by adding a --revert= option.
573         WebKitLandingScripts.run_webkit_script("prepare-ChangeLog")
574         for changelog_path in changelog_paths:
575             ChangeLog(changelog_path).update_for_revert(revision)
576
577     @staticmethod
578     def _parse_bug_id_from_revision_diff(tool, revision):
579         original_diff = tool.scm().diff_for_revision(revision)
580         return parse_bug_id(original_diff)
581
582     @staticmethod
583     def _reopen_bug_after_rollout(tool, bug_id, comment_text):
584         if bug_id:
585             tool.bugs.reopen_bug(bug_id, comment_text)
586         else:
587             log(comment_text)
588             log("No bugs were updated or re-opened to reflect this rollout.")
589
590     def execute(self, options, args, tool):
591         if not args:
592             error("REVISION is required, see --help.")
593         revision = args[0]
594         bug_id = self._parse_bug_id_from_revision_diff(tool, revision)
595         if options.complete_rollout:
596             if bug_id:
597                 log("Will re-open bug %s after rollout." % bug_id)
598             else:
599                 log("Failed to parse bug number from diff.  No bugs will be updated/reopened after the rollout.")
600
601         WebKitLandingScripts.prepare_clean_working_directory(tool.scm(), options)
602         tool.scm().update_webkit()
603         tool.scm().apply_reverse_diff(revision)
604         self._create_changelogs_for_revert(tool.scm(), revision)
605
606         # FIXME: Fully automated rollout is not 100% idiot-proof yet, so for now just log with instructions on how to complete the rollout.
607         # Once we trust rollout we will remove this option.
608         if not options.complete_rollout:
609             log("\nNOTE: Rollout support is experimental.\nPlease verify the rollout diff and use \"bugzilla-tool land-diff %s\" to commit the rollout." % bug_id)
610         else:
611             comment_text = WebKitLandingScripts.build_and_commit(tool.scm(), options)
612             self._reopen_bug_after_rollout(tool, bug_id, comment_text)
613
614
615 class CreateBug(Command):
616     def __init__(self):
617         options = [
618             make_option("--cc", action="store", type="string", dest="cc", help="Comma-separated list of email addresses to carbon-copy."),
619             make_option("--component", action="store", type="string", dest="component", help="Component for the new bug."),
620             make_option("--no-prompt", action="store_false", dest="prompt", default=True, help="Do not prompt for bug title and comment; use commit log instead."),
621             make_option("--no-review", action="store_false", dest="review", default=True, help="Do not mark the patch for review."),
622             make_option("--request-commit", action="store_true", dest="request_commit", default=False, help="Mark the patch as needing auto-commit after review."),
623         ]
624         Command.__init__(self, "Create a bug from local changes or local commits.", "[COMMITISH]", options=options)
625
626     def create_bug_from_commit(self, options, args, tool):
627         commit_ids = tool.scm().commit_ids_from_commitish_arguments(args)
628         if len(commit_ids) > 3:
629             error("Are you sure you want to create one bug with %s patches?" % len(commit_ids))
630
631         commit_id = commit_ids[0]
632
633         bug_title = ""
634         comment_text = ""
635         if options.prompt:
636             (bug_title, comment_text) = self.prompt_for_bug_title_and_comment()
637         else:
638             commit_message = tool.scm().commit_message_for_local_commit(commit_id)
639             bug_title = commit_message.description(lstrip=True, strip_url=True)
640             comment_text = commit_message.body(lstrip=True)
641             comment_text += "---\n"
642             comment_text += tool.scm().files_changed_summary_for_commit(commit_id)
643
644         diff = tool.scm().create_patch_from_local_commit(commit_id)
645         diff_file = StringIO.StringIO(diff) # create_bug_with_patch expects a file-like object
646         bug_id = tool.bugs.create_bug_with_patch(bug_title, comment_text, options.component, diff_file, "Patch", cc=options.cc, mark_for_review=options.review, mark_for_commit_queue=options.request_commit)
647
648         if bug_id and len(commit_ids) > 1:
649             options.bug_id = bug_id
650             options.obsolete_patches = False
651             # FIXME: We should pass through --no-comment switch as well.
652             PostCommits.execute(self, options, commit_ids[1:], tool)
653
654     def create_bug_from_patch(self, options, args, tool):
655         bug_title = ""
656         comment_text = ""
657         if options.prompt:
658             (bug_title, comment_text) = self.prompt_for_bug_title_and_comment()
659         else:
660             commit_message = commit_message_for_this_commit(tool.scm())
661             bug_title = commit_message.description(lstrip=True, strip_url=True)
662             comment_text = commit_message.body(lstrip=True)
663
664         diff = tool.scm().create_patch()
665         diff_file = StringIO.StringIO(diff) # create_bug_with_patch expects a file-like object
666         bug_id = tool.bugs.create_bug_with_patch(bug_title, comment_text, options.component, diff_file, "Patch", cc=options.cc, mark_for_review=options.review, mark_for_commit_queue=options.request_commit)
667
668     def prompt_for_bug_title_and_comment(self):
669         bug_title = raw_input("Bug title: ")
670         print "Bug comment (hit ^D on blank line to end):"
671         lines = sys.stdin.readlines()
672         try:
673             sys.stdin.seek(0, os.SEEK_END)
674         except IOError:
675             # Cygwin raises an Illegal Seek (errno 29) exception when the above
676             # seek() call is made. Ignoring it seems to cause no harm.
677             # FIXME: Figure out a way to get avoid the exception in the first
678             # place.
679             pass
680         comment_text = "".join(lines)
681         return (bug_title, comment_text)
682
683     def execute(self, options, args, tool):
684         if len(args):
685             if (not tool.scm().supports_local_commits()):
686                 error("Extra arguments not supported; patch is taken from working directory.")
687             self.create_bug_from_commit(options, args, tool)
688         else:
689             self.create_bug_from_patch(options, args, tool)
690
691
692 class TreeStatus(Command):
693     def __init__(self):
694         Command.__init__(self, "Print out the status of the webkit builders.")
695
696     def execute(self, options, args, tool):
697         for builder in tool.buildbot.builder_statuses():
698             status_string = "ok" if builder["is_green"] else "FAIL"
699             print "%s : %s" % (status_string.ljust(4), builder["name"])
700
701
702 class AbstractQueue(Command, WorkQueueDelegate):
703     def __init__(self, name):
704         self._name = name
705         options = [
706             make_option("--no-confirm", action="store_false", dest="confirm", default=True, help="Do not ask the user for confirmation before running the queue.  Dangerous!"),
707             make_option("--status-host", action="store", type="string", dest="status_host", default=StatusBot.default_host, help="Hostname (e.g. localhost or commit.webkit.org) where status updates should be posted."),
708         ]
709         Command.__init__(self, "Run the %s." % self._name, options=options)
710
711     def queue_log_path(self):
712         return "%s.log" % self._name
713
714     def work_logs_directory(self):
715         return "%s-logs" % self._name
716
717     def status_host(self):
718         return self.options.status_host
719
720     def begin_work_queue(self):
721         log("CAUTION: %s will discard all local changes in %s" % (self._name, self.tool.scm().checkout_root))
722         if self.options.confirm:
723             response = raw_input("Are you sure?  Type \"yes\" to continue: ")
724             if (response != "yes"):
725                 error("User declined.")
726         log("Running WebKit %s. %s" % (self._name, datetime.now().strftime(WorkQueue.log_date_format)))
727
728     def should_continue_work_queue(self):
729         return True
730
731     def next_work_item(self):
732         raise NotImplementedError, "subclasses must implement"
733
734     def should_proceed_with_work_item(self, work_item):
735         raise NotImplementedError, "subclasses must implement"
736
737     def process_work_item(self, work_item):
738         raise NotImplementedError, "subclasses must implement"
739
740     def handle_unexpected_error(self, work_item, message):
741         raise NotImplementedError, "subclasses must implement"
742
743     @staticmethod
744     def run_bugzilla_tool(args):
745         bugzilla_tool_path = __file__ # re-execute this script
746         bugzilla_tool_args = [bugzilla_tool_path] + args
747         WebKitLandingScripts.run_and_throw_if_fail(bugzilla_tool_args)
748
749     def log_progress(self, patch_ids):
750         log("%s in %s [%s]" % (pluralize("patch", len(patch_ids)), self._name, ", ".join(patch_ids)))
751
752     def execute(self, options, args, tool):
753         self.options = options
754         self.tool = tool
755         work_queue = WorkQueue(self)
756         work_queue.run()
757
758
759 class CommitQueue(AbstractQueue):
760     def __init__(self):
761         AbstractQueue.__init__(self, "commit-queue")
762
763     def begin_work_queue(self):
764         AbstractQueue.begin_work_queue(self)
765
766     def next_work_item(self):
767         patches = self.tool.bugs.fetch_patches_from_commit_queue(reject_invalid_patches=True)
768         if not patches:
769             return None
770         # Only bother logging if we have patches in the queue.
771         self.log_progress([patch['id'] for patch in patches])
772         return patches[0]
773
774     def should_proceed_with_work_item(self, patch):
775         red_builders_names = self.tool.buildbot.red_core_builders_names()
776         if red_builders_names:
777             red_builders_names = map(lambda name: "\"%s\"" % name, red_builders_names) # Add quotes around the names.
778             return (False, "Builders [%s] are red. See http://build.webkit.org." % ", ".join(red_builders_names), None)
779         return (True, "Landing patch %s from bug %s." % (patch["id"], patch["bug_id"]), patch["bug_id"])
780
781     def process_work_item(self, patch):
782         self.run_bugzilla_tool(["land-attachment", "--force-clean", "--non-interactive", "--quiet", patch["id"]])
783
784     def handle_unexpected_error(self, patch, message):
785         self.tool.bugs.reject_patch_from_commit_queue(patch["id"], message)
786
787
788 class StyleQueue(AbstractQueue):
789     def __init__(self):
790         AbstractQueue.__init__(self, "style-queue")
791
792     def status_host(self):
793         return None # FIXME: A hack until we come up with a more generic status page.
794
795     def begin_work_queue(self):
796         AbstractQueue.begin_work_queue(self)
797         self._patches = PatchCollection(self.tool.bugs)
798         self._patches.add_patches(self.tool.bugs.fetch_patches_from_review_queue(limit=10))
799
800     def next_work_item(self):
801         self.log_progress(self._patches.patch_ids())
802         return self._patches.next()
803
804     def should_proceed_with_work_item(self, patch):
805         return (True, "Checking style for patch %s on bug %s." % (patch["id"], patch["bug_id"]), patch["bug_id"])
806
807     def process_work_item(self, patch):
808         self.run_bugzilla_tool(["check-style", "--force-clean", patch["id"]])
809
810     def handle_unexpected_error(self, patch, message):
811         log(message)
812
813
814 class BugzillaTool(MultiCommandTool):
815     def __init__(self):
816         # HACK: Set self.cached_scm before calling MultiCommandTool.__init__ because
817         # MultiCommandTool._commands_usage() will call self.should_show_command_help which uses scm().
818         # This hack can be removed by overriding usage() printing in HelpPrintingOptionParser
819         # so that we don't need to create 'epilog' before constructing HelpPrintingOptionParser.
820         self.cached_scm = None
821         
822         # FIXME: Commands should know their own name and register themselves with the BugzillaTool instead of having a manual list.
823         MultiCommandTool.__init__(self, commands=[
824             { "name" : "bugs-to-commit", "object" : BugsToCommit() },
825             { "name" : "patches-to-commit", "object" : PatchesToCommit() },
826             { "name" : "reviewed-patches", "object" : ReviewedPatches() },
827             { "name" : "create-bug", "object" : CreateBug() },
828             { "name" : "apply-attachment", "object" : ApplyAttachment() },
829             { "name" : "apply-patches", "object" : ApplyPatches() },
830             { "name" : "land-diff", "object" : LandDiff() },
831             { "name" : "land-attachment", "object" : LandAttachment() },
832             { "name" : "land-patches", "object" : LandPatches() },
833             { "name" : "check-style", "object" : CheckStyle() },
834             { "name" : "commit-message", "object" : CommitMessageForCurrentDiff() },
835             { "name" : "obsolete-attachments", "object" : ObsoleteAttachments() },
836             { "name" : "post-diff", "object" : PostDiff() },
837             { "name" : "post-commits", "object" : PostCommits() },
838             { "name" : "tree-status", "object" : TreeStatus() },
839             { "name" : "commit-queue", "object" : CommitQueue() },
840             { "name" : "style-queue", "object" : StyleQueue() },
841             { "name" : "rollout", "object" : Rollout() },
842         ])
843         self.global_option_parser.add_option("--dry-run", action="callback", help="do not touch remote servers", callback=self.dry_run_callback)
844
845         self.bugs = Bugzilla()
846         self.buildbot = BuildBot()
847
848     def dry_run_callback(self, option, opt, value, parser):
849         self.scm().dryrun = True
850         self.bugs.dryrun = True
851
852     def scm(self):
853         # Lazily initialize SCM to not error-out before command line parsing (or when running non-scm commands).
854         original_cwd = os.path.abspath(".")
855         if not self.cached_scm:
856             self.cached_scm = detect_scm_system(original_cwd)
857
858         if not self.cached_scm:
859             script_directory = os.path.abspath(sys.path[0])
860             webkit_directory = os.path.abspath(os.path.join(script_directory, "../.."))
861             self.cached_scm = detect_scm_system(webkit_directory)
862             if self.cached_scm:
863                 log("The current directory (%s) is not a WebKit checkout, using %s" % (original_cwd, webkit_directory))
864             else:
865                 error("FATAL: Failed to determine the SCM system for either %s or %s" % (original_cwd, webkit_directory))
866
867         return self.cached_scm
868
869     def should_show_command_help(self, command):
870         if command["object"].requires_local_commits:
871             return self.scm().supports_local_commits()
872         return True
873
874     def should_execute_command(self, command):
875         if command["object"].requires_local_commits and not self.scm().supports_local_commits():
876             failure_reason = "%s requires local commits using %s in %s." % (command["name"], self.scm().display_name(), self.scm().checkout_root)
877             return (False, failure_reason)
878         return (True, None)
879
880
881 if __name__ == "__main__":
882     BugzillaTool().main()