269b9b8193f261e3e44a81accf16877fc32455b7
[WebKit-https.git] / WebKitTools / Scripts / bugzilla-tool
1 #!/usr/bin/env python
2 # Copyright (c) 2009, Google Inc. All rights reserved.
3 # Copyright (c) 2009 Apple Inc. All rights reserved.
4 #
5 # Redistribution and use in source and binary forms, with or without
6 # modification, are permitted provided that the following conditions are
7 # met:
8
9 #     * Redistributions of source code must retain the above copyright
10 # notice, this list of conditions and the following disclaimer.
11 #     * Redistributions in binary form must reproduce the above
12 # copyright notice, this list of conditions and the following disclaimer
13 # in the documentation and/or other materials provided with the
14 # distribution.
15 #     * Neither the name of Google Inc. nor the names of its
16 # contributors may be used to endorse or promote products derived from
17 # this software without specific prior written permission.
18
19 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30 #
31 # A tool for automating dealing with bugzilla, posting patches, committing patches, etc.
32
33 import os
34 import re
35 import StringIO # for add_patch_to_bug file wrappers
36 import subprocess
37 import sys
38 import time
39
40 from datetime import datetime, timedelta
41 from optparse import OptionParser, IndentedHelpFormatter, SUPPRESS_USAGE, make_option
42
43 # Import WebKit-specific modules.
44 from modules.bugzilla import Bugzilla, parse_bug_id
45 from modules.changelogs import ChangeLog
46 from modules.comments import bug_comment_from_commit_text
47 from modules.logging import error, log, tee
48 from modules.patchcollection import PatchCollection
49 from modules.scm import CommitMessage, detect_scm_system, ScriptError, CheckoutNeedsUpdate
50 from modules.buildbot import BuildBot
51 from modules.statusbot import StatusBot
52 from modules.workqueue import WorkQueue, WorkQueueDelegate
53
54 def plural(noun):
55     # This is a dumb plural() implementation which was just enough for our uses.
56     if re.search('h$', noun):
57         return noun + 'es'
58     else:
59         return noun + 's'
60
61 def pluralize(noun, count):
62     if count != 1:
63         noun = plural(noun)
64     return "%d %s" % (count, noun)
65
66 def commit_message_for_this_commit(scm):
67     changelog_paths = scm.modified_changelogs()
68     if not len(changelog_paths):
69         raise ScriptError(message="Found no modified ChangeLogs, cannot create a commit message.\n"
70                           "All changes require a ChangeLog.  See:\n"
71                           "http://webkit.org/coding/contributing.html")
72
73     changelog_messages = []
74     for changelog_path in changelog_paths:
75         log("Parsing ChangeLog: %s" % changelog_path)
76         changelog_entry = ChangeLog(changelog_path).latest_entry()
77         if not changelog_entry:
78             raise ScriptError(message="Failed to parse ChangeLog: " + os.path.abspath(changelog_path))
79         changelog_messages.append(changelog_entry)
80
81     # FIXME: We should sort and label the ChangeLog messages like commit-log-editor does.
82     return CommitMessage(''.join(changelog_messages).splitlines())
83
84
85 class Command:
86     def __init__(self, help_text, argument_names="", options=[], requires_local_commits=False):
87         self.help_text = help_text
88         self.argument_names = argument_names
89         self.options = options
90         self.option_parser = HelpPrintingOptionParser(usage=SUPPRESS_USAGE, add_help_option=False, option_list=self.options)
91         self.requires_local_commits = requires_local_commits
92     
93     def name_with_arguments(self, command_name):
94         usage_string = command_name
95         if len(self.options) > 0:
96             usage_string += " [options]"
97         if self.argument_names:
98             usage_string += " " + self.argument_names
99         return usage_string
100
101     def parse_args(self, args):
102         return self.option_parser.parse_args(args)
103
104     def execute(self, options, args, tool):
105         raise NotImplementedError, "subclasses must implement"
106
107
108 class BugsInCommitQueue(Command):
109     def __init__(self):
110         Command.__init__(self, 'Bugs in the commit queue')
111
112     def execute(self, options, args, tool):
113         bug_ids = tool.bugs.fetch_bug_ids_from_commit_queue()
114         for bug_id in bug_ids:
115             print "%s" % bug_id
116
117
118 class PatchesInCommitQueue(Command):
119     def __init__(self):
120         Command.__init__(self, 'Patches in the commit queue')
121
122     def execute(self, options, args, tool):
123         patches = tool.bugs.fetch_patches_from_commit_queue()
124         log("Patches in commit queue:")
125         for patch in patches:
126             print "%s" % patch['url']
127
128
129 class ReviewedPatchesOnBug(Command):
130     def __init__(self):
131         Command.__init__(self, 'r+\'d patches on a bug', 'BUGID')
132
133     def execute(self, options, args, tool):
134         bug_id = args[0]
135         patches_to_land = tool.bugs.fetch_reviewed_patches_from_bug(bug_id)
136         for patch in patches_to_land:
137             print "%s" % patch['url']
138
139
140 class CheckStyle(Command):
141     def __init__(self):
142         options = WebKitLandingScripts.cleaning_options()
143         Command.__init__(self, 'Runs check-webkit-style on the specified attachment', 'ATTACHMENT_ID', options=options)
144
145     @classmethod
146     def check_style(cls, patch, options, tool):
147         tool.scm().update_webkit()
148         log("Checking style for patch %s from bug %s." % (patch['id'], patch['bug_id']))
149         try:
150             # FIXME: check-webkit-style shouldn't really have to apply the patch to check the style.
151             tool.scm().apply_patch(patch)
152             WebKitLandingScripts.run_webkit_script("check-webkit-style")
153         except ScriptError, e:
154             log("Patch %s from bug %s failed to apply and check style." % (patch['id'], patch['bug_id']))
155             log(e.output)
156
157         # This is safe because in order to get here the working directory had to be
158         # clean at the beginning.  Clean it out again before we exit.
159         tool.scm().ensure_clean_working_directory(force_clean=True)
160
161     def execute(self, options, args, tool):
162         attachment_id = args[0]
163         attachment = tool.bugs.fetch_attachment(attachment_id)
164
165         WebKitLandingScripts.prepare_clean_working_directory(tool.scm(), options)
166         self.check_style(attachment, options, tool)
167
168
169 class ApplyAttachment(Command):
170     def __init__(self):
171         options = WebKitApplyingScripts.apply_options() + WebKitLandingScripts.cleaning_options()
172         Command.__init__(self, 'Applies an attachment to the local working directory.', 'ATTACHMENT_ID', options=options)
173
174     def execute(self, options, args, tool):
175         WebKitApplyingScripts.setup_for_patch_apply(tool.scm(), options)
176         attachment_id = args[0]
177         attachment = tool.bugs.fetch_attachment(attachment_id)
178         WebKitApplyingScripts.apply_patches_with_options(tool.scm(), [attachment], options)
179
180
181 class ApplyPatchesFromBug(Command):
182     def __init__(self):
183         options = WebKitApplyingScripts.apply_options() + WebKitLandingScripts.cleaning_options()
184         Command.__init__(self, 'Applies all patches on a bug to the local working directory.', 'BUGID', options=options)
185
186     def execute(self, options, args, tool):
187         WebKitApplyingScripts.setup_for_patch_apply(tool.scm(), options)
188         bug_id = args[0]
189         patches = tool.bugs.fetch_reviewed_patches_from_bug(bug_id)
190         WebKitApplyingScripts.apply_patches_with_options(tool.scm(), patches, options)
191
192
193 class WebKitApplyingScripts:
194     @staticmethod
195     def apply_options():
196         return [
197             make_option("--no-update", action="store_false", dest="update", default=True, help="Don't update the working directory before applying patches"),
198             make_option("--local-commit", action="store_true", dest="local_commit", default=False, help="Make a local commit for each applied patch"),
199         ]
200
201     @staticmethod
202     def setup_for_patch_apply(scm, options):
203         WebKitLandingScripts.prepare_clean_working_directory(scm, options, allow_local_commits=True)
204         if options.update:
205             scm.update_webkit()
206
207     @staticmethod
208     def apply_patches_with_options(scm, patches, options):
209         if options.local_commit and not scm.supports_local_commits():
210             error("--local-commit passed, but %s does not support local commits" % scm.display_name())
211
212         for patch in patches:
213             log("Applying attachment %s from bug %s" % (patch['id'], patch['bug_id']))
214             scm.apply_patch(patch)
215             if options.local_commit:
216                 commit_message = commit_message_for_this_commit(scm)
217                 scm.commit_locally_with_message(commit_message.message() or patch['name'])
218
219
220 class WebKitLandingScripts:
221     @staticmethod
222     def cleaning_options():
223         return [
224             make_option("--force-clean", action="store_true", dest="force_clean", default=False, help="Clean working directory before applying patches (removes local changes and commits)"),
225             make_option("--no-clean", action="store_false", dest="clean", default=True, help="Don't check if the working directory is clean before applying patches"),
226         ]
227
228     @staticmethod
229     def land_options():
230         return [
231             make_option("--ignore-builders", action="store_false", dest="check_builders", default=True, help="Don't check to see if the build.webkit.org builders are green before landing."),
232             make_option("--no-close", action="store_false", dest="close_bug", default=True, help="Leave bug open after landing."),
233             make_option("--no-build", action="store_false", dest="build", default=True, help="Commit without building first, implies --no-test."),
234             make_option("--no-test", action="store_false", dest="test", default=True, help="Commit without running run-webkit-tests."),
235             make_option("--quiet", action="store_true", dest="quiet", default=False, help="Produce less console output."),
236             make_option("--non-interactive", action="store_true", dest="non_interactive", default=False, help="Never prompt the user, fail as fast as possible."),
237         ]
238
239     @staticmethod
240     def run_command_with_teed_output(args, teed_output):
241         child_process = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
242
243         # Use our own custom wait loop because Popen ignores a tee'd stderr/stdout.
244         # FIXME: This could be improved not to flatten output to stdout.
245         while True:
246             output_line = child_process.stdout.readline()
247             if output_line == '' and child_process.poll() != None:
248                 return child_process.poll()
249             teed_output.write(output_line)
250
251     @staticmethod
252     def run_and_throw_if_fail(args, quiet=False):
253         # Cache the child's output locally so it can be used for error reports.
254         child_out_file = StringIO.StringIO()
255         if quiet:
256             dev_null = open(os.devnull, "w")
257         child_stdout = tee(child_out_file, dev_null if quiet else sys.stdout)
258         exit_code = WebKitLandingScripts.run_command_with_teed_output(args, child_stdout)
259         if quiet:
260             dev_null.close()
261
262         child_output = child_out_file.getvalue()
263         child_out_file.close()
264
265         if exit_code:
266             raise ScriptError(script_args=args, exit_code=exit_code, output=child_output)
267
268     # We might need to pass scm into this function for scm.checkout_root
269     @staticmethod
270     def webkit_script_path(script_name):
271         return os.path.join("WebKitTools", "Scripts", script_name)
272
273     @classmethod
274     def run_webkit_script(cls, script_name, quiet=False):
275         log("Running %s" % script_name)
276         cls.run_and_throw_if_fail(cls.webkit_script_path(script_name), quiet)
277
278     @classmethod
279     def build_webkit(cls, quiet=False):
280         cls.run_webkit_script("build-webkit", quiet)
281
282     @staticmethod
283     def ensure_builders_are_green(buildbot, options):
284         if not options.check_builders or buildbot.core_builders_are_green():
285             return
286         error("Builders at %s are red, please do not commit.  Pass --ignore-builders to bypass this check." % (buildbot.buildbot_host))
287
288     @classmethod
289     def run_webkit_tests(cls, launch_safari, fail_fast=False, quiet=False):
290         args = [cls.webkit_script_path("run-webkit-tests")]
291         if not launch_safari:
292             args.append("--no-launch-safari")
293         if quiet:
294             args.append("--quiet")
295         if fail_fast:
296             args.append("--exit-after-n-failures=1")
297         cls.run_and_throw_if_fail(args)
298
299     @staticmethod
300     def prepare_clean_working_directory(scm, options, allow_local_commits=False):
301         os.chdir(scm.checkout_root)
302         if not allow_local_commits:
303             scm.ensure_no_local_commits(options.force_clean)
304         if options.clean:
305             scm.ensure_clean_working_directory(force_clean=options.force_clean)
306
307     @classmethod
308     def build_and_commit(cls, scm, options):
309         if options.build:
310             cls.build_webkit(quiet=options.quiet)
311             if options.test:
312                 # When running the commit-queue we don't want to launch Safari and we want to exit after the first failure.
313                 cls.run_webkit_tests(launch_safari=not options.non_interactive, fail_fast=options.non_interactive, quiet=options.quiet)
314         commit_message = commit_message_for_this_commit(scm)
315         commit_log = scm.commit_with_message(commit_message.message())
316         return bug_comment_from_commit_text(scm, commit_log)
317
318     @classmethod
319     def _close_bug_if_no_active_patches(cls, bugs, bug_id):
320         # Check to make sure there are no r? or r+ patches on the bug before closing.
321         # Assume that r- patches are just previous patches someone forgot to obsolete.
322         patches = bugs.fetch_patches_from_bug(bug_id)
323         for patch in patches:
324             review_flag = patch.get('review')
325             if review_flag == '?' or review_flag == '+':
326                 log("Not closing bug %s as attachment %s has review=%s.  Assuming there are more patches to land from this bug." % (patch['bug_id'], patch['id'], review_flag))
327                 return
328         bugs.close_bug_as_fixed(bug_id, "All reviewed patches have been landed.  Closing bug.")
329
330     @classmethod
331     def _land_patch(cls, patch, options, tool):
332         tool.scm().update_webkit() # Update before every patch in case the tree has changed
333         log("Applying patch %s from bug %s." % (patch['id'], patch['bug_id']))
334         tool.scm().apply_patch(patch, force=options.non_interactive)
335
336         # Make sure the tree is still green after updating, before building this patch.
337         # The first patch ends up checking tree status twice, but that's OK.
338         WebKitLandingScripts.ensure_builders_are_green(tool.buildbot, options)
339         comment_text = WebKitLandingScripts.build_and_commit(tool.scm(), options)
340         tool.bugs.clear_attachment_flags(patch['id'], comment_text)
341
342     @classmethod
343     def land_patch_and_handle_errors(cls, patch, options, tool):
344         try:
345             cls._land_patch(patch, options, tool)
346             if options.close_bug:
347                 cls._close_bug_if_no_active_patches(tool.bugs, patch['bug_id'])
348         except CheckoutNeedsUpdate, e:
349             log("Commit failed because the checkout is out of date.  Please update and try again.")
350             log("You can pass --no-build to skip building/testing after update if you believe the new commits did not affect the results.")
351             WorkQueue.exit_after_handled_error(e)
352         except ScriptError, e:
353             # Mark the patch as commit-queue- and comment in the bug.
354             tool.bugs.reject_patch_from_commit_queue(patch['id'], e.message_with_output())
355             WorkQueue.exit_after_handled_error(e)
356
357
358 class LandAndUpdateBug(Command):
359     def __init__(self):
360         options = [
361             make_option("-r", "--reviewer", action="store", type="string", dest="reviewer", help="Update ChangeLogs to say Reviewed by REVIEWER."),
362         ]
363         options += WebKitLandingScripts.land_options()
364         Command.__init__(self, 'Lands the current working directory diff and updates the bug if provided.', '[BUGID]', options=options)
365
366     def guess_reviewer_from_bug(self, bugs, bug_id):
367         patches = bugs.fetch_reviewed_patches_from_bug(bug_id)
368         if len(patches) != 1:
369             log("%s on bug %s, cannot infer reviewer." % (pluralize("reviewed patch", len(patches)), bug_id))
370             return None
371         patch = patches[0]
372         reviewer = patch['reviewer']
373         log('Guessing "%s" as reviewer from attachment %s on bug %s.' % (reviewer, patch['id'], bug_id))
374         return reviewer
375
376     def update_changelogs_with_reviewer(self, reviewer, bug_id, tool):
377         if not reviewer:
378             if not bug_id:
379                 log("No bug id provided and --reviewer= not provided.  Not updating ChangeLogs with reviewer.")
380                 return
381             reviewer = self.guess_reviewer_from_bug(tool.bugs, bug_id)
382
383         if not reviewer:
384             log("Failed to guess reviewer from bug %s and --reviewer= not provided.  Not updating ChangeLogs with reviewer." % bug_id)
385             return
386
387         for changelog_path in tool.scm().modified_changelogs():
388             ChangeLog(changelog_path).set_reviewer(reviewer)
389
390     def execute(self, options, args, tool):
391         bug_id = (args and args[0]) or parse_bug_id(tool.scm().create_patch())
392
393         WebKitLandingScripts.ensure_builders_are_green(tool.buildbot, options)
394
395         os.chdir(tool.scm().checkout_root)
396         self.update_changelogs_with_reviewer(options.reviewer, bug_id, tool)
397
398         comment_text = WebKitLandingScripts.build_and_commit(tool.scm(), options)
399         if bug_id:
400             log("Updating bug %s" % bug_id)
401             if options.close_bug:
402                 tool.bugs.close_bug_as_fixed(bug_id, comment_text)
403             else:
404                 # FIXME: We should a smart way to figure out if the patch is attached
405                 # to the bug, and if so obsolete it.
406                 tool.bugs.post_comment_to_bug(bug_id, comment_text)
407         else:
408             log(comment_text)
409             log("No bug id provided.")
410
411
412 class AbstractPatchLandingCommand(Command):
413     def __init__(self, description, args_description):
414         options = WebKitLandingScripts.cleaning_options() + WebKitLandingScripts.land_options()
415         Command.__init__(self, description, args_description, options=options)
416
417     @staticmethod
418     def _fetch_list_of_patches_to_land(options, args, tool):
419         raise NotImplementedError, "subclasses must implement"
420
421     @staticmethod
422     def _collect_patches_by_bug(patches):
423         bugs_to_patches = {}
424         for patch in patches:
425             bug_id = patch['bug_id']
426             bugs_to_patches[bug_id] = bugs_to_patches.get(bug_id, []).append(patch)
427         return bugs_to_patches
428
429     def execute(self, options, args, tool):
430         if not args:
431             error("%s required" % self.argument_names)
432
433         # Check the tree status first so we can fail early.
434         WebKitLandingScripts.ensure_builders_are_green(tool.buildbot, options)
435         WebKitLandingScripts.prepare_clean_working_directory(tool.scm(), options)
436
437         patches = self._fetch_list_of_patches_to_land(options, args, tool)
438
439         # It's nice to print out total statistics.
440         bugs_to_patches = self._collect_patches_by_bug(patches)
441         log("Landing %s from %s." % (pluralize("patch", len(patches)), pluralize("bug", len(bugs_to_patches))))
442
443         for patch in patches:
444             WebKitLandingScripts.land_patch_and_handle_errors(patch, options, tool)
445
446
447 class LandAttachment(AbstractPatchLandingCommand):
448     def __init__(self):
449         AbstractPatchLandingCommand.__init__(self, 'Lands a patches from bugzilla, optionally building and testing them first', 'ATTACHMENT_ID [ATTACHMENT_IDS]')
450
451     @staticmethod
452     def _fetch_list_of_patches_to_land(options, args, tool):
453         return map(lambda patch_id: tool.bugs.fetch_attachment(patch_id), args)
454
455
456 class LandPatchesFromBugs(AbstractPatchLandingCommand):
457     def __init__(self):
458         AbstractPatchLandingCommand.__init__(self, 'Lands all patches on the given bugs, optionally building and testing them first', 'BUGID [BUGIDS]')
459
460     @staticmethod
461     def _fetch_list_of_patches_to_land(options, args, tool):
462         all_patches = []
463         for bug_id in args:
464             patches = tool.bugs.fetch_reviewed_patches_from_bug(bug_id)
465             log("%s found on bug %s." % (pluralize("reviewed patch", len(patches)), bug_id))
466             all_patches += patches
467         return all_patches
468
469
470 class CommitMessageForCurrentDiff(Command):
471     def __init__(self):
472         Command.__init__(self, 'Prints a commit message suitable for the uncommitted changes.')
473
474     def execute(self, options, args, tool):
475         os.chdir(tool.scm().checkout_root)
476         print "%s" % commit_message_for_this_commit(tool.scm()).message()
477
478
479 class ObsoleteAttachmentsOnBug(Command):
480     def __init__(self):
481         Command.__init__(self, 'Marks all attachments on a bug as obsolete.', 'BUGID')
482
483     def execute(self, options, args, tool):
484         bug_id = args[0]
485         attachments = tool.bugs.fetch_attachments_from_bug(bug_id)
486         for attachment in attachments:
487             if not attachment['is_obsolete']:
488                 tool.bugs.obsolete_attachment(attachment['id'])
489
490
491 class PostDiffAsPatchToBug(Command):
492     def __init__(self):
493         options = [
494             make_option("-m", "--description", action="store", type="string", dest="description", help="Description string for the attachment (default: 'patch')"),
495         ]
496         options += self.posting_options()
497         Command.__init__(self, 'Attaches the current working directory diff to a bug as a patch file.', '[BUGID]', options=options)
498
499     @staticmethod
500     def posting_options():
501         return [
502             make_option("--no-obsolete", action="store_false", dest="obsolete_patches", default=True, help="Do not obsolete old patches before posting this one."),
503             make_option("--no-review", action="store_false", dest="review", default=True, help="Do not mark the patch for review."),
504             make_option("--request-commit", action="store_true", dest="request_commit", default=False, help="Mark the patch as needing auto-commit after review."),
505         ]
506
507     @staticmethod
508     def obsolete_patches_on_bug(bug_id, bugs):
509         patches = bugs.fetch_patches_from_bug(bug_id)
510         if len(patches):
511             log("Obsoleting %s on bug %s" % (pluralize('old patch', len(patches)), bug_id))
512             for patch in patches:
513                 bugs.obsolete_attachment(patch['id'])
514
515     def execute(self, options, args, tool):
516         # Perfer a bug id passed as an argument over a bug url in the diff (i.e. ChangeLogs).
517         bug_id = (args and args[0]) or parse_bug_id(tool.scm().create_patch())
518         if not bug_id:
519             error("No bug id passed and no bug url found in diff, can't post.")
520
521         if options.obsolete_patches:
522             self.obsolete_patches_on_bug(bug_id, tool.bugs)
523
524         diff = tool.scm().create_patch()
525         diff_file = StringIO.StringIO(diff) # add_patch_to_bug expects a file-like object
526
527         description = options.description or "Patch"
528         tool.bugs.add_patch_to_bug(bug_id, diff_file, description, mark_for_review=options.review, mark_for_commit_queue=options.request_commit)
529
530
531 class PostCommitsAsPatchesToBug(Command):
532     def __init__(self):
533         options = [
534             make_option("-b", "--bug-id", action="store", type="string", dest="bug_id", help="Specify bug id if no URL is provided in the commit log."),
535             make_option("--add-log-as-comment", action="store_true", dest="add_log_as_comment", default=False, help="Add commit log message as a comment when uploading the patch."),
536             make_option("-m", "--description", action="store", type="string", dest="description", help="Description string for the attachment (default: description from commit message)"),
537         ]
538         options += PostDiffAsPatchToBug.posting_options()
539         Command.__init__(self, 'Attaches a range of local commits to bugs as patch files.', 'COMMITISH', options=options, requires_local_commits=True)
540
541     def _comment_text_for_commit(self, options, commit_message, tool, commit_id):
542         comment_text = None
543         if (options.add_log_as_comment):
544             comment_text = commit_message.body(lstrip=True)
545             comment_text += "---\n"
546             comment_text += tool.scm().files_changed_summary_for_commit(commit_id)
547         return comment_text
548
549     def _diff_file_for_commit(self, tool, commit_id):
550         diff = tool.scm().create_patch_from_local_commit(commit_id)
551         return StringIO.StringIO(diff) # add_patch_to_bug expects a file-like object
552
553     def execute(self, options, args, tool):
554         if not args:
555             error("%s argument is required" % self.argument_names)
556
557         commit_ids = tool.scm().commit_ids_from_commitish_arguments(args)
558         if len(commit_ids) > 10: # We could lower this limit, 10 is too many for one bug as-is.
559             error("bugzilla-tool does not support attaching %s at once.  Are you sure you passed the right commit range?" % (pluralize('patch', len(commit_ids))))
560
561         have_obsoleted_patches = set()
562         for commit_id in commit_ids:
563             commit_message = tool.scm().commit_message_for_local_commit(commit_id)
564
565             # Prefer --bug-id=, then a bug url in the commit message, then a bug url in the entire commit diff (i.e. ChangeLogs).
566             bug_id = options.bug_id or parse_bug_id(commit_message.message()) or parse_bug_id(tool.scm().create_patch_from_local_commit(commit_id))
567             if not bug_id:
568                 log("Skipping %s: No bug id found in commit or specified with --bug-id." % commit_id)
569                 continue
570
571             if options.obsolete_patches and bug_id not in have_obsoleted_patches:
572                 PostDiffAsPatchToBug.obsolete_patches_on_bug(bug_id, tool.bugs)
573                 have_obsoleted_patches.add(bug_id)
574
575             diff_file = self._diff_file_for_commit(tool, commit_id)
576             description = options.description or commit_message.description(lstrip=True, strip_url=True)
577             comment_text = self._comment_text_for_commit(options, commit_message, tool, commit_id)
578             tool.bugs.add_patch_to_bug(bug_id, diff_file, description, comment_text, mark_for_review=options.review, mark_for_commit_queue=options.request_commit)
579
580
581 class RolloutCommit(Command):
582     def __init__(self):
583         options = WebKitLandingScripts.land_options()
584         options += WebKitLandingScripts.cleaning_options()
585         options.append(make_option("--complete-rollout", action="store_true", dest="complete_rollout", help="Experimental support for complete unsupervised rollouts, including re-opening the bug.  Not recommended."))
586         Command.__init__(self, 'Reverts the given revision and commits the revert and re-opens the original bug.', 'REVISION [BUGID]', options=options)
587
588     @staticmethod
589     def _create_changelogs_for_revert(scm, revision):
590         # First, discard the ChangeLog changes from the rollout.
591         changelog_paths = scm.modified_changelogs()
592         scm.revert_files(changelog_paths)
593
594         # Second, make new ChangeLog entries for this rollout.
595         # This could move to prepare-ChangeLog by adding a --revert= option.
596         WebKitLandingScripts.run_webkit_script("prepare-ChangeLog")
597         for changelog_path in changelog_paths:
598             ChangeLog(changelog_path).update_for_revert(revision)
599
600     @staticmethod
601     def _parse_bug_id_from_revision_diff(tool, revision):
602         original_diff = tool.scm().diff_for_revision(revision)
603         return parse_bug_id(original_diff)
604
605     @staticmethod
606     def _reopen_bug_after_rollout(tool, bug_id, comment_text):
607         if bug_id:
608             tool.bugs.reopen_bug(bug_id, comment_text)
609         else:
610             log(comment_text)
611             log("No bugs were updated or re-opened to reflect this rollout.")
612
613     def execute(self, options, args, tool):
614         if not args:
615             error("REVISION is required, see --help.")
616         revision = args[0]
617         bug_id = self._parse_bug_id_from_revision_diff(tool, revision)
618         if options.complete_rollout:
619             if bug_id:
620                 log("Will re-open bug %s after rollout." % bug_id)
621             else:
622                 log("Failed to parse bug number from diff.  No bugs will be updated/reopened after the rollout.")
623
624         WebKitLandingScripts.prepare_clean_working_directory(tool.scm(), options)
625         tool.scm().update_webkit()
626         tool.scm().apply_reverse_diff(revision)
627         self._create_changelogs_for_revert(tool.scm(), revision)
628
629         # FIXME: Fully automated rollout is not 100% idiot-proof yet, so for now just log with instructions on how to complete the rollout.
630         # Once we trust rollout we will remove this option.
631         if not options.complete_rollout:
632             log("\nNOTE: Rollout support is experimental.\nPlease verify the rollout diff and use 'bugzilla-tool land-diff %s' to commit the rollout." % bug_id)
633         else:
634             comment_text = WebKitLandingScripts.build_and_commit(tool.scm(), options)
635             self._reopen_bug_after_rollout(tool, bug_id, comment_text)
636
637
638 class CreateBug(Command):
639     def __init__(self):
640         options = [
641             make_option("--cc", action="store", type="string", dest="cc", help="Comma-separated list of email addresses to carbon-copy."),
642             make_option("--component", action="store", type="string", dest="component", help="Component for the new bug."),
643             make_option("--no-prompt", action="store_false", dest="prompt", default=True, help="Do not prompt for bug title and comment; use commit log instead."),
644             make_option("--no-review", action="store_false", dest="review", default=True, help="Do not mark the patch for review."),
645             make_option("--request-commit", action="store_true", dest="request_commit", default=False, help="Mark the patch as needing auto-commit after review."),
646         ]
647         Command.__init__(self, 'Create a bug from local changes or local commits.', '[COMMITISH]', options=options)
648
649     def create_bug_from_commit(self, options, args, tool):
650         commit_ids = tool.scm().commit_ids_from_commitish_arguments(args)
651         if len(commit_ids) > 3:
652             error("Are you sure you want to create one bug with %s patches?" % len(commit_ids))
653
654         commit_id = commit_ids[0]
655
656         bug_title = ""
657         comment_text = ""
658         if options.prompt:
659             (bug_title, comment_text) = self.prompt_for_bug_title_and_comment()
660         else:
661             commit_message = tool.scm().commit_message_for_local_commit(commit_id)
662             bug_title = commit_message.description(lstrip=True, strip_url=True)
663             comment_text = commit_message.body(lstrip=True)
664             comment_text += "---\n"
665             comment_text += tool.scm().files_changed_summary_for_commit(commit_id)
666
667         diff = tool.scm().create_patch_from_local_commit(commit_id)
668         diff_file = StringIO.StringIO(diff) # create_bug_with_patch expects a file-like object
669         bug_id = tool.bugs.create_bug_with_patch(bug_title, comment_text, options.component, diff_file, "Patch", cc=options.cc, mark_for_review=options.review, mark_for_commit_queue=options.request_commit)
670
671         if bug_id and len(commit_ids) > 1:
672             options.bug_id = bug_id
673             options.obsolete_patches = False
674             # FIXME: We should pass through --no-comment switch as well.
675             PostCommitsAsPatchesToBug.execute(self, options, commit_ids[1:], tool)
676
677     def create_bug_from_patch(self, options, args, tool):
678         bug_title = ""
679         comment_text = ""
680         if options.prompt:
681             (bug_title, comment_text) = self.prompt_for_bug_title_and_comment()
682         else:
683             commit_message = commit_message_for_this_commit(tool.scm())
684             bug_title = commit_message.description(lstrip=True, strip_url=True)
685             comment_text = commit_message.body(lstrip=True)
686
687         diff = tool.scm().create_patch()
688         diff_file = StringIO.StringIO(diff) # create_bug_with_patch expects a file-like object
689         bug_id = tool.bugs.create_bug_with_patch(bug_title, comment_text, options.component, diff_file, "Patch", cc=options.cc, mark_for_review=options.review, mark_for_commit_queue=options.request_commit)
690
691     def prompt_for_bug_title_and_comment(self):
692         bug_title = raw_input("Bug title: ")
693         print "Bug comment (hit ^D on blank line to end):"
694         lines = sys.stdin.readlines()
695         try:
696             sys.stdin.seek(0, os.SEEK_END)
697         except IOError:
698             # Cygwin raises an Illegal Seek (errno 29) exception when the above
699             # seek() call is made. Ignoring it seems to cause no harm.
700             # FIXME: Figure out a way to get avoid the exception in the first
701             # place.
702             pass
703         comment_text = ''.join(lines)
704         return (bug_title, comment_text)
705
706     def execute(self, options, args, tool):
707         if len(args):
708             if (not tool.scm().supports_local_commits()):
709                 error("Extra arguments not supported; patch is taken from working directory.")
710             self.create_bug_from_commit(options, args, tool)
711         else:
712             self.create_bug_from_patch(options, args, tool)
713
714
715 class CheckTreeStatus(Command):
716     def __init__(self):
717         Command.__init__(self, 'Print out the status of the webkit builders.')
718
719     def execute(self, options, args, tool):
720         for builder in tool.buildbot.builder_statuses():
721             status_string = "ok" if builder['is_green'] else 'FAIL'
722             print "%s : %s" % (status_string.ljust(4), builder['name'])
723
724
725 class AbstractQueue(Command, WorkQueueDelegate):
726     def __init__(self, name):
727         self._name = name
728         options = [
729             make_option("--no-confirm", action="store_false", dest="confirm", default=True, help="Do not ask the user for confirmation before running the queue.  Dangerous!"),
730             make_option("--status-host", action="store", type="string", dest="status_host", default=StatusBot.default_host, help="Hostname (e.g. localhost or commit.webkit.org) where status updates should be posted."),
731         ]
732         Command.__init__(self, 'Run the %s.' % self._name, options=options)
733
734     def queue_log_path(self):
735         return '%s.log' % self._name
736
737     def work_logs_directory(self):
738         return '%s-logs' % self._name
739
740     def status_host(self):
741         return self.options.status_host
742
743     def begin_work_queue(self):
744         log("CAUTION: %s will discard all local changes in %s" % (self._name, self.tool.scm().checkout_root))
745         if self.options.confirm:
746             response = raw_input("Are you sure?  Type 'yes' to continue: ")
747             if (response != 'yes'):
748                 error("User declined.")
749         log("Running WebKit %s. %s" % (self._name, datetime.now().strftime(WorkQueue.log_date_format)))
750
751     def should_continue_work_queue(self):
752         return True
753
754     def next_work_item(self):
755         raise NotImplementedError, "subclasses must implement"
756
757     def should_proceed_with_work_item(self, work_item):
758         raise NotImplementedError, "subclasses must implement"
759
760     def process_work_item(self, work_item):
761         raise NotImplementedError, "subclasses must implement"
762
763     def handle_unexpected_error(self, work_item, message):
764         raise NotImplementedError, "subclasses must implement"
765
766     @staticmethod
767     def run_bugzilla_tool(args):
768         bugzilla_tool_path = __file__ # re-execute this script
769         bugzilla_tool_args = [bugzilla_tool_path] + args
770         WebKitLandingScripts.run_and_throw_if_fail(bugzilla_tool_args)
771
772     def log_progress(self, patch_ids):
773         log("%s in %s [%s]" % (pluralize('patch', len(patch_ids)), self._name, ", ".join(patch_ids)))
774
775     def execute(self, options, args, tool):
776         self.options = options
777         self.tool = tool
778         work_queue = WorkQueue(self)
779         work_queue.run()
780
781
782 class CommitQueue(AbstractQueue):
783     def __init__(self):
784         AbstractQueue.__init__(self, "commit-queue")
785
786     def begin_work_queue(self):
787         AbstractQueue.begin_work_queue(self)
788
789     def next_work_item(self):
790         patches = self.tool.bugs.fetch_patches_from_commit_queue(reject_invalid_patches=True)
791         self.log_progress([patch['id'] for patch in patches])
792         return patches[0] if patches else None
793
794     def should_proceed_with_work_item(self, patch):
795         red_builders_names = self.tool.buildbot.red_core_builders_names()
796         if red_builders_names:
797             red_builders_names = map(lambda name: '"%s"' % name, red_builders_names) # Add quotes around the names.
798             return (False, "Builders [%s] are red. See http://build.webkit.org." % ", ".join(red_builders_names), None)
799         return (True, "Landing patch %s from bug %s." % (patch['id'], patch['bug_id']), patch['bug_id'])
800
801     def process_work_item(self, patch):
802         self.run_bugzilla_tool(['land-attachment', '--force-clean', '--non-interactive', '--quiet', patch['id']])
803
804     def handle_unexpected_error(self, patch, message):
805         self.tool.bugs.reject_patch_from_commit_queue(patch['id'], message)
806
807
808 class StyleQueue(AbstractQueue):
809     def __init__(self):
810         AbstractQueue.__init__(self, "style-queue")
811
812     def status_host(self):
813         return None # FIXME: A hack until we come up with a more generic status page.
814
815     def begin_work_queue(self):
816         AbstractQueue.begin_work_queue(self)
817         self._patches = PatchCollection(self.tool.bugs)
818         self._patches.add_patches(self.tool.bugs.fetch_patches_from_review_queue(limit=10))
819
820     def next_work_item(self):
821         self.log_progress(self._patches.patch_ids())
822         return self._patches.next()
823
824     def should_proceed_with_work_item(self, patch):
825         return (True, "Checking style for patch %s on bug %s." % (patch['id'], patch['bug_id']), patch['bug_id'])
826
827     def process_work_item(self, patch):
828         self.run_bugzilla_tool(['check-style', '--force-clean', patch['id']])
829
830     def handle_unexpected_error(self, patch, message):
831         log(message)
832
833
834 class NonWrappingEpilogIndentedHelpFormatter(IndentedHelpFormatter):
835     def __init__(self):
836         IndentedHelpFormatter.__init__(self)
837
838     # The standard IndentedHelpFormatter paragraph-wraps the epilog, killing our custom formatting.
839     def format_epilog(self, epilog):
840         if epilog:
841             return "\n" + epilog + "\n"
842         return ""
843
844
845 class HelpPrintingOptionParser(OptionParser):
846     def error(self, msg):
847         self.print_usage(sys.stderr)
848         error_message = "%s: error: %s\n" % (self.get_prog_name(), msg)
849         error_message += "\nType '" + self.get_prog_name() + " --help' to see usage.\n"
850         self.exit(2, error_message)
851
852
853 class BugzillaTool:
854     def __init__(self):
855         self.cached_scm = None
856         self.bugs = Bugzilla()
857         self.buildbot = BuildBot()
858         self.commands = [
859             { 'name' : 'bugs-to-commit', 'object' : BugsInCommitQueue() },
860             { 'name' : 'patches-to-commit', 'object' : PatchesInCommitQueue() },
861             { 'name' : 'reviewed-patches', 'object' : ReviewedPatchesOnBug() },
862             { 'name' : 'create-bug', 'object' : CreateBug() },
863             { 'name' : 'apply-attachment', 'object' : ApplyAttachment() },
864             { 'name' : 'apply-patches', 'object' : ApplyPatchesFromBug() },
865             { 'name' : 'land-diff', 'object' : LandAndUpdateBug() },
866             { 'name' : 'land-attachment', 'object' : LandAttachment() },
867             { 'name' : 'land-patches', 'object' : LandPatchesFromBugs() },
868             { 'name' : 'check-style', 'object' : CheckStyle() },
869             { 'name' : 'commit-message', 'object' : CommitMessageForCurrentDiff() },
870             { 'name' : 'obsolete-attachments', 'object' : ObsoleteAttachmentsOnBug() },
871             { 'name' : 'post-diff', 'object' : PostDiffAsPatchToBug() },
872             { 'name' : 'post-commits', 'object' : PostCommitsAsPatchesToBug() },
873             { 'name' : 'tree-status', 'object' : CheckTreeStatus() },
874             { 'name' : 'commit-queue', 'object' : CommitQueue() },
875             { 'name' : 'style-queue', 'object' : StyleQueue() },
876             { 'name' : 'rollout', 'object' : RolloutCommit() },
877         ]
878
879         self.global_option_parser = HelpPrintingOptionParser(usage=self.usage_line(), formatter=NonWrappingEpilogIndentedHelpFormatter(), epilog=self.commands_usage())
880         self.global_option_parser.add_option("--dry-run", action="store_true", dest="dryrun", help="do not touch remote servers", default=False)
881
882     def scm(self):
883         # Lazily initialize SCM to not error-out before command line parsing (or when running non-scm commands).
884         original_cwd = os.path.abspath('.')
885         if not self.cached_scm:
886             self.cached_scm = detect_scm_system(original_cwd)
887         
888         if not self.cached_scm:
889             script_directory = os.path.abspath(sys.path[0])
890             webkit_directory = os.path.abspath(os.path.join(script_directory, "../.."))
891             self.cached_scm = detect_scm_system(webkit_directory)
892             if self.cached_scm:
893                 log("The current directory (%s) is not a WebKit checkout, using %s" % (original_cwd, webkit_directory))
894             else:
895                 error("FATAL: Failed to determine the SCM system for either %s or %s" % (original_cwd, webkit_directory))
896         
897         return self.cached_scm
898
899     @staticmethod
900     def usage_line():
901         return "Usage: %prog [options] command [command-options] [command-arguments]"
902
903     def commands_usage(self):
904         commands_text = "Commands:\n"
905         longest_name_length = 0
906         command_rows = []
907         scm_supports_local_commits = self.scm().supports_local_commits()
908         for command in self.commands:
909             command_object = command['object']
910             if command_object.requires_local_commits and not scm_supports_local_commits:
911                 continue
912             command_name_and_args = command_object.name_with_arguments(command['name'])
913             command_rows.append({ 'name-and-args': command_name_and_args, 'object': command_object })
914             longest_name_length = max([longest_name_length, len(command_name_and_args)])
915         
916         # Use our own help formatter so as to indent enough.
917         formatter = IndentedHelpFormatter()
918         formatter.indent()
919         formatter.indent()
920         
921         for row in command_rows:
922             command_object = row['object']
923             commands_text += "  " + row['name-and-args'].ljust(longest_name_length + 3) + command_object.help_text + "\n"
924             commands_text += command_object.option_parser.format_option_help(formatter)
925         return commands_text
926
927     def handle_global_args(self, args):
928         (options, args) = self.global_option_parser.parse_args(args)
929         if len(args):
930             # We'll never hit this because split_args splits at the first arg without a leading '-'
931             self.global_option_parser.error("Extra arguments before command: " + args)
932         
933         if options.dryrun:
934             self.scm().dryrun = True
935             self.bugs.dryrun = True
936     
937     @staticmethod
938     def split_args(args):
939         # Assume the first argument which doesn't start with '-' is the command name.
940         command_index = 0
941         for arg in args:
942             if arg[0] != '-':
943                 break
944             command_index += 1
945         else:
946             return (args[:], None, [])
947
948         global_args = args[:command_index]
949         command = args[command_index]
950         command_args = args[command_index + 1:]
951         return (global_args, command, command_args)
952     
953     def command_by_name(self, command_name):
954         for command in self.commands:
955             if command_name == command['name']:
956                 return command
957         return None
958     
959     def main(self):
960         (global_args, command_name, args_after_command_name) = self.split_args(sys.argv[1:])
961         
962         # Handle --help, etc:
963         self.handle_global_args(global_args)
964         
965         if not command_name:
966             self.global_option_parser.error("No command specified")
967         
968         command = self.command_by_name(command_name)
969         if not command:
970             self.global_option_parser.error(command_name + " is not a recognized command")
971
972         command_object = command['object']
973
974         if command_object.requires_local_commits and not self.scm().supports_local_commits():
975             error(command_name + " requires local commits using %s in %s." % (self.scm().display_name(), self.scm().checkout_root))
976
977         (command_options, command_args) = command_object.parse_args(args_after_command_name)
978         return command_object.execute(command_options, command_args, self)
979
980
981 def main():
982     tool = BugzillaTool()
983     return tool.main()
984
985 if __name__ == "__main__":
986     main()