2009-11-16 Eric Seidel <eric@webkit.org>
[WebKit-https.git] / WebKitTools / Scripts / bugzilla-tool
1 #!/usr/bin/env python
2 # Copyright (c) 2009, Google Inc. All rights reserved.
3 # Copyright (c) 2009 Apple Inc. All rights reserved.
4 #
5 # Redistribution and use in source and binary forms, with or without
6 # modification, are permitted provided that the following conditions are
7 # met:
8
9 #     * Redistributions of source code must retain the above copyright
10 # notice, this list of conditions and the following disclaimer.
11 #     * Redistributions in binary form must reproduce the above
12 # copyright notice, this list of conditions and the following disclaimer
13 # in the documentation and/or other materials provided with the
14 # distribution.
15 #     * Neither the name of Google Inc. nor the names of its
16 # contributors may be used to endorse or promote products derived from
17 # this software without specific prior written permission.
18
19 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30 #
31 # A tool for automating dealing with bugzilla, posting patches, committing patches, etc.
32
33 import os
34 import re
35 import StringIO # for add_patch_to_bug file wrappers
36 import subprocess
37 import sys
38 import time
39
40 from datetime import datetime, timedelta
41 from optparse import OptionParser, IndentedHelpFormatter, SUPPRESS_USAGE, make_option
42
43 # Import WebKit-specific modules.
44 from modules.bugzilla import Bugzilla, parse_bug_id
45 from modules.changelogs import ChangeLog
46 from modules.comments import bug_comment_from_commit_text
47 from modules.logging import error, log, tee
48 from modules.scm import CommitMessage, detect_scm_system, ScriptError, CheckoutNeedsUpdate
49 from modules.buildbot import BuildBot
50 from modules.statusbot import StatusBot
51 from modules.workqueue import WorkQueue, WorkQueueDelegate
52
53 def plural(noun):
54     # This is a dumb plural() implementation which was just enough for our uses.
55     if re.search('h$', noun):
56         return noun + 'es'
57     else:
58         return noun + 's'
59
60 def pluralize(noun, count):
61     if count != 1:
62         noun = plural(noun)
63     return "%d %s" % (count, noun)
64
65 def commit_message_for_this_commit(scm):
66     changelog_paths = scm.modified_changelogs()
67     if not len(changelog_paths):
68         raise ScriptError(message="Found no modified ChangeLogs, cannot create a commit message.\n"
69                           "All changes require a ChangeLog.  See:\n"
70                           "http://webkit.org/coding/contributing.html")
71
72     changelog_messages = []
73     for changelog_path in changelog_paths:
74         log("Parsing ChangeLog: %s" % changelog_path)
75         changelog_entry = ChangeLog(changelog_path).latest_entry()
76         if not changelog_entry:
77             raise ScriptError(message="Failed to parse ChangeLog: " + os.path.abspath(changelog_path))
78         changelog_messages.append(changelog_entry)
79
80     # FIXME: We should sort and label the ChangeLog messages like commit-log-editor does.
81     return CommitMessage(''.join(changelog_messages).splitlines())
82
83
84 class Command:
85     def __init__(self, help_text, argument_names="", options=[], requires_local_commits=False):
86         self.help_text = help_text
87         self.argument_names = argument_names
88         self.options = options
89         self.option_parser = HelpPrintingOptionParser(usage=SUPPRESS_USAGE, add_help_option=False, option_list=self.options)
90         self.requires_local_commits = requires_local_commits
91     
92     def name_with_arguments(self, command_name):
93         usage_string = command_name
94         if len(self.options) > 0:
95             usage_string += " [options]"
96         if self.argument_names:
97             usage_string += " " + self.argument_names
98         return usage_string
99
100     def parse_args(self, args):
101         return self.option_parser.parse_args(args)
102
103     def execute(self, options, args, tool):
104         raise NotImplementedError, "subclasses must implement"
105
106
107 class BugsInCommitQueue(Command):
108     def __init__(self):
109         Command.__init__(self, 'Bugs in the commit queue')
110
111     def execute(self, options, args, tool):
112         bug_ids = tool.bugs.fetch_bug_ids_from_commit_queue()
113         for bug_id in bug_ids:
114             print "%s" % bug_id
115
116
117 class PatchesInCommitQueue(Command):
118     def __init__(self):
119         Command.__init__(self, 'Patches in the commit queue')
120
121     def execute(self, options, args, tool):
122         patches = tool.bugs.fetch_patches_from_commit_queue()
123         log("Patches in commit queue:")
124         for patch in patches:
125             print "%s" % patch['url']
126
127
128 class ReviewedPatchesOnBug(Command):
129     def __init__(self):
130         Command.__init__(self, 'r+\'d patches on a bug', 'BUGID')
131
132     def execute(self, options, args, tool):
133         bug_id = args[0]
134         patches_to_land = tool.bugs.fetch_reviewed_patches_from_bug(bug_id)
135         for patch in patches_to_land:
136             print "%s" % patch['url']
137
138
139 class CheckStyleOnBug(Command):
140     def __init__(self):
141         options = WebKitLandingScripts.cleaning_options()
142         Command.__init__(self, 'Runs check-webkit-style on the r? patches on a bug', 'BUGID', options=options)
143
144     @classmethod
145     def check_style(cls, bug_id, patch, options, tool):
146         tool.scm().update_webkit()
147         log("Checking style for patch %s from bug %s." % (patch['id'], bug_id))
148         try:
149             tool.scm().apply_patch(patch)
150             WebKitLandingScripts.run_webkit_script("check-webkit-style")
151         except ScriptError, e:
152             log("Patch failed to apply and check style")
153             log(e.output)
154
155         # This is safe because in order to get here the working directory had to be
156         # clean at the beginning.  Clean it out again before we exit.
157         tool.scm().ensure_clean_working_directory(force_clean=True)
158
159     def execute(self, options, args, tool):
160         bug_id = args[0]
161         patches = tool.bugs.fetch_unreviewed_patches_from_bug(bug_id)
162
163         WebKitLandingScripts.setup_for_landing(tool.scm(), options)
164
165         for patch in patches:
166             self.check_style(bug_id, patch, options, tool)
167
168
169 class ApplyPatchesFromBug(Command):
170     def __init__(self):
171         options = [
172             make_option("--no-update", action="store_false", dest="update", default=True, help="Don't update the working directory before applying patches"),
173             make_option("--local-commit", action="store_true", dest="local_commit", default=False, help="Make a local commit for each applied patch"),
174         ]
175         options += WebKitLandingScripts.cleaning_options()
176         Command.__init__(self, 'Applies all patches on a bug to the local working directory without committing.', 'BUGID', options=options)
177
178     @staticmethod
179     def apply_patches(patches, scm, commit_each):
180         for patch in patches:
181             scm.apply_patch(patch)
182             if commit_each:
183                 commit_message = commit_message_for_this_commit(scm)
184                 scm.commit_locally_with_message(commit_message.message() or patch['name'])
185
186     def execute(self, options, args, tool):
187         bug_id = args[0]
188         patches = tool.bugs.fetch_reviewed_patches_from_bug(bug_id)
189         os.chdir(tool.scm().checkout_root)
190         if options.clean:
191             tool.scm().ensure_clean_working_directory(options.force_clean)
192         if options.update:
193             tool.scm().update_webkit()
194         
195         if options.local_commit and not tool.scm().supports_local_commits():
196             error("--local-commit passed, but %s does not support local commits" % tool.scm().display_name())
197         
198         self.apply_patches(patches, tool.scm(), options.local_commit)
199
200
201 class WebKitLandingScripts:
202     @staticmethod
203     def cleaning_options():
204         return [
205             make_option("--force-clean", action="store_true", dest="force_clean", default=False, help="Clean working directory before applying patches (removes local changes and commits)"),
206             make_option("--no-clean", action="store_false", dest="clean", default=True, help="Don't check if the working directory is clean before applying patches"),
207         ]
208
209     @staticmethod
210     def land_options():
211         return [
212             make_option("--ignore-builders", action="store_false", dest="check_builders", default=True, help="Don't check to see if the build.webkit.org builders are green before landing."),
213             make_option("--no-close", action="store_false", dest="close_bug", default=True, help="Leave bug open after landing."),
214             make_option("--no-build", action="store_false", dest="build", default=True, help="Commit without building first, implies --no-test."),
215             make_option("--no-test", action="store_false", dest="test", default=True, help="Commit without running run-webkit-tests."),
216             make_option("--quiet", action="store_true", dest="quiet", default=False, help="Produce less console output."),
217             make_option("--commit-queue", action="store_true", dest="commit_queue", default=False, help="Run in commit queue mode (no user interaction)."),
218         ]
219
220     @staticmethod
221     def run_command_with_teed_output(args, teed_output):
222         child_process = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
223
224         # Use our own custom wait loop because Popen ignores a tee'd stderr/stdout.
225         # FIXME: This could be improved not to flatten output to stdout.
226         while True:
227             output_line = child_process.stdout.readline()
228             if output_line == '' and child_process.poll() != None:
229                 return child_process.poll()
230             teed_output.write(output_line)
231
232     @staticmethod
233     def run_and_throw_if_fail(args, quiet=False):
234         # Cache the child's output locally so it can be used for error reports.
235         child_out_file = StringIO.StringIO()
236         if quiet:
237             dev_null = open(os.devnull, "w")
238         child_stdout = tee(child_out_file, dev_null if quiet else sys.stdout)
239         exit_code = WebKitLandingScripts.run_command_with_teed_output(args, child_stdout)
240         if quiet:
241             dev_null.close()
242
243         child_output = child_out_file.getvalue()
244         child_out_file.close()
245
246         if exit_code:
247             raise ScriptError(script_args=args, exit_code=exit_code, output=child_output)
248
249     # We might need to pass scm into this function for scm.checkout_root
250     @staticmethod
251     def webkit_script_path(script_name):
252         return os.path.join("WebKitTools", "Scripts", script_name)
253
254     @classmethod
255     def run_webkit_script(cls, script_name, quiet=False):
256         log("Running %s" % script_name)
257         cls.run_and_throw_if_fail(cls.webkit_script_path(script_name), quiet)
258
259     @classmethod
260     def build_webkit(cls, quiet=False):
261         cls.run_webkit_script("build-webkit", quiet)
262
263     @staticmethod
264     def ensure_builders_are_green(buildbot, options):
265         if not options.check_builders or buildbot.core_builders_are_green():
266             return
267         error("Builders at %s are red, please do not commit.  Pass --ignore-builders to bypass this check." % (buildbot.buildbot_host))
268
269     @classmethod
270     def run_webkit_tests(cls, launch_safari, fail_fast=False, quiet=False):
271         args = [cls.webkit_script_path("run-webkit-tests")]
272         if not launch_safari:
273             args.append("--no-launch-safari")
274         if quiet:
275             args.append("--quiet")
276         if fail_fast:
277             args.append("--exit-after-n-failures=1")
278         cls.run_and_throw_if_fail(args)
279
280     @staticmethod
281     def setup_for_landing(scm, options):
282         os.chdir(scm.checkout_root)
283         scm.ensure_no_local_commits(options.force_clean)
284         if options.clean:
285             scm.ensure_clean_working_directory(force_clean=options.force_clean)
286
287     @classmethod
288     def build_and_commit(cls, scm, options):
289         if options.build:
290             cls.build_webkit(quiet=options.quiet)
291             if options.test:
292                 # When running the commit-queue we don't want to launch Safari and we want to exit after the first failure.
293                 cls.run_webkit_tests(launch_safari=not options.commit_queue, fail_fast=options.commit_queue, quiet=options.quiet)
294         commit_message = commit_message_for_this_commit(scm)
295         commit_log = scm.commit_with_message(commit_message.message())
296         return bug_comment_from_commit_text(scm, commit_log)
297
298
299 class LandAndUpdateBug(Command):
300     def __init__(self):
301         options = [
302             make_option("-r", "--reviewer", action="store", type="string", dest="reviewer", help="Update ChangeLogs to say Reviewed by REVIEWER."),
303         ]
304         options += WebKitLandingScripts.land_options()
305         Command.__init__(self, 'Lands the current working directory diff and updates the bug if provided.', '[BUGID]', options=options)
306
307     def guess_reviewer_from_bug(self, bugs, bug_id):
308         patches = bugs.fetch_reviewed_patches_from_bug(bug_id)
309         if len(patches) != 1:
310             log("%s on bug %s, cannot infer reviewer." % (pluralize("reviewed patch", len(patches)), bug_id))
311             return None
312         patch = patches[0]
313         reviewer = patch['reviewer']
314         log('Guessing "%s" as reviewer from attachment %s on bug %s.' % (reviewer, patch['id'], bug_id))
315         return reviewer
316
317     def update_changelogs_with_reviewer(self, reviewer, bug_id, tool):
318         if not reviewer:
319             if not bug_id:
320                 log("No bug id provided and --reviewer= not provided.  Not updating ChangeLogs with reviewer.")
321                 return
322             reviewer = self.guess_reviewer_from_bug(tool.bugs, bug_id)
323
324         if not reviewer:
325             log("Failed to guess reviewer from bug %s and --reviewer= not provided.  Not updating ChangeLogs with reviewer." % bug_id)
326             return
327
328         for changelog_path in tool.scm().modified_changelogs():
329             ChangeLog(changelog_path).set_reviewer(reviewer)
330
331     def execute(self, options, args, tool):
332         bug_id = args[0] if len(args) else None
333         os.chdir(tool.scm().checkout_root)
334
335         WebKitLandingScripts.ensure_builders_are_green(tool.buildbot, options)
336
337         self.update_changelogs_with_reviewer(options.reviewer, bug_id, tool)
338
339         comment_text = WebKitLandingScripts.build_and_commit(tool.scm(), options)
340         if bug_id:
341             log("Updating bug %s" % bug_id)
342             if options.close_bug:
343                 tool.bugs.close_bug_as_fixed(bug_id, comment_text)
344             else:
345                 # FIXME: We should a smart way to figure out if the patch is attached
346                 # to the bug, and if so obsolete it.
347                 tool.bugs.post_comment_to_bug(bug_id, comment_text)
348         else:
349             log(comment_text)
350             log("No bug id provided.")
351
352
353 class LandPatchesFromBugs(Command):
354     def __init__(self):
355         options = WebKitLandingScripts.cleaning_options()
356         options += WebKitLandingScripts.land_options()
357         Command.__init__(self, 'Lands all patches on a bug optionally testing them first', 'BUGID', options=options)
358
359     @staticmethod
360     def handled_error(error):
361         log(error)
362         exit(2) # Exit 2 insted of 1 to indicate to the commit-queue to indicate we handled the error, and that the queue should keep looping.
363
364     @classmethod
365     def land_patches(cls, bug_id, patches, options, tool):
366         try:
367             comment_text = ""
368             for patch in patches:
369                 tool.scm().update_webkit() # Update before every patch in case the tree has changed
370                 log("Applying %s from bug %s." % (patch['id'], bug_id))
371                 tool.scm().apply_patch(patch, force=options.commit_queue)
372                 # Make sure the tree is still green after updating, before building this patch.
373                 # The first patch ends up checking tree status twice, but that's OK.
374                 WebKitLandingScripts.ensure_builders_are_green(tool.buildbot, options)
375                 comment_text = WebKitLandingScripts.build_and_commit(tool.scm(), options)
376                 tool.bugs.clear_attachment_flags(patch['id'], comment_text)
377
378             if options.close_bug:
379                 tool.bugs.close_bug_as_fixed(bug_id, "All reviewed patches have been landed.  Closing bug.")
380         except CheckoutNeedsUpdate, e:
381             log("Commit failed because the checkout is out of date.  Please update and try again.")
382             log("You can pass --no-build to skip building/testing after update if you believe the new commits did not affect the results.")
383             cls.handled_error(e)
384         except ScriptError, e:
385             # Mark the patch as commit-queue- and comment in the bug.
386             tool.bugs.reject_patch_from_commit_queue(patch['id'], e.message_with_output())
387             cls.handled_error(e)
388
389     @staticmethod
390     def _fetch_list_of_patches_to_land(options, args, tool):
391         bugs_to_patches = {}
392         patch_count = 0
393         for bug_id in args:
394             patches = []
395             if options.commit_queue:
396                 patches = tool.bugs.fetch_commit_queue_patches_from_bug(bug_id, reject_invalid_patches=True)
397             else:
398                 patches = tool.bugs.fetch_reviewed_patches_from_bug(bug_id)
399
400             patches_found = len(patches)
401             log("%s found on bug %s." % (pluralize("reviewed patch", patches_found), bug_id))
402
403             patch_count += patches_found
404             if patches_found:
405                 bugs_to_patches[bug_id] = patches
406
407         log("Landing %s from %s." % (pluralize("patch", patch_count), pluralize("bug", len(args))))
408         return bugs_to_patches
409
410     def execute(self, options, args, tool):
411         if not len(args):
412             error("bug-id(s) required")
413
414         # Check the tree status here so we can fail early
415         WebKitLandingScripts.ensure_builders_are_green(tool.buildbot, options)
416
417         bugs_to_patches = self._fetch_list_of_patches_to_land(options, args, tool)
418
419         WebKitLandingScripts.setup_for_landing(tool.scm(), options)
420
421         for bug_id in bugs_to_patches.keys():
422             self.land_patches(bug_id, bugs_to_patches[bug_id], options, tool)
423
424
425 class CommitMessageForCurrentDiff(Command):
426     def __init__(self):
427         Command.__init__(self, 'Prints a commit message suitable for the uncommitted changes.')
428
429     def execute(self, options, args, tool):
430         os.chdir(tool.scm().checkout_root)
431         print "%s" % commit_message_for_this_commit(tool.scm()).message()
432
433
434 class ObsoleteAttachmentsOnBug(Command):
435     def __init__(self):
436         Command.__init__(self, 'Marks all attachments on a bug as obsolete.', 'BUGID')
437
438     def execute(self, options, args, tool):
439         bug_id = args[0]
440         attachments = tool.bugs.fetch_attachments_from_bug(bug_id)
441         for attachment in attachments:
442             if not attachment['is_obsolete']:
443                 tool.bugs.obsolete_attachment(attachment['id'])
444
445
446 class PostDiffAsPatchToBug(Command):
447     def __init__(self):
448         options = [
449             make_option("-m", "--description", action="store", type="string", dest="description", help="Description string for the attachment (default: 'patch')"),
450         ]
451         options += self.posting_options()
452         Command.__init__(self, 'Attaches the current working directory diff to a bug as a patch file.', '[BUGID]', options=options)
453
454     @staticmethod
455     def posting_options():
456         return [
457             make_option("--no-obsolete", action="store_false", dest="obsolete_patches", default=True, help="Do not obsolete old patches before posting this one."),
458             make_option("--no-review", action="store_false", dest="review", default=True, help="Do not mark the patch for review."),
459             make_option("--request-commit", action="store_true", dest="request_commit", default=False, help="Mark the patch as needing auto-commit after review."),
460         ]
461
462     @staticmethod
463     def obsolete_patches_on_bug(bug_id, bugs):
464         patches = bugs.fetch_patches_from_bug(bug_id)
465         if len(patches):
466             log("Obsoleting %s on bug %s" % (pluralize('old patch', len(patches)), bug_id))
467             for patch in patches:
468                 bugs.obsolete_attachment(patch['id'])
469
470     def execute(self, options, args, tool):
471         # Perfer a bug id passed as an argument over a bug url in the diff (i.e. ChangeLogs).
472         bug_id = (args and args[0]) or parse_bug_id(tool.scm().create_patch())
473         if not bug_id:
474             error("No bug id passed and no bug url found in diff, can't post.")
475
476         if options.obsolete_patches:
477             self.obsolete_patches_on_bug(bug_id, tool.bugs)
478
479         diff = tool.scm().create_patch()
480         diff_file = StringIO.StringIO(diff) # add_patch_to_bug expects a file-like object
481
482         description = options.description or "Patch"
483         tool.bugs.add_patch_to_bug(bug_id, diff_file, description, mark_for_review=options.review, mark_for_commit_queue=options.request_commit)
484
485
486 class PostCommitsAsPatchesToBug(Command):
487     def __init__(self):
488         options = [
489             make_option("-b", "--bug-id", action="store", type="string", dest="bug_id", help="Specify bug id if no URL is provided in the commit log."),
490             make_option("--add-log-as-comment", action="store_true", dest="add_log_as_comment", default=False, help="Add commit log message as a comment when uploading the patch."),
491             make_option("-m", "--description", action="store", type="string", dest="description", help="Description string for the attachment (default: description from commit message)"),
492         ]
493         options += PostDiffAsPatchToBug.posting_options()
494         Command.__init__(self, 'Attaches a range of local commits to bugs as patch files.', 'COMMITISH', options=options, requires_local_commits=True)
495
496     def _comment_text_for_commit(self, options, commit_message, tool, commit_id):
497         comment_text = None
498         if (options.add_log_as_comment):
499             comment_text = commit_message.body(lstrip=True)
500             comment_text += "---\n"
501             comment_text += tool.scm().files_changed_summary_for_commit(commit_id)
502         return comment_text
503
504     def _diff_file_for_commit(self, tool, commit_id):
505         diff = tool.scm().create_patch_from_local_commit(commit_id)
506         return StringIO.StringIO(diff) # add_patch_to_bug expects a file-like object
507
508     def execute(self, options, args, tool):
509         if not args:
510             error("%s argument is required" % self.argument_names)
511
512         commit_ids = tool.scm().commit_ids_from_commitish_arguments(args)
513         if len(commit_ids) > 10: # We could lower this limit, 10 is too many for one bug as-is.
514             error("bugzilla-tool does not support attaching %s at once.  Are you sure you passed the right commit range?" % (pluralize('patch', len(commit_ids))))
515
516         have_obsoleted_patches = set()
517         for commit_id in commit_ids:
518             commit_message = tool.scm().commit_message_for_local_commit(commit_id)
519
520             # Prefer --bug-id=, then a bug url in the commit message, then a bug url in the entire commit diff (i.e. ChangeLogs).
521             bug_id = options.bug_id or parse_bug_id(commit_message.message()) or parse_bug_id(tool.scm().create_patch_from_local_commit(commit_id))
522             if not bug_id:
523                 log("Skipping %s: No bug id found in commit or specified with --bug-id." % commit_id)
524                 continue
525
526             if options.obsolete_patches and bug_id not in have_obsoleted_patches:
527                 PostDiffAsPatchToBug.obsolete_patches_on_bug(bug_id, tool.bugs)
528                 have_obsoleted_patches.add(bug_id)
529
530             diff_file = self._diff_file_for_commit(tool, commit_id)
531             description = options.description or commit_message.description(lstrip=True, strip_url=True)
532             comment_text = self._comment_text_for_commit(options, commit_message, tool, commit_id)
533             tool.bugs.add_patch_to_bug(bug_id, diff_file, description, comment_text, mark_for_review=options.review, mark_for_commit_queue=options.request_commit)
534
535
536 class RolloutCommit(Command):
537     def __init__(self):
538         options = WebKitLandingScripts.land_options()
539         options += WebKitLandingScripts.cleaning_options()
540         options.append(make_option("--complete-rollout", action="store_true", dest="complete_rollout", help="Experimental support for complete unsupervised rollouts, including re-opening the bug.  Not recommended."))
541         Command.__init__(self, 'Reverts the given revision and commits the revert and re-opens the original bug.', 'REVISION [BUGID]', options=options)
542
543     @staticmethod
544     def _create_changelogs_for_revert(scm, revision):
545         # First, discard the ChangeLog changes from the rollout.
546         changelog_paths = scm.modified_changelogs()
547         scm.revert_files(changelog_paths)
548
549         # Second, make new ChangeLog entries for this rollout.
550         # This could move to prepare-ChangeLog by adding a --revert= option.
551         WebKitLandingScripts.run_webkit_script("prepare-ChangeLog")
552         for changelog_path in changelog_paths:
553             ChangeLog(changelog_path).update_for_revert(revision)
554
555     @staticmethod
556     def _parse_bug_id_from_revision_diff(tool, revision):
557         original_diff = tool.scm().diff_for_revision(revision)
558         return parse_bug_id(original_diff)
559
560     @staticmethod
561     def _reopen_bug_after_rollout(tool, bug_id, comment_text):
562         if bug_id:
563             tool.bugs.reopen_bug(bug_id, comment_text)
564         else:
565             log(comment_text)
566             log("No bugs were updated or re-opened to reflect this rollout.")
567
568     def execute(self, options, args, tool):
569         if not args:
570             error("REVISION is required, see --help.")
571         revision = args[0]
572         bug_id = self._parse_bug_id_from_revision_diff(tool, revision)
573         if options.complete_rollout:
574             if bug_id:
575                 log("Will re-open bug %s after rollout." % bug_id)
576             else:
577                 log("Failed to parse bug number from diff.  No bugs will be updated/reopened after the rollout.")
578
579         WebKitLandingScripts.setup_for_landing(tool.scm(), options)
580         tool.scm().update_webkit()
581         tool.scm().apply_reverse_diff(revision)
582         self._create_changelogs_for_revert(tool.scm(), revision)
583
584         # FIXME: Fully automated rollout is not 100% idiot-proof yet, so for now just log with instructions on how to complete the rollout.
585         # Once we trust rollout we will remove this option.
586         if not options.complete_rollout:
587             log("\nNOTE: Rollout support is experimental.\nPlease verify the rollout diff and use 'bugzilla-tool land-diff %s' to commit the rollout." % bug_id)
588         else:
589             comment_text = WebKitLandingScripts.build_and_commit(tool.scm(), options)
590             self._reopen_bug_after_rollout(tool, bug_id, comment_text)
591
592
593 class CreateBug(Command):
594     def __init__(self):
595         options = [
596             make_option("--cc", action="store", type="string", dest="cc", help="Comma-separated list of email addresses to carbon-copy."),
597             make_option("--component", action="store", type="string", dest="component", help="Component for the new bug."),
598             make_option("--no-prompt", action="store_false", dest="prompt", default=True, help="Do not prompt for bug title and comment; use commit log instead."),
599             make_option("--no-review", action="store_false", dest="review", default=True, help="Do not mark the patch for review."),
600             make_option("--request-commit", action="store_true", dest="request_commit", default=False, help="Mark the patch as needing auto-commit after review."),
601         ]
602         Command.__init__(self, 'Create a bug from local changes or local commits.', '[COMMITISH]', options=options)
603
604     def create_bug_from_commit(self, options, args, tool):
605         commit_ids = tool.scm().commit_ids_from_commitish_arguments(args)
606         if len(commit_ids) > 3:
607             error("Are you sure you want to create one bug with %s patches?" % len(commit_ids))
608
609         commit_id = commit_ids[0]
610
611         bug_title = ""
612         comment_text = ""
613         if options.prompt:
614             (bug_title, comment_text) = self.prompt_for_bug_title_and_comment()
615         else:
616             commit_message = tool.scm().commit_message_for_local_commit(commit_id)
617             bug_title = commit_message.description(lstrip=True, strip_url=True)
618             comment_text = commit_message.body(lstrip=True)
619             comment_text += "---\n"
620             comment_text += tool.scm().files_changed_summary_for_commit(commit_id)
621
622         diff = tool.scm().create_patch_from_local_commit(commit_id)
623         diff_file = StringIO.StringIO(diff) # create_bug_with_patch expects a file-like object
624         bug_id = tool.bugs.create_bug_with_patch(bug_title, comment_text, options.component, diff_file, "Patch", cc=options.cc, mark_for_review=options.review, mark_for_commit_queue=options.request_commit)
625
626         if bug_id and len(commit_ids) > 1:
627             options.bug_id = bug_id
628             options.obsolete_patches = False
629             # FIXME: We should pass through --no-comment switch as well.
630             PostCommitsAsPatchesToBug.execute(self, options, commit_ids[1:], tool)
631
632     def create_bug_from_patch(self, options, args, tool):
633         bug_title = ""
634         comment_text = ""
635         if options.prompt:
636             (bug_title, comment_text) = self.prompt_for_bug_title_and_comment()
637         else:
638             commit_message = commit_message_for_this_commit(tool.scm())
639             bug_title = commit_message.description(lstrip=True, strip_url=True)
640             comment_text = commit_message.body(lstrip=True)
641
642         diff = tool.scm().create_patch()
643         diff_file = StringIO.StringIO(diff) # create_bug_with_patch expects a file-like object
644         bug_id = tool.bugs.create_bug_with_patch(bug_title, comment_text, options.component, diff_file, "Patch", cc=options.cc, mark_for_review=options.review, mark_for_commit_queue=options.request_commit)
645
646     def prompt_for_bug_title_and_comment(self):
647         bug_title = raw_input("Bug title: ")
648         print "Bug comment (hit ^D on blank line to end):"
649         lines = sys.stdin.readlines()
650         try:
651             sys.stdin.seek(0, os.SEEK_END)
652         except IOError:
653             # Cygwin raises an Illegal Seek (errno 29) exception when the above
654             # seek() call is made. Ignoring it seems to cause no harm.
655             # FIXME: Figure out a way to get avoid the exception in the first
656             # place.
657             pass
658         comment_text = ''.join(lines)
659         return (bug_title, comment_text)
660
661     def execute(self, options, args, tool):
662         if len(args):
663             if (not tool.scm().supports_local_commits()):
664                 error("Extra arguments not supported; patch is taken from working directory.")
665             self.create_bug_from_commit(options, args, tool)
666         else:
667             self.create_bug_from_patch(options, args, tool)
668
669
670 class CheckTreeStatus(Command):
671     def __init__(self):
672         Command.__init__(self, 'Print out the status of the webkit builders.')
673
674     def execute(self, options, args, tool):
675         for builder in tool.buildbot.builder_statuses():
676             status_string = "ok" if builder['is_green'] else 'FAIL'
677             print "%s : %s" % (status_string.ljust(4), builder['name'])
678
679
680 class LandPatchesFromCommitQueue(Command, WorkQueueDelegate):
681     def __init__(self):
682         options = [
683             make_option("--no-confirm", action="store_false", dest="confirm", default=True, help="Do not ask the user for confirmation before running the queue.  Dangerous!"),
684             make_option("--status-host", action="store", type="string", dest="status_host", default=StatusBot.default_host, help="Do not ask the user for confirmation before running the queue.  Dangerous!"),
685         ]
686         Command.__init__(self, 'Run the commit queue.', options=options)
687
688     def queue_log_path(self):
689         return 'commit_queue.log'
690
691     def work_logs_directory(self):
692         return 'commit_queue_logs'
693
694     def status_host(self):
695         return self.options.status_host
696
697     def begin_work_queue(self):
698         log("CAUTION: commit-queue will discard all local changes in %s" % self.tool.scm().checkout_root)
699         if self.options.confirm:
700             response = raw_input("Are you sure?  Type 'yes' to continue: ")
701             if (response != 'yes'):
702                 error("User declined.")
703         log("Running WebKit Commit Queue. %s" % datetime.now().strftime(WorkQueue.log_date_format))
704
705     def next_work_item(self):
706         # Fetch patches instead of just bug ids to that we validate reviewer/committer flags on every patch.
707         patches = self.tool.bugs.fetch_patches_from_commit_queue(reject_invalid_patches=True)
708         if not len(patches):
709             return None
710         patch_ids = map(lambda patch: patch['id'], patches)
711         log("%s in commit queue [%s]" % (pluralize('patch', len(patches)), ", ".join(patch_ids)))
712         return patches[0]['bug_id']
713
714     def should_proceed_with_work_item(self, bug_id):
715         red_builders_names = self.tool.buildbot.red_core_builders_names()
716         if red_builders_names:
717             red_builders_names = map(lambda name: '"%s"' % name, red_builders_names) # Add quotes around the names.
718             return (False, "Builders [%s] are red. See http://build.webkit.org." % ", ".join(red_builders_names), None)
719         return (True, "Landing patches from bug %s." % bug_id, bug_id)
720
721     def process_work_item(self, bug_id):
722         bugzilla_tool_path = __file__ # re-execute this script
723         bugzilla_tool_args = [bugzilla_tool_path, 'land-patches', '--force-clean', '--commit-queue', '--quiet', bug_id]
724         WebKitLandingScripts.run_and_throw_if_fail(bugzilla_tool_args)
725
726     def handle_unexpected_error(self, bug_id, message):
727         # We don't have a patch id at this point, so try to grab the first patch off
728         # of the bug in question.  We plan to update the commit-queue to opearate
729         # off of patch ids in the near future.
730         patches = self.tool.bugs.fetch_commit_queue_patches_from_bug(bug_id)
731         non_obsolete_patches = filter(lambda patch: not patch['is_obsolete'], patches)
732         if not len(non_obsolete_patches):
733             # If there are no patches left on the bug, assume land-patches already closed it before dying, and just continue.
734             log(message)
735             return
736         bug_id = non_obsolete_patches[0]['id']
737         self.tool.bugs.reject_patch_from_commit_queue(bug_id, message)
738
739     def execute(self, options, args, tool):
740         self.options = options
741         self.tool = tool
742         work_queue = WorkQueue(self)
743         work_queue.run()
744
745 class NonWrappingEpilogIndentedHelpFormatter(IndentedHelpFormatter):
746     def __init__(self):
747         IndentedHelpFormatter.__init__(self)
748
749     # The standard IndentedHelpFormatter paragraph-wraps the epilog, killing our custom formatting.
750     def format_epilog(self, epilog):
751         if epilog:
752             return "\n" + epilog + "\n"
753         return ""
754
755
756 class HelpPrintingOptionParser(OptionParser):
757     def error(self, msg):
758         self.print_usage(sys.stderr)
759         error_message = "%s: error: %s\n" % (self.get_prog_name(), msg)
760         error_message += "\nType '" + self.get_prog_name() + " --help' to see usage.\n"
761         self.exit(2, error_message)
762
763
764 class BugzillaTool:
765     def __init__(self):
766         self.cached_scm = None
767         self.bugs = Bugzilla()
768         self.buildbot = BuildBot()
769         self.commands = [
770             { 'name' : 'bugs-to-commit', 'object' : BugsInCommitQueue() },
771             { 'name' : 'patches-to-commit', 'object' : PatchesInCommitQueue() },
772             { 'name' : 'reviewed-patches', 'object' : ReviewedPatchesOnBug() },
773             { 'name' : 'create-bug', 'object' : CreateBug() },
774             { 'name' : 'apply-patches', 'object' : ApplyPatchesFromBug() },
775             { 'name' : 'land-diff', 'object' : LandAndUpdateBug() },
776             { 'name' : 'land-patches', 'object' : LandPatchesFromBugs() },
777             { 'name' : 'check-style', 'object' : CheckStyleOnBug() },
778             { 'name' : 'commit-message', 'object' : CommitMessageForCurrentDiff() },
779             { 'name' : 'obsolete-attachments', 'object' : ObsoleteAttachmentsOnBug() },
780             { 'name' : 'post-diff', 'object' : PostDiffAsPatchToBug() },
781             { 'name' : 'post-commits', 'object' : PostCommitsAsPatchesToBug() },
782             { 'name' : 'tree-status', 'object' : CheckTreeStatus() },
783             { 'name' : 'commit-queue', 'object' : LandPatchesFromCommitQueue() },
784             { 'name' : 'rollout', 'object' : RolloutCommit() },
785         ]
786
787         self.global_option_parser = HelpPrintingOptionParser(usage=self.usage_line(), formatter=NonWrappingEpilogIndentedHelpFormatter(), epilog=self.commands_usage())
788         self.global_option_parser.add_option("--dry-run", action="store_true", dest="dryrun", help="do not touch remote servers", default=False)
789
790     def scm(self):
791         # Lazily initialize SCM to not error-out before command line parsing (or when running non-scm commands).
792         original_cwd = os.path.abspath('.')
793         if not self.cached_scm:
794             self.cached_scm = detect_scm_system(original_cwd)
795         
796         if not self.cached_scm:
797             script_directory = os.path.abspath(sys.path[0])
798             webkit_directory = os.path.abspath(os.path.join(script_directory, "../.."))
799             self.cached_scm = detect_scm_system(webkit_directory)
800             if self.cached_scm:
801                 log("The current directory (%s) is not a WebKit checkout, using %s" % (original_cwd, webkit_directory))
802             else:
803                 error("FATAL: Failed to determine the SCM system for either %s or %s" % (original_cwd, webkit_directory))
804         
805         return self.cached_scm
806
807     @staticmethod
808     def usage_line():
809         return "Usage: %prog [options] command [command-options] [command-arguments]"
810
811     def commands_usage(self):
812         commands_text = "Commands:\n"
813         longest_name_length = 0
814         command_rows = []
815         scm_supports_local_commits = self.scm().supports_local_commits()
816         for command in self.commands:
817             command_object = command['object']
818             if command_object.requires_local_commits and not scm_supports_local_commits:
819                 continue
820             command_name_and_args = command_object.name_with_arguments(command['name'])
821             command_rows.append({ 'name-and-args': command_name_and_args, 'object': command_object })
822             longest_name_length = max([longest_name_length, len(command_name_and_args)])
823         
824         # Use our own help formatter so as to indent enough.
825         formatter = IndentedHelpFormatter()
826         formatter.indent()
827         formatter.indent()
828         
829         for row in command_rows:
830             command_object = row['object']
831             commands_text += "  " + row['name-and-args'].ljust(longest_name_length + 3) + command_object.help_text + "\n"
832             commands_text += command_object.option_parser.format_option_help(formatter)
833         return commands_text
834
835     def handle_global_args(self, args):
836         (options, args) = self.global_option_parser.parse_args(args)
837         if len(args):
838             # We'll never hit this because split_args splits at the first arg without a leading '-'
839             self.global_option_parser.error("Extra arguments before command: " + args)
840         
841         if options.dryrun:
842             self.scm().dryrun = True
843             self.bugs.dryrun = True
844     
845     @staticmethod
846     def split_args(args):
847         # Assume the first argument which doesn't start with '-' is the command name.
848         command_index = 0
849         for arg in args:
850             if arg[0] != '-':
851                 break
852             command_index += 1
853         else:
854             return (args[:], None, [])
855
856         global_args = args[:command_index]
857         command = args[command_index]
858         command_args = args[command_index + 1:]
859         return (global_args, command, command_args)
860     
861     def command_by_name(self, command_name):
862         for command in self.commands:
863             if command_name == command['name']:
864                 return command
865         return None
866     
867     def main(self):
868         (global_args, command_name, args_after_command_name) = self.split_args(sys.argv[1:])
869         
870         # Handle --help, etc:
871         self.handle_global_args(global_args)
872         
873         if not command_name:
874             self.global_option_parser.error("No command specified")
875         
876         command = self.command_by_name(command_name)
877         if not command:
878             self.global_option_parser.error(command_name + " is not a recognized command")
879
880         command_object = command['object']
881
882         if command_object.requires_local_commits and not self.scm().supports_local_commits():
883             error(command_name + " requires local commits using %s in %s." % (self.scm().display_name(), self.scm().checkout_root))
884
885         (command_options, command_args) = command_object.parse_args(args_after_command_name)
886         return command_object.execute(command_options, command_args, self)
887
888
889 def main():
890     tool = BugzillaTool()
891     return tool.main()
892
893 if __name__ == "__main__":
894     main()