2009-06-24 Eric Seidel <eric@webkit.org>
[WebKit-https.git] / WebKitTools / Scripts / bugzilla-tool
1 #!/usr/bin/python
2 # Copyright (c) 2009, Google Inc. All rights reserved.
3 #
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
6 # met:
7
8 #     * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 #     * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
13 # distribution.
14 #     * Neither the name of Google Inc. nor the names of its
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
17
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29 #
30 # A tool for automating dealing with bugzilla, posting patches, committing patches, etc.
31
32 import os
33 import re
34 import subprocess
35 import sys
36
37 from optparse import OptionParser, IndentedHelpFormatter, SUPPRESS_USAGE, make_option
38
39 # Import WebKit-specific modules.
40 from modules.bugzilla import Bugzilla
41 from modules.scm import detect_scm_system, ScriptError
42
43 def log(string):
44     print >> sys.stderr, string
45
46 def error(string):
47     log(string)
48     exit(1)
49
50 def plural(noun):
51     # This is a dumb plural() implementation which was just enough for our uses.
52     if re.search('h$', noun):
53         return noun + 'es'
54     else:
55         return noun + 's'
56
57 def pluralize(noun, count):
58     if count != 1:
59         noun = plural(noun)
60     return "%d %s" % (count, noun)
61
62 # These could be put in some sort of changelogs.py.
63 def latest_changelog_entry(changelog_path):
64     # e.g. 2009-06-03  Eric Seidel  <eric@webkit.org>
65     changelog_date_line_regexp = re.compile('^(\d{4}-\d{2}-\d{2})' # Consume the date.
66                                   + '\s+(.+)\s+' # Consume the name.
67                                   + '<([^<>]+)>$') # And finally the email address.
68     
69     entry_lines = []
70     changelog = open(changelog_path)
71     try:
72         log("Parsing ChangeLog: " + changelog_path)
73         # The first line should be a date line.
74         first_line = changelog.readline()
75         if not changelog_date_line_regexp.match(first_line):
76             return None
77         entry_lines.append(first_line)
78         
79         for line in changelog:
80             # If we've hit the next entry, return.
81             if changelog_date_line_regexp.match(line):
82                 return ''.join(entry_lines)
83             entry_lines.append(line)
84     finally:
85         changelog.close()
86     # We never found a date line!
87     return None
88
89 def modified_changelogs(scm):
90     changelog_paths = []
91     paths = scm.changed_files()
92     for path in paths:
93         if os.path.basename(path) == "ChangeLog":
94             changelog_paths.append(path)
95     return changelog_paths
96
97 def commit_message_for_this_commit(scm):
98     changelog_paths = modified_changelogs(scm)
99     if not len(changelog_paths):
100         error("Found no modified ChangeLogs, can't create a commit message.")
101
102     changelog_messages = []
103     for path in changelog_paths:
104         changelog_entry = latest_changelog_entry(path)
105         if not changelog_entry:
106             error("Failed to parse ChangeLog: " + os.path.abspath(path))
107         changelog_messages.append(changelog_entry)
108     
109     # FIXME: We should sort and label the ChangeLog messages like commit-log-editor does.
110     return ''.join(changelog_messages)
111
112
113 class Command:
114     def __init__(self, help_text, argument_names="", options=[]):
115         self.help_text = help_text
116         self.argument_names = argument_names
117         self.options = options
118         self.option_parser = OptionParser(usage=SUPPRESS_USAGE, add_help_option=False, option_list=self.options)
119     
120     def name_with_arguments(self, command_name):
121         usage_string = command_name
122         if len(self.options) > 0:
123             usage_string += " [options]"
124         if self.argument_names:
125             usage_string += " " + self.argument_names
126         return usage_string
127
128     def parse_args(self, args):
129         return self.option_parser.parse_args(args)
130
131     def execute(self, options, args, tool):
132         raise NotImplementedError, "subclasses must implement"
133
134
135 class BugsInCommitQueue(Command):
136     def __init__(self):
137         Command.__init__(self, 'Bugs in the commit queue')
138
139     def execute(self, options, args, tool):
140         bug_ids = tool.bugs.fetch_bug_ids_from_commit_queue()
141         for bug_id in bug_ids:
142             print "%s" % tool.bugs.bug_url_for_bug_id(bug_id)
143
144
145 class PatchesInCommitQueue(Command):
146     def __init__(self):
147         Command.__init__(self, 'Patches attached to bugs in the commit queue')
148
149     def execute(self, options, args, tool):
150         patches = tool.bugs.fetch_patches_from_commit_queue()
151         log("Patches in commit queue:")
152         for patch in patches:
153             print "%s" % patch['url']
154
155
156 class ReviewedPatchesOnBug(Command):
157     def __init__(self):
158         Command.__init__(self, 'r+\'d patches on a bug', 'BUGID')
159
160     def execute(self, options, args, tool):
161         bug_id = args[0]
162         patches_to_land = tool.bugs.fetch_reviewed_patches_from_bug(bug_id)
163         for patch in patches_to_land:
164             print "%s" % patch['url']
165
166
167 class ApplyPatchesFromBug(Command):
168     def __init__(self):
169         options = [
170             make_option("--no-update", action="store_false", dest="update", default=True, help="Don't update the working directory before applying patches"),
171             make_option("--force-clean", action="store_true", dest="force_clean", default=False, help="Clean working directory before applying patches (removes local changes and commits)"),
172             make_option("--no-clean", action="store_false", dest="clean", default=True, help="Don't check if the working directory is clean before applying patches"),
173             make_option("--local-commit", action="store_true", dest="local_commit", default=False, help="Make a local commit for each applied patch"),
174         ]
175         Command.__init__(self, 'Applies all patches on a bug to the local working directory without committing.', 'BUGID', options=options)
176
177     @staticmethod
178     def apply_patches(patches, scm, commit_each):
179         for patch in patches:
180             scm.apply_patch(patch)
181             if commit_each:
182                 commit_message = commit_message_for_this_commit(scm)
183                 scm.commit_locally_with_message(commit_message or patch['name'])
184
185     def execute(self, options, args, tool):
186         bug_id = args[0]
187         patches = tool.bugs.fetch_reviewed_patches_from_bug(bug_id)
188         os.chdir(tool.scm().checkout_root)
189         if options.clean:
190             tool.scm().ensure_clean_working_directory(options.force_clean)
191         if options.update:
192             tool.scm().update_webkit()
193         
194         if options.local_commit and not tool.scm().supports_local_commits():
195             error("--local-commit passed, but %s does not support local commits" % tool.scm().display_name())
196         
197         self.apply_patches(patches, tool.scm(), options.local_commit)
198
199 def bug_comment_from_commit_text(commit_text):
200     comment_lines = []
201     commit_lines = commit_text.splitlines()
202     for line in commit_lines:
203         comment_lines.append(line)
204         match = re.match("^Committed r(\d+)$", line)
205         if match:
206             revision = match.group(1)
207             comment_lines.append("http://trac.webkit.org/changeset/" + revision)
208             break
209     return "\n".join(comment_lines)
210
211
212 class LandAndUpdateBug(Command):
213     def __init__(self):
214         Command.__init__(self, 'Lands the current working directory diff and updates the bug.', 'BUGID')
215
216     def execute(self, options, args, tool):
217         bug_id = args[0]
218         os.chdir(tool.scm().checkout_root)
219         commit_message = commit_message_for_this_commit(tool.scm())
220         commit_log = tool.scm().commit_with_message(commit_message)
221         comment_text = bug_comment_from_commit_text(commit_log)
222         tool.bugs.close_bug_as_fixed(bug_id, comment_text)
223
224
225 class LandPatchesFromBugs(Command):
226     def __init__(self):
227         options = [
228             make_option("--no-update", action="store_false", dest="update", default=True, help="Don't update the working directory before applying patches"),
229             make_option("--force-clean", action="store_true", dest="force_clean", default=False, help="Clean working directory before applying patches (removes local changes and commits)"),
230             make_option("--no-clean", action="store_false", dest="clean", default=True, help="Don't check if the working directory is clean before applying patches"),
231             make_option("--no-build", action="store_false", dest="build", default=True, help="Commit without building first, implies --no-test."),
232             make_option("--no-test", action="store_false", dest="test", default=True, help="Commit without runnning run-webkit-tests."),
233         ]
234         Command.__init__(self, 'Lands all patches on a bug optionally testing them first', 'BUGID', options=options)
235
236     @staticmethod
237     def run_and_throw_if_fail(script_name):
238         build_webkit_process = subprocess.Popen(script_name, shell=True)
239         return_code = build_webkit_process.wait()
240         if return_code:
241             raise ScriptError("%s failed with exit code %d" % (script_name, return_code))
242
243     @classmethod
244     def build_webkit(cls):
245         cls.run_and_throw_if_fail("build-webkit")
246
247     @classmethod
248     def run_webkit_tests(cls):
249         cls.run_and_throw_if_fail("run-webkit-tests")
250
251     @staticmethod
252     def setup_for_landing(scm, options):
253         os.chdir(scm.checkout_root)
254         scm.ensure_no_local_commits(options.force_clean)
255         if options.clean:
256             scm.ensure_clean_working_directory(options.force_clean)
257         if options.update:
258             scm.update_webkit()
259
260     @classmethod
261     def build_and_commit(cls, scm, options):
262         if options.build:
263             cls.build_webkit()
264             if options.test:
265                 cls.run_webkit_tests()
266         commit_message = commit_message_for_this_commit(scm)
267         commit_log = scm.commit_with_message(commit_message)
268         return bug_comment_from_commit_text(commit_log)
269
270     @classmethod
271     def land_patches(cls, bug_id, patches, options, tool):
272         try:
273             comment_text = ""
274             for patch in patches:
275                 tool.scm().apply_patch(patch)
276                 comment_text = cls.build_and_commit(tool.scm(), options)
277
278                 # If we're commiting more than one patch, update the bug as we go.
279                 if len(patches) > 1:
280                     tool.bugs.obsolete_attachment(patch['id'], comment_text)
281
282             if len(patches) > 1:
283                 comment_text = "All reviewed patches landed, closing."
284
285             tool.bugs.close_bug_as_fixed(bug_id, comment_text)
286         except ScriptError, e:
287             # We should add a comment to the bug, and r- the patch on failure
288             error(e)
289
290     def execute(self, options, args, tool):
291         if not len(args):
292             error("bug-id(s) required")
293
294         bugs_to_patches = {}
295         patch_count = 0
296         for bug_id in args:
297             patches = tool.bugs.fetch_reviewed_patches_from_bug(bug_id)
298             if not len(patches):
299                 exit("No reviewed patches found on %s" % bug_id)
300             patch_count += len(patches)
301             bugs_to_patches[bug_id] = patches
302
303         log("Landing %s from %s." % (pluralize("patch", patch_count), pluralize("bug", len(args))))
304         
305         self.setup_for_landing(tool.scm(), options)
306
307         for bug_id in args:
308             self.land_patches(bug_id, bugs_to_patches[bug_id], options, tool)
309
310 class CommitMessageForCurrentDiff(Command):
311     def __init__(self):
312         Command.__init__(self, 'Prints a commit message suitable for the uncommitted changes.')
313
314     def execute(self, options, args, tool):
315         os.chdir(tool.scm().checkout_root)
316         print "%s" % commit_message_for_this_commit(tool.scm())
317
318
319 class ObsoleteAttachmentsOnBug(Command):
320     def __init__(self):
321         Command.__init__(self, 'Marks all attachments on a bug as obsolete.', 'BUGID')
322
323     def execute(self, options, args, tool):
324         bug_id = args[0]
325         attachments = tool.bugs.fetch_attachments_from_bug(bug_id)
326         for attachment in attachments:
327             if not attachment['obsolete']:
328                 tool.bugs.obsolete_attachment(attachment['id'])
329
330
331 class PostDiffAsPatchToBug(Command):
332     def __init__(self):
333         options = [
334             make_option("--no-review", action="store_false", dest="review", default=True, help="Do not mark the patch for review."),
335             make_option("-m", "--description", action="store", type="string", dest="description", help="Description string for the attachment (default: 'patch')"),
336         ]
337         Command.__init__(self, 'Attaches the current working directory diff to a bug as a patch file.', 'BUGID', options=options)
338
339     def execute(self, options, args, tool):
340         bug_id = args[0]
341         
342         diff_process = subprocess.Popen(tool.scm().create_patch_command(), stdout=subprocess.PIPE, shell=True)
343         diff_process.wait() # Make sure svn-create-patch is done before we continue.
344         
345         description = options.description or "patch"
346         tool.bugs.add_patch_to_bug(bug_id, diff_process.stdout, description, mark_for_review=options.review)
347
348
349 class PostCommitsAsPatchesToBug(Command):
350     def __init__(self):
351         options = [
352             make_option("--no-review", action="store_false", dest="review", default=True, help="Do not mark the patch for review."),
353         ]
354         Command.__init__(self, 'Attaches a range of local commits to a bug as patch files.', 'BUGID COMMITISH', options=options)
355
356     def execute(self, options, args, tool):
357         bug_id = args[0]
358         
359         if not tool.scm().supports_local_commits():
360             log(tool.scm().display_name() + " does not support local commits.")
361             exit(1)
362         
363         commit_ids = tool.scm().commit_ids_from_range_arguments(args[1:])
364         
365         if len(commit_ids) > 10:
366             log("Are you sure you want to attach %s to bug %s?" % (pluralize('patch', len(commit_ids)), bug_id))
367             # Could add a --patches-limit option.
368             exit(1)
369         
370         log("Attaching %s as patches to bug %s" % (pluralize('commit', len(commit_ids)), bug_id))
371         for commit_id in commit_ids:
372             commit_message = tool.scm().commit_message_for_commit(commit_id)
373             commit_lines = commit_message.splitlines()
374             
375             description = commit_lines[0]
376             comment_text = "\n".join(commit_lines[1:])
377             comment_text += "---\n"
378             comment_text += tool.scm().files_changed_summary_for_commit(commit_id)
379         
380             # This is a little bit of a hack, that we pass stdout as the patch file.
381             # We could alternatively make an in-memory file-like object with the patch contents.
382             diff_process = subprocess.Popen(tool.scm().show_diff_command_for_commit(commit_id), stdout=subprocess.PIPE, shell=True)
383             tool.bugs.add_patch_to_bug(bug_id, diff_process.stdout, description, comment_text, mark_for_review=options.review)
384
385
386 class NonWrappingEpilogIndentedHelpFormatter(IndentedHelpFormatter):
387     def __init__(self):
388         IndentedHelpFormatter.__init__(self)
389
390     # The standard IndentedHelpFormatter paragraph-wraps the epilog, killing our custom formatting.
391     def format_epilog(self, epilog):
392         if epilog:
393             return "\n" + epilog + "\n"
394         return ""
395
396 class BugzillaTool:
397     def __init__(self):
398         self.cached_scm = None
399         self.bugs = Bugzilla()
400         self.commands = [
401             { 'name' : 'bugs-to-commit', 'object' : BugsInCommitQueue() },
402             { 'name' : 'patches-to-commit', 'object' : PatchesInCommitQueue() },
403             { 'name' : 'reviewed-patches', 'object' : ReviewedPatchesOnBug() },
404             { 'name' : 'apply-patches', 'object' : ApplyPatchesFromBug() },
405             { 'name' : 'land-and-update', 'object' : LandAndUpdateBug() },
406             { 'name' : 'land-patches', 'object' : LandPatchesFromBugs() },
407             { 'name' : 'commit-message', 'object' : CommitMessageForCurrentDiff() },
408             { 'name' : 'obsolete-attachments', 'object' : ObsoleteAttachmentsOnBug() },
409             { 'name' : 'post-diff', 'object' : PostDiffAsPatchToBug() },
410             { 'name' : 'post-commits', 'object' : PostCommitsAsPatchesToBug() },
411         ]
412         
413         self.global_option_parser = OptionParser(usage=self.usage_line(), formatter=NonWrappingEpilogIndentedHelpFormatter(), epilog=self.commands_usage())
414         self.global_option_parser.add_option("--dry-run", action="store_true", dest="dryrun", help="do not touch remote servers", default=False)
415     
416     def scm(self):
417         # Lazily initialize SCM to not error-out before command line parsing (or when running non-scm commands).
418         original_cwd = os.path.abspath('.')
419         if not self.cached_scm:
420             self.cached_scm = detect_scm_system(original_cwd)
421         
422         if not self.cached_scm:
423             script_directory = os.path.abspath(sys.path[0])
424             webkit_directory = os.path.abspath(os.path.join(script_directory, "../.."))
425             self.cached_scm = detect_scm_system(webkit_directory)
426             if self.cached_scm:
427                 log("The current directory (%s) is not a WebKit checkout, using %s" % (original_cwd, webkit_directory))
428             else:
429                 error("FATAL: Failed to determine the SCM system for either %s or %s" % (original_cwd, webkit_directory))
430         
431         return self.cached_scm
432     
433     @staticmethod
434     def usage_line():
435         return "Usage: %prog [options] command [command-options] [command-arguments]"
436     
437     def commands_usage(self):
438         commands_text = "Commands:\n"
439         longest_name_length = 0
440         command_rows = []
441         for command in self.commands:
442             command_object = command['object']
443             command_name_and_args = command_object.name_with_arguments(command['name'])
444             command_rows.append({ 'name-and-args': command_name_and_args, 'object': command_object })
445             longest_name_length = max([longest_name_length, len(command_name_and_args)])
446         
447         # Use our own help formatter so as to indent enough.
448         formatter = IndentedHelpFormatter()
449         formatter.indent()
450         formatter.indent()
451         
452         for row in command_rows:
453             command_object = row['object']
454             commands_text += "  " + row['name-and-args'].ljust(longest_name_length + 3) + command_object.help_text + "\n"
455             commands_text += command_object.option_parser.format_option_help(formatter)
456         return commands_text
457
458     def handle_global_args(self, args):
459         (options, args) = self.global_option_parser.parse_args(args)
460         if len(args):
461             # We'll never hit this because split_args splits at the first arg without a leading '-'
462             self.global_option_parser.error("Extra arguments before command: " + args)
463         
464         if options.dryrun:
465             self.scm().dryrun = True
466             self.bugs.dryrun = True
467     
468     @staticmethod
469     def split_args(args):
470         # Assume the first argument which doesn't start with '-' is the command name.
471         command_index = 0
472         for arg in args:
473             if arg[0] != '-':
474                 break
475             command_index += 1
476         else:
477             return (args[:], None, [])
478
479         global_args = args[:command_index]
480         command = args[command_index]
481         command_args = args[command_index + 1:]
482         return (global_args, command, command_args)
483     
484     def command_by_name(self, command_name):
485         for command in self.commands:
486             if command_name == command['name']:
487                 return command
488         return None
489     
490     def main(self):
491         (global_args, command_name, args_after_command_name) = self.split_args(sys.argv[1:])
492         
493         # Handle --help, etc:
494         self.handle_global_args(global_args)
495         
496         if not command_name:
497             self.global_option_parser.error("No command specified")
498         
499         command = self.command_by_name(command_name)
500         if not command:
501             self.global_option_parser.error(command_name + " is not a recognized command")
502         
503         command_object = command['object']
504         (command_options, command_args) = command_object.parse_args(args_after_command_name)
505         return command_object.execute(command_options, command_args, self)
506
507
508 def main():
509     tool = BugzillaTool()
510     return tool.main()
511
512 if __name__ == "__main__":
513     main()