2009-06-26 Eric Seidel <eric@webkit.org>
[WebKit-https.git] / WebKitTools / Scripts / bugzilla-tool
1 #!/usr/bin/python
2 # Copyright (c) 2009, Google Inc. All rights reserved.
3 #
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
6 # met:
7
8 #     * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 #     * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
13 # distribution.
14 #     * Neither the name of Google Inc. nor the names of its
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
17
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29 #
30 # A tool for automating dealing with bugzilla, posting patches, committing patches, etc.
31
32 import os
33 import re
34 import subprocess
35 import sys
36
37 from optparse import OptionParser, IndentedHelpFormatter, SUPPRESS_USAGE, make_option
38
39 # Import WebKit-specific modules.
40 from modules.bugzilla import Bugzilla
41 from modules.scm import detect_scm_system, ScriptError
42
43 def log(string):
44     print >> sys.stderr, string
45
46 def error(string):
47     log(string)
48     exit(1)
49
50 def plural(noun):
51     # This is a dumb plural() implementation which was just enough for our uses.
52     if re.search('h$', noun):
53         return noun + 'es'
54     else:
55         return noun + 's'
56
57 def pluralize(noun, count):
58     if count != 1:
59         noun = plural(noun)
60     return "%d %s" % (count, noun)
61
62 # These could be put in some sort of changelogs.py.
63 def latest_changelog_entry(changelog_path):
64     # e.g. 2009-06-03  Eric Seidel  <eric@webkit.org>
65     changelog_date_line_regexp = re.compile('^(\d{4}-\d{2}-\d{2})' # Consume the date.
66                                   + '\s+(.+)\s+' # Consume the name.
67                                   + '<([^<>]+)>$') # And finally the email address.
68     
69     entry_lines = []
70     changelog = open(changelog_path)
71     try:
72         log("Parsing ChangeLog: " + changelog_path)
73         # The first line should be a date line.
74         first_line = changelog.readline()
75         if not changelog_date_line_regexp.match(first_line):
76             return None
77         entry_lines.append(first_line)
78         
79         for line in changelog:
80             # If we've hit the next entry, return.
81             if changelog_date_line_regexp.match(line):
82                 return ''.join(entry_lines)
83             entry_lines.append(line)
84     finally:
85         changelog.close()
86     # We never found a date line!
87     return None
88
89 def modified_changelogs(scm):
90     changelog_paths = []
91     paths = scm.changed_files()
92     for path in paths:
93         if os.path.basename(path) == "ChangeLog":
94             changelog_paths.append(path)
95     return changelog_paths
96
97 def commit_message_for_this_commit(scm):
98     changelog_paths = modified_changelogs(scm)
99     if not len(changelog_paths):
100         error("Found no modified ChangeLogs, can't create a commit message.")
101
102     changelog_messages = []
103     for path in changelog_paths:
104         changelog_entry = latest_changelog_entry(path)
105         if not changelog_entry:
106             error("Failed to parse ChangeLog: " + os.path.abspath(path))
107         changelog_messages.append(changelog_entry)
108     
109     # FIXME: We should sort and label the ChangeLog messages like commit-log-editor does.
110     return ''.join(changelog_messages)
111
112
113 class Command:
114     def __init__(self, help_text, argument_names="", options=[]):
115         self.help_text = help_text
116         self.argument_names = argument_names
117         self.options = options
118         self.option_parser = OptionParser(usage=SUPPRESS_USAGE, add_help_option=False, option_list=self.options)
119     
120     def name_with_arguments(self, command_name):
121         usage_string = command_name
122         if len(self.options) > 0:
123             usage_string += " [options]"
124         if self.argument_names:
125             usage_string += " " + self.argument_names
126         return usage_string
127
128     def parse_args(self, args):
129         return self.option_parser.parse_args(args)
130
131     def execute(self, options, args, tool):
132         raise NotImplementedError, "subclasses must implement"
133
134
135 class BugsInCommitQueue(Command):
136     def __init__(self):
137         Command.__init__(self, 'Bugs in the commit queue')
138
139     def execute(self, options, args, tool):
140         bug_ids = tool.bugs.fetch_bug_ids_from_commit_queue()
141         for bug_id in bug_ids:
142             print "%s" % tool.bugs.bug_url_for_bug_id(bug_id)
143
144
145 class PatchesInCommitQueue(Command):
146     def __init__(self):
147         Command.__init__(self, 'Patches attached to bugs in the commit queue')
148
149     def execute(self, options, args, tool):
150         patches = tool.bugs.fetch_patches_from_commit_queue()
151         log("Patches in commit queue:")
152         for patch in patches:
153             print "%s" % patch['url']
154
155
156 class ReviewedPatchesOnBug(Command):
157     def __init__(self):
158         Command.__init__(self, 'r+\'d patches on a bug', 'BUGID')
159
160     def execute(self, options, args, tool):
161         bug_id = args[0]
162         patches_to_land = tool.bugs.fetch_reviewed_patches_from_bug(bug_id)
163         for patch in patches_to_land:
164             print "%s" % patch['url']
165
166
167 class ApplyPatchesFromBug(Command):
168     def __init__(self):
169         options = [
170             make_option("--no-update", action="store_false", dest="update", default=True, help="Don't update the working directory before applying patches"),
171             make_option("--force-clean", action="store_true", dest="force_clean", default=False, help="Clean working directory before applying patches (removes local changes and commits)"),
172             make_option("--no-clean", action="store_false", dest="clean", default=True, help="Don't check if the working directory is clean before applying patches"),
173             make_option("--local-commit", action="store_true", dest="local_commit", default=False, help="Make a local commit for each applied patch"),
174         ]
175         Command.__init__(self, 'Applies all patches on a bug to the local working directory without committing.', 'BUGID', options=options)
176
177     @staticmethod
178     def apply_patches(patches, scm, commit_each):
179         for patch in patches:
180             scm.apply_patch(patch)
181             if commit_each:
182                 commit_message = commit_message_for_this_commit(scm)
183                 scm.commit_locally_with_message(commit_message or patch['name'])
184
185     def execute(self, options, args, tool):
186         bug_id = args[0]
187         patches = tool.bugs.fetch_reviewed_patches_from_bug(bug_id)
188         os.chdir(tool.scm().checkout_root)
189         if options.clean:
190             tool.scm().ensure_clean_working_directory(options.force_clean)
191         if options.update:
192             tool.scm().update_webkit()
193         
194         if options.local_commit and not tool.scm().supports_local_commits():
195             error("--local-commit passed, but %s does not support local commits" % tool.scm().display_name())
196         
197         self.apply_patches(patches, tool.scm(), options.local_commit)
198
199 def bug_comment_from_commit_text(commit_text):
200     comment_lines = []
201     commit_lines = commit_text.splitlines()
202     for line in commit_lines:
203         comment_lines.append(line)
204         match = re.match("^Committed r(\d+)$", line)
205         if match:
206             revision = match.group(1)
207             comment_lines.append("http://trac.webkit.org/changeset/" + revision)
208             break
209     return "\n".join(comment_lines)
210
211
212 class LandAndUpdateBug(Command):
213     def __init__(self):
214         Command.__init__(self, 'Lands the current working directory diff and updates the bug.', 'BUGID')
215
216     def execute(self, options, args, tool):
217         bug_id = args[0]
218         os.chdir(tool.scm().checkout_root)
219         commit_message = commit_message_for_this_commit(tool.scm())
220         commit_log = tool.scm().commit_with_message(commit_message)
221         comment_text = bug_comment_from_commit_text(commit_log)
222         tool.bugs.close_bug_as_fixed(bug_id, comment_text)
223
224
225 class LandPatchesFromBugs(Command):
226     def __init__(self):
227         options = [
228             make_option("--force-clean", action="store_true", dest="force_clean", default=False, help="Clean working directory before applying patches (removes local changes and commits)"),
229             make_option("--no-clean", action="store_false", dest="clean", default=True, help="Don't check if the working directory is clean before applying patches"),
230             make_option("--no-build", action="store_false", dest="build", default=True, help="Commit without building first, implies --no-test."),
231             make_option("--no-test", action="store_false", dest="test", default=True, help="Commit without runnning run-webkit-tests."),
232         ]
233         Command.__init__(self, 'Lands all patches on a bug optionally testing them first', 'BUGID', options=options)
234
235     @staticmethod
236     def run_and_throw_if_fail(script_name):
237         build_webkit_process = subprocess.Popen(script_name)
238         return_code = build_webkit_process.wait()
239         if return_code:
240             raise ScriptError("%s failed with exit code %d" % (script_name, return_code))
241
242     @classmethod
243     def run_webkit_script(cls, script_name):
244         # We might need to pass scm into this function for scm.checkout_root
245         cls.run_and_throw_if_fail(os.path.join("WebKitTools", "Scripts", script_name))
246
247     @classmethod
248     def build_webkit(cls):
249         cls.run_webkit_script("build-webkit")
250
251     @classmethod
252     def run_webkit_tests(cls):
253         cls.run_webkit_script("run-webkit-tests")
254
255     @staticmethod
256     def setup_for_landing(scm, options):
257         os.chdir(scm.checkout_root)
258         scm.ensure_no_local_commits(options.force_clean)
259         if options.clean:
260             scm.ensure_clean_working_directory(options.force_clean)
261
262     @classmethod
263     def build_and_commit(cls, scm, options):
264         if options.build:
265             cls.build_webkit()
266             if options.test:
267                 cls.run_webkit_tests()
268         commit_message = commit_message_for_this_commit(scm)
269         commit_log = scm.commit_with_message(commit_message)
270         return bug_comment_from_commit_text(commit_log)
271
272     @classmethod
273     def land_patches(cls, bug_id, patches, options, tool):
274         try:
275             comment_text = ""
276             for patch in patches:
277                 tool.scm().update_webkit() # Update before every patch in case the tree has changed
278                 tool.scm().apply_patch(patch)
279                 comment_text = cls.build_and_commit(tool.scm(), options)
280
281                 # If we're commiting more than one patch, update the bug as we go.
282                 if len(patches) > 1:
283                     tool.bugs.obsolete_attachment(patch['id'], comment_text)
284
285             if len(patches) > 1:
286                 comment_text = "All reviewed patches landed, closing."
287
288             tool.bugs.close_bug_as_fixed(bug_id, comment_text)
289         except ScriptError, e:
290             # We should add a comment to the bug, and r- the patch on failure
291             error(e)
292
293     def execute(self, options, args, tool):
294         if not len(args):
295             error("bug-id(s) required")
296
297         bugs_to_patches = {}
298         patch_count = 0
299         for bug_id in args:
300             patches = tool.bugs.fetch_reviewed_patches_from_bug(bug_id)
301             if not len(patches):
302                 exit("No reviewed patches found on %s" % bug_id)
303             patch_count += len(patches)
304             bugs_to_patches[bug_id] = patches
305
306         log("Landing %s from %s." % (pluralize("patch", patch_count), pluralize("bug", len(args))))
307         
308         self.setup_for_landing(tool.scm(), options)
309
310         for bug_id in args:
311             self.land_patches(bug_id, bugs_to_patches[bug_id], options, tool)
312
313 class CommitMessageForCurrentDiff(Command):
314     def __init__(self):
315         Command.__init__(self, 'Prints a commit message suitable for the uncommitted changes.')
316
317     def execute(self, options, args, tool):
318         os.chdir(tool.scm().checkout_root)
319         print "%s" % commit_message_for_this_commit(tool.scm())
320
321
322 class ObsoleteAttachmentsOnBug(Command):
323     def __init__(self):
324         Command.__init__(self, 'Marks all attachments on a bug as obsolete.', 'BUGID')
325
326     def execute(self, options, args, tool):
327         bug_id = args[0]
328         attachments = tool.bugs.fetch_attachments_from_bug(bug_id)
329         for attachment in attachments:
330             if not attachment['is_obsolete']:
331                 tool.bugs.obsolete_attachment(attachment['id'])
332
333
334 class PostDiffAsPatchToBug(Command):
335     def __init__(self):
336         options = [
337             make_option("--no-obsolete", action="store_false", dest="obsolete_patches", default=True, help="Do not obsolete old patches before posting this one."),
338             make_option("--no-review", action="store_false", dest="review", default=True, help="Do not mark the patch for review."),
339             make_option("-m", "--description", action="store", type="string", dest="description", help="Description string for the attachment (default: 'patch')"),
340         ]
341         Command.__init__(self, 'Attaches the current working directory diff to a bug as a patch file.', 'BUGID', options=options)
342
343     @staticmethod
344     def obsolete_patches_on_bug(bug_id, bugs):
345         patches = bugs.fetch_patches_from_bug(bug_id)
346         if len(patches):
347             log("Obsoleting %s on bug %s" % (pluralize('old patch', len(patches)), bug_id))
348             for patch in patches:
349                 bugs.obsolete_attachment(patch['id'])
350
351     def execute(self, options, args, tool):
352         bug_id = args[0]
353
354         if options.obsolete_patches:
355             self.obsolete_patches_on_bug(bug_id, tool.bugs)
356
357         diff_process = subprocess.Popen(tool.scm().create_patch_command(), stdout=subprocess.PIPE, shell=True)
358         diff_process.wait() # Make sure svn-create-patch is done before we continue.
359         
360         description = options.description or "patch"
361         tool.bugs.add_patch_to_bug(bug_id, diff_process.stdout, description, mark_for_review=options.review)
362
363
364 class PostCommitsAsPatchesToBug(Command):
365     def __init__(self):
366         options = [
367             make_option("--no-obsolete", action="store_false", dest="obsolete_patches", default=True, help="Do not obsolete old patches before posting new ones."),
368             make_option("--no-review", action="store_false", dest="review", default=True, help="Do not mark the patch for review."),
369         ]
370         Command.__init__(self, 'Attaches a range of local commits to a bug as patch files.', 'BUGID COMMITISH', options=options)
371
372     def execute(self, options, args, tool):
373         bug_id = args[0]
374         
375         if not tool.scm().supports_local_commits():
376             error(tool.scm().display_name() + " does not support local commits.")
377         
378         commit_ids = tool.scm().commit_ids_from_range_arguments(args[1:])
379         
380         if len(commit_ids) > 10:
381             error("Are you sure you want to attach %s to bug %s?" % (pluralize('patch', len(commit_ids)), bug_id))
382             # Could add a --patches-limit option.
383
384         if options.obsolete_patches:
385             PostDiffAsPatchToBug.obsolete_patches_on_bug(bug_id, tool.bugs)
386         
387         log("Attaching %s as patches to bug %s" % (pluralize('commit', len(commit_ids)), bug_id))
388         for commit_id in commit_ids:
389             commit_message = tool.scm().commit_message_for_commit(commit_id)
390             commit_lines = commit_message.splitlines()
391             
392             description = commit_lines[0]
393             comment_text = "\n".join(commit_lines[1:])
394             comment_text += "---\n"
395             comment_text += tool.scm().files_changed_summary_for_commit(commit_id)
396         
397             # This is a little bit of a hack, that we pass stdout as the patch file.
398             # We could alternatively make an in-memory file-like object with the patch contents.
399             diff_process = subprocess.Popen(tool.scm().show_diff_command_for_commit(commit_id), stdout=subprocess.PIPE, shell=True)
400             tool.bugs.add_patch_to_bug(bug_id, diff_process.stdout, description, comment_text, mark_for_review=options.review)
401
402
403 class NonWrappingEpilogIndentedHelpFormatter(IndentedHelpFormatter):
404     def __init__(self):
405         IndentedHelpFormatter.__init__(self)
406
407     # The standard IndentedHelpFormatter paragraph-wraps the epilog, killing our custom formatting.
408     def format_epilog(self, epilog):
409         if epilog:
410             return "\n" + epilog + "\n"
411         return ""
412
413 class BugzillaTool:
414     def __init__(self):
415         self.cached_scm = None
416         self.bugs = Bugzilla()
417         self.commands = [
418             { 'name' : 'bugs-to-commit', 'object' : BugsInCommitQueue() },
419             { 'name' : 'patches-to-commit', 'object' : PatchesInCommitQueue() },
420             { 'name' : 'reviewed-patches', 'object' : ReviewedPatchesOnBug() },
421             { 'name' : 'apply-patches', 'object' : ApplyPatchesFromBug() },
422             { 'name' : 'land-and-update', 'object' : LandAndUpdateBug() },
423             { 'name' : 'land-patches', 'object' : LandPatchesFromBugs() },
424             { 'name' : 'commit-message', 'object' : CommitMessageForCurrentDiff() },
425             { 'name' : 'obsolete-attachments', 'object' : ObsoleteAttachmentsOnBug() },
426             { 'name' : 'post-diff', 'object' : PostDiffAsPatchToBug() },
427             { 'name' : 'post-commits', 'object' : PostCommitsAsPatchesToBug() },
428         ]
429         
430         self.global_option_parser = OptionParser(usage=self.usage_line(), formatter=NonWrappingEpilogIndentedHelpFormatter(), epilog=self.commands_usage())
431         self.global_option_parser.add_option("--dry-run", action="store_true", dest="dryrun", help="do not touch remote servers", default=False)
432     
433     def scm(self):
434         # Lazily initialize SCM to not error-out before command line parsing (or when running non-scm commands).
435         original_cwd = os.path.abspath('.')
436         if not self.cached_scm:
437             self.cached_scm = detect_scm_system(original_cwd)
438         
439         if not self.cached_scm:
440             script_directory = os.path.abspath(sys.path[0])
441             webkit_directory = os.path.abspath(os.path.join(script_directory, "../.."))
442             self.cached_scm = detect_scm_system(webkit_directory)
443             if self.cached_scm:
444                 log("The current directory (%s) is not a WebKit checkout, using %s" % (original_cwd, webkit_directory))
445             else:
446                 error("FATAL: Failed to determine the SCM system for either %s or %s" % (original_cwd, webkit_directory))
447         
448         return self.cached_scm
449     
450     @staticmethod
451     def usage_line():
452         return "Usage: %prog [options] command [command-options] [command-arguments]"
453     
454     def commands_usage(self):
455         commands_text = "Commands:\n"
456         longest_name_length = 0
457         command_rows = []
458         for command in self.commands:
459             command_object = command['object']
460             command_name_and_args = command_object.name_with_arguments(command['name'])
461             command_rows.append({ 'name-and-args': command_name_and_args, 'object': command_object })
462             longest_name_length = max([longest_name_length, len(command_name_and_args)])
463         
464         # Use our own help formatter so as to indent enough.
465         formatter = IndentedHelpFormatter()
466         formatter.indent()
467         formatter.indent()
468         
469         for row in command_rows:
470             command_object = row['object']
471             commands_text += "  " + row['name-and-args'].ljust(longest_name_length + 3) + command_object.help_text + "\n"
472             commands_text += command_object.option_parser.format_option_help(formatter)
473         return commands_text
474
475     def handle_global_args(self, args):
476         (options, args) = self.global_option_parser.parse_args(args)
477         if len(args):
478             # We'll never hit this because split_args splits at the first arg without a leading '-'
479             self.global_option_parser.error("Extra arguments before command: " + args)
480         
481         if options.dryrun:
482             self.scm().dryrun = True
483             self.bugs.dryrun = True
484     
485     @staticmethod
486     def split_args(args):
487         # Assume the first argument which doesn't start with '-' is the command name.
488         command_index = 0
489         for arg in args:
490             if arg[0] != '-':
491                 break
492             command_index += 1
493         else:
494             return (args[:], None, [])
495
496         global_args = args[:command_index]
497         command = args[command_index]
498         command_args = args[command_index + 1:]
499         return (global_args, command, command_args)
500     
501     def command_by_name(self, command_name):
502         for command in self.commands:
503             if command_name == command['name']:
504                 return command
505         return None
506     
507     def main(self):
508         (global_args, command_name, args_after_command_name) = self.split_args(sys.argv[1:])
509         
510         # Handle --help, etc:
511         self.handle_global_args(global_args)
512         
513         if not command_name:
514             self.global_option_parser.error("No command specified")
515         
516         command = self.command_by_name(command_name)
517         if not command:
518             self.global_option_parser.error(command_name + " is not a recognized command")
519         
520         command_object = command['object']
521         (command_options, command_args) = command_object.parse_args(args_after_command_name)
522         return command_object.execute(command_options, command_args, self)
523
524
525 def main():
526     tool = BugzillaTool()
527     return tool.main()
528
529 if __name__ == "__main__":
530     main()