2009-06-25 Eric Seidel <eric@webkit.org>
[WebKit-https.git] / WebKitTools / Scripts / bugzilla-tool
1 #!/usr/bin/python
2 # Copyright (c) 2009, Google Inc. All rights reserved.
3 #
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
6 # met:
7
8 #     * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 #     * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
13 # distribution.
14 #     * Neither the name of Google Inc. nor the names of its
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
17
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29 #
30 # A tool for automating dealing with bugzilla, posting patches, committing patches, etc.
31
32 import os
33 import re
34 import subprocess
35 import sys
36
37 from optparse import OptionParser, IndentedHelpFormatter, SUPPRESS_USAGE, make_option
38
39 # Import WebKit-specific modules.
40 from modules.bugzilla import Bugzilla
41 from modules.scm import detect_scm_system, ScriptError
42
43 def log(string):
44     print >> sys.stderr, string
45
46 def error(string):
47     log(string)
48     exit(1)
49
50 def plural(noun):
51     # This is a dumb plural() implementation which was just enough for our uses.
52     if re.search('h$', noun):
53         return noun + 'es'
54     else:
55         return noun + 's'
56
57 def pluralize(noun, count):
58     if count != 1:
59         noun = plural(noun)
60     return "%d %s" % (count, noun)
61
62 # These could be put in some sort of changelogs.py.
63 def latest_changelog_entry(changelog_path):
64     # e.g. 2009-06-03  Eric Seidel  <eric@webkit.org>
65     changelog_date_line_regexp = re.compile('^(\d{4}-\d{2}-\d{2})' # Consume the date.
66                                   + '\s+(.+)\s+' # Consume the name.
67                                   + '<([^<>]+)>$') # And finally the email address.
68     
69     entry_lines = []
70     changelog = open(changelog_path)
71     try:
72         log("Parsing ChangeLog: " + changelog_path)
73         # The first line should be a date line.
74         first_line = changelog.readline()
75         if not changelog_date_line_regexp.match(first_line):
76             return None
77         entry_lines.append(first_line)
78         
79         for line in changelog:
80             # If we've hit the next entry, return.
81             if changelog_date_line_regexp.match(line):
82                 return ''.join(entry_lines)
83             entry_lines.append(line)
84     finally:
85         changelog.close()
86     # We never found a date line!
87     return None
88
89 def modified_changelogs(scm):
90     changelog_paths = []
91     paths = scm.changed_files()
92     for path in paths:
93         if os.path.basename(path) == "ChangeLog":
94             changelog_paths.append(path)
95     return changelog_paths
96
97 def commit_message_for_this_commit(scm):
98     changelog_paths = modified_changelogs(scm)
99     if not len(changelog_paths):
100         error("Found no modified ChangeLogs, can't create a commit message.")
101
102     changelog_messages = []
103     for path in changelog_paths:
104         changelog_entry = latest_changelog_entry(path)
105         if not changelog_entry:
106             error("Failed to parse ChangeLog: " + os.path.abspath(path))
107         changelog_messages.append(changelog_entry)
108     
109     # FIXME: We should sort and label the ChangeLog messages like commit-log-editor does.
110     return ''.join(changelog_messages)
111
112
113 class Command:
114     def __init__(self, help_text, argument_names="", options=[]):
115         self.help_text = help_text
116         self.argument_names = argument_names
117         self.options = options
118         self.option_parser = OptionParser(usage=SUPPRESS_USAGE, add_help_option=False, option_list=self.options)
119     
120     def name_with_arguments(self, command_name):
121         usage_string = command_name
122         if len(self.options) > 0:
123             usage_string += " [options]"
124         if self.argument_names:
125             usage_string += " " + self.argument_names
126         return usage_string
127
128     def parse_args(self, args):
129         return self.option_parser.parse_args(args)
130
131     def execute(self, options, args, tool):
132         raise NotImplementedError, "subclasses must implement"
133
134
135 class BugsInCommitQueue(Command):
136     def __init__(self):
137         Command.__init__(self, 'Bugs in the commit queue')
138
139     def execute(self, options, args, tool):
140         bug_ids = tool.bugs.fetch_bug_ids_from_commit_queue()
141         for bug_id in bug_ids:
142             print "%s" % tool.bugs.bug_url_for_bug_id(bug_id)
143
144
145 class PatchesInCommitQueue(Command):
146     def __init__(self):
147         Command.__init__(self, 'Patches attached to bugs in the commit queue')
148
149     def execute(self, options, args, tool):
150         patches = tool.bugs.fetch_patches_from_commit_queue()
151         log("Patches in commit queue:")
152         for patch in patches:
153             print "%s" % patch['url']
154
155
156 class ReviewedPatchesOnBug(Command):
157     def __init__(self):
158         Command.__init__(self, 'r+\'d patches on a bug', 'BUGID')
159
160     def execute(self, options, args, tool):
161         bug_id = args[0]
162         patches_to_land = tool.bugs.fetch_reviewed_patches_from_bug(bug_id)
163         for patch in patches_to_land:
164             print "%s" % patch['url']
165
166
167 class ApplyPatchesFromBug(Command):
168     def __init__(self):
169         options = [
170             make_option("--no-update", action="store_false", dest="update", default=True, help="Don't update the working directory before applying patches"),
171             make_option("--force-clean", action="store_true", dest="force_clean", default=False, help="Clean working directory before applying patches (removes local changes and commits)"),
172             make_option("--no-clean", action="store_false", dest="clean", default=True, help="Don't check if the working directory is clean before applying patches"),
173             make_option("--local-commit", action="store_true", dest="local_commit", default=False, help="Make a local commit for each applied patch"),
174         ]
175         Command.__init__(self, 'Applies all patches on a bug to the local working directory without committing.', 'BUGID', options=options)
176
177     @staticmethod
178     def apply_patches(patches, scm, commit_each):
179         for patch in patches:
180             scm.apply_patch(patch)
181             if commit_each:
182                 commit_message = commit_message_for_this_commit(scm)
183                 scm.commit_locally_with_message(commit_message or patch['name'])
184
185     def execute(self, options, args, tool):
186         bug_id = args[0]
187         patches = tool.bugs.fetch_reviewed_patches_from_bug(bug_id)
188         os.chdir(tool.scm().checkout_root)
189         if options.clean:
190             tool.scm().ensure_clean_working_directory(options.force_clean)
191         if options.update:
192             tool.scm().update_webkit()
193         
194         if options.local_commit and not tool.scm().supports_local_commits():
195             error("--local-commit passed, but %s does not support local commits" % tool.scm().display_name())
196         
197         self.apply_patches(patches, tool.scm(), options.local_commit)
198
199 def bug_comment_from_commit_text(commit_text):
200     comment_lines = []
201     commit_lines = commit_text.splitlines()
202     for line in commit_lines:
203         comment_lines.append(line)
204         match = re.match("^Committed r(\d+)$", line)
205         if match:
206             revision = match.group(1)
207             comment_lines.append("http://trac.webkit.org/changeset/" + revision)
208             break
209     return "\n".join(comment_lines)
210
211
212 class LandAndUpdateBug(Command):
213     def __init__(self):
214         Command.__init__(self, 'Lands the current working directory diff and updates the bug.', 'BUGID')
215
216     def execute(self, options, args, tool):
217         bug_id = args[0]
218         os.chdir(tool.scm().checkout_root)
219         commit_message = commit_message_for_this_commit(tool.scm())
220         commit_log = tool.scm().commit_with_message(commit_message)
221         comment_text = bug_comment_from_commit_text(commit_log)
222         tool.bugs.close_bug_as_fixed(bug_id, comment_text)
223
224
225 class LandPatchesFromBugs(Command):
226     def __init__(self):
227         options = [
228             make_option("--no-update", action="store_false", dest="update", default=True, help="Don't update the working directory before applying patches"),
229             make_option("--force-clean", action="store_true", dest="force_clean", default=False, help="Clean working directory before applying patches (removes local changes and commits)"),
230             make_option("--no-clean", action="store_false", dest="clean", default=True, help="Don't check if the working directory is clean before applying patches"),
231             make_option("--no-build", action="store_false", dest="build", default=True, help="Commit without building first, implies --no-test."),
232             make_option("--no-test", action="store_false", dest="test", default=True, help="Commit without runnning run-webkit-tests."),
233         ]
234         Command.__init__(self, 'Lands all patches on a bug optionally testing them first', 'BUGID', options=options)
235
236     @staticmethod
237     def run_and_throw_if_fail(script_name):
238         build_webkit_process = subprocess.Popen(script_name)
239         return_code = build_webkit_process.wait()
240         if return_code:
241             raise ScriptError("%s failed with exit code %d" % (script_name, return_code))
242
243     @classmethod
244     def run_webkit_script(cls, script_name):
245         # We might need to pass scm into this function for scm.checkout_root
246         cls.run_and_throw_if_fail(os.path.join("WebKitTools", "Scripts", script_name))
247
248     @classmethod
249     def build_webkit(cls):
250         cls.run_webkit_script("build-webkit")
251
252     @classmethod
253     def run_webkit_tests(cls):
254         cls.run_webkit_script("run-webkit-tests")
255
256     @staticmethod
257     def setup_for_landing(scm, options):
258         os.chdir(scm.checkout_root)
259         scm.ensure_no_local_commits(options.force_clean)
260         if options.clean:
261             scm.ensure_clean_working_directory(options.force_clean)
262         if options.update:
263             scm.update_webkit()
264
265     @classmethod
266     def build_and_commit(cls, scm, options):
267         if options.build:
268             cls.build_webkit()
269             if options.test:
270                 cls.run_webkit_tests()
271         commit_message = commit_message_for_this_commit(scm)
272         commit_log = scm.commit_with_message(commit_message)
273         return bug_comment_from_commit_text(commit_log)
274
275     @classmethod
276     def land_patches(cls, bug_id, patches, options, tool):
277         try:
278             comment_text = ""
279             for patch in patches:
280                 tool.scm().apply_patch(patch)
281                 comment_text = cls.build_and_commit(tool.scm(), options)
282
283                 # If we're commiting more than one patch, update the bug as we go.
284                 if len(patches) > 1:
285                     tool.bugs.obsolete_attachment(patch['id'], comment_text)
286
287             if len(patches) > 1:
288                 comment_text = "All reviewed patches landed, closing."
289
290             tool.bugs.close_bug_as_fixed(bug_id, comment_text)
291         except ScriptError, e:
292             # We should add a comment to the bug, and r- the patch on failure
293             error(e)
294
295     def execute(self, options, args, tool):
296         if not len(args):
297             error("bug-id(s) required")
298
299         bugs_to_patches = {}
300         patch_count = 0
301         for bug_id in args:
302             patches = tool.bugs.fetch_reviewed_patches_from_bug(bug_id)
303             if not len(patches):
304                 exit("No reviewed patches found on %s" % bug_id)
305             patch_count += len(patches)
306             bugs_to_patches[bug_id] = patches
307
308         log("Landing %s from %s." % (pluralize("patch", patch_count), pluralize("bug", len(args))))
309         
310         self.setup_for_landing(tool.scm(), options)
311
312         for bug_id in args:
313             self.land_patches(bug_id, bugs_to_patches[bug_id], options, tool)
314
315 class CommitMessageForCurrentDiff(Command):
316     def __init__(self):
317         Command.__init__(self, 'Prints a commit message suitable for the uncommitted changes.')
318
319     def execute(self, options, args, tool):
320         os.chdir(tool.scm().checkout_root)
321         print "%s" % commit_message_for_this_commit(tool.scm())
322
323
324 class ObsoleteAttachmentsOnBug(Command):
325     def __init__(self):
326         Command.__init__(self, 'Marks all attachments on a bug as obsolete.', 'BUGID')
327
328     def execute(self, options, args, tool):
329         bug_id = args[0]
330         attachments = tool.bugs.fetch_attachments_from_bug(bug_id)
331         for attachment in attachments:
332             if not attachment['obsolete']:
333                 tool.bugs.obsolete_attachment(attachment['id'])
334
335
336 class PostDiffAsPatchToBug(Command):
337     def __init__(self):
338         options = [
339             make_option("--no-review", action="store_false", dest="review", default=True, help="Do not mark the patch for review."),
340             make_option("-m", "--description", action="store", type="string", dest="description", help="Description string for the attachment (default: 'patch')"),
341         ]
342         Command.__init__(self, 'Attaches the current working directory diff to a bug as a patch file.', 'BUGID', options=options)
343
344     def execute(self, options, args, tool):
345         bug_id = args[0]
346         
347         diff_process = subprocess.Popen(tool.scm().create_patch_command(), stdout=subprocess.PIPE, shell=True)
348         diff_process.wait() # Make sure svn-create-patch is done before we continue.
349         
350         description = options.description or "patch"
351         tool.bugs.add_patch_to_bug(bug_id, diff_process.stdout, description, mark_for_review=options.review)
352
353
354 class PostCommitsAsPatchesToBug(Command):
355     def __init__(self):
356         options = [
357             make_option("--no-review", action="store_false", dest="review", default=True, help="Do not mark the patch for review."),
358         ]
359         Command.__init__(self, 'Attaches a range of local commits to a bug as patch files.', 'BUGID COMMITISH', options=options)
360
361     def execute(self, options, args, tool):
362         bug_id = args[0]
363         
364         if not tool.scm().supports_local_commits():
365             log(tool.scm().display_name() + " does not support local commits.")
366             exit(1)
367         
368         commit_ids = tool.scm().commit_ids_from_range_arguments(args[1:])
369         
370         if len(commit_ids) > 10:
371             log("Are you sure you want to attach %s to bug %s?" % (pluralize('patch', len(commit_ids)), bug_id))
372             # Could add a --patches-limit option.
373             exit(1)
374         
375         log("Attaching %s as patches to bug %s" % (pluralize('commit', len(commit_ids)), bug_id))
376         for commit_id in commit_ids:
377             commit_message = tool.scm().commit_message_for_commit(commit_id)
378             commit_lines = commit_message.splitlines()
379             
380             description = commit_lines[0]
381             comment_text = "\n".join(commit_lines[1:])
382             comment_text += "---\n"
383             comment_text += tool.scm().files_changed_summary_for_commit(commit_id)
384         
385             # This is a little bit of a hack, that we pass stdout as the patch file.
386             # We could alternatively make an in-memory file-like object with the patch contents.
387             diff_process = subprocess.Popen(tool.scm().show_diff_command_for_commit(commit_id), stdout=subprocess.PIPE, shell=True)
388             tool.bugs.add_patch_to_bug(bug_id, diff_process.stdout, description, comment_text, mark_for_review=options.review)
389
390
391 class NonWrappingEpilogIndentedHelpFormatter(IndentedHelpFormatter):
392     def __init__(self):
393         IndentedHelpFormatter.__init__(self)
394
395     # The standard IndentedHelpFormatter paragraph-wraps the epilog, killing our custom formatting.
396     def format_epilog(self, epilog):
397         if epilog:
398             return "\n" + epilog + "\n"
399         return ""
400
401 class BugzillaTool:
402     def __init__(self):
403         self.cached_scm = None
404         self.bugs = Bugzilla()
405         self.commands = [
406             { 'name' : 'bugs-to-commit', 'object' : BugsInCommitQueue() },
407             { 'name' : 'patches-to-commit', 'object' : PatchesInCommitQueue() },
408             { 'name' : 'reviewed-patches', 'object' : ReviewedPatchesOnBug() },
409             { 'name' : 'apply-patches', 'object' : ApplyPatchesFromBug() },
410             { 'name' : 'land-and-update', 'object' : LandAndUpdateBug() },
411             { 'name' : 'land-patches', 'object' : LandPatchesFromBugs() },
412             { 'name' : 'commit-message', 'object' : CommitMessageForCurrentDiff() },
413             { 'name' : 'obsolete-attachments', 'object' : ObsoleteAttachmentsOnBug() },
414             { 'name' : 'post-diff', 'object' : PostDiffAsPatchToBug() },
415             { 'name' : 'post-commits', 'object' : PostCommitsAsPatchesToBug() },
416         ]
417         
418         self.global_option_parser = OptionParser(usage=self.usage_line(), formatter=NonWrappingEpilogIndentedHelpFormatter(), epilog=self.commands_usage())
419         self.global_option_parser.add_option("--dry-run", action="store_true", dest="dryrun", help="do not touch remote servers", default=False)
420     
421     def scm(self):
422         # Lazily initialize SCM to not error-out before command line parsing (or when running non-scm commands).
423         original_cwd = os.path.abspath('.')
424         if not self.cached_scm:
425             self.cached_scm = detect_scm_system(original_cwd)
426         
427         if not self.cached_scm:
428             script_directory = os.path.abspath(sys.path[0])
429             webkit_directory = os.path.abspath(os.path.join(script_directory, "../.."))
430             self.cached_scm = detect_scm_system(webkit_directory)
431             if self.cached_scm:
432                 log("The current directory (%s) is not a WebKit checkout, using %s" % (original_cwd, webkit_directory))
433             else:
434                 error("FATAL: Failed to determine the SCM system for either %s or %s" % (original_cwd, webkit_directory))
435         
436         return self.cached_scm
437     
438     @staticmethod
439     def usage_line():
440         return "Usage: %prog [options] command [command-options] [command-arguments]"
441     
442     def commands_usage(self):
443         commands_text = "Commands:\n"
444         longest_name_length = 0
445         command_rows = []
446         for command in self.commands:
447             command_object = command['object']
448             command_name_and_args = command_object.name_with_arguments(command['name'])
449             command_rows.append({ 'name-and-args': command_name_and_args, 'object': command_object })
450             longest_name_length = max([longest_name_length, len(command_name_and_args)])
451         
452         # Use our own help formatter so as to indent enough.
453         formatter = IndentedHelpFormatter()
454         formatter.indent()
455         formatter.indent()
456         
457         for row in command_rows:
458             command_object = row['object']
459             commands_text += "  " + row['name-and-args'].ljust(longest_name_length + 3) + command_object.help_text + "\n"
460             commands_text += command_object.option_parser.format_option_help(formatter)
461         return commands_text
462
463     def handle_global_args(self, args):
464         (options, args) = self.global_option_parser.parse_args(args)
465         if len(args):
466             # We'll never hit this because split_args splits at the first arg without a leading '-'
467             self.global_option_parser.error("Extra arguments before command: " + args)
468         
469         if options.dryrun:
470             self.scm().dryrun = True
471             self.bugs.dryrun = True
472     
473     @staticmethod
474     def split_args(args):
475         # Assume the first argument which doesn't start with '-' is the command name.
476         command_index = 0
477         for arg in args:
478             if arg[0] != '-':
479                 break
480             command_index += 1
481         else:
482             return (args[:], None, [])
483
484         global_args = args[:command_index]
485         command = args[command_index]
486         command_args = args[command_index + 1:]
487         return (global_args, command, command_args)
488     
489     def command_by_name(self, command_name):
490         for command in self.commands:
491             if command_name == command['name']:
492                 return command
493         return None
494     
495     def main(self):
496         (global_args, command_name, args_after_command_name) = self.split_args(sys.argv[1:])
497         
498         # Handle --help, etc:
499         self.handle_global_args(global_args)
500         
501         if not command_name:
502             self.global_option_parser.error("No command specified")
503         
504         command = self.command_by_name(command_name)
505         if not command:
506             self.global_option_parser.error(command_name + " is not a recognized command")
507         
508         command_object = command['object']
509         (command_options, command_args) = command_object.parse_args(args_after_command_name)
510         return command_object.execute(command_options, command_args, self)
511
512
513 def main():
514     tool = BugzillaTool()
515     return tool.main()
516
517 if __name__ == "__main__":
518     main()