4c4357d22bc772bd9925fe48ab1d3927fd63d454
[WebKit-https.git] / WebKitTools / Scripts / bugzilla-tool
1 #!/usr/bin/python
2 # Copyright (c) 2009, Google Inc. All rights reserved.
3 #
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
6 # met:
7
8 #     * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 #     * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
13 # distribution.
14 #     * Neither the name of Google Inc. nor the names of its
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
17
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29 #
30 # A tool for automating dealing with bugzilla, posting patches, committing patches, etc.
31
32 import os
33 import re
34 import subprocess
35 import sys
36
37 from optparse import OptionParser, IndentedHelpFormatter, SUPPRESS_USAGE, make_option
38
39 # Import WebKit-specific modules.
40 from modules.bugzilla import Bugzilla
41 from modules.scm import detect_scm_system, ScriptError
42
43 def log(string):
44     print >> sys.stderr, string
45
46 def error(string):
47     log(string)
48     exit(1)
49
50 def plural(noun):
51     # This is a dumb plural() implementation which was just enough for our uses.
52     if re.search('h$', noun):
53         return noun + 'es'
54     else:
55         return noun + 's'
56
57 def pluralize(noun, count):
58     if count != 1:
59         noun = plural(noun)
60     return "%d %s" % (count, noun)
61
62 # These could be put in some sort of changelogs.py.
63 def latest_changelog_entry(changelog_path):
64     # e.g. 2009-06-03  Eric Seidel  <eric@webkit.org>
65     changelog_date_line_regexp = re.compile('^(\d{4}-\d{2}-\d{2})' # Consume the date.
66                                   + '\s+(.+)\s+' # Consume the name.
67                                   + '<([^<>]+)>$') # And finally the email address.
68     
69     entry_lines = []
70     changelog = open(changelog_path)
71     try:
72         log("Parsing ChangeLog: " + changelog_path)
73         # The first line should be a date line.
74         first_line = changelog.readline()
75         if not changelog_date_line_regexp.match(first_line):
76             return None
77         entry_lines.append(first_line)
78         
79         for line in changelog:
80             # If we've hit the next entry, return.
81             if changelog_date_line_regexp.match(line):
82                 return ''.join(entry_lines)
83             entry_lines.append(line)
84     finally:
85         changelog.close()
86     # We never found a date line!
87     return None
88
89 def modified_changelogs(scm):
90     changelog_paths = []
91     paths = scm.changed_files()
92     for path in paths:
93         if os.path.basename(path) == "ChangeLog":
94             changelog_paths.append(path)
95     return changelog_paths
96
97 def commit_message_for_this_commit(scm):
98     changelog_paths = modified_changelogs(scm)
99     if not len(changelog_paths):
100         error("Found no modified ChangeLogs, can't create a commit message.")
101
102     changelog_messages = []
103     for path in changelog_paths:
104         changelog_entry = latest_changelog_entry(path)
105         if not changelog_entry:
106             error("Failed to parse ChangeLog: " + os.path.abspath(path))
107         changelog_messages.append(changelog_entry)
108     
109     # FIXME: We should sort and label the ChangeLog messages like commit-log-editor does.
110     return ''.join(changelog_messages)
111
112
113 class Command:
114     def __init__(self, help_text, argument_names="", options=[]):
115         self.help_text = help_text
116         self.argument_names = argument_names
117         self.options = options
118         self.option_parser = OptionParser(usage=SUPPRESS_USAGE, add_help_option=False, option_list=self.options)
119     
120     def name_with_arguments(self, command_name):
121         usage_string = command_name
122         if len(self.options) > 0:
123             usage_string += " [options]"
124         if self.argument_names:
125             usage_string += " " + self.argument_names
126         return usage_string
127
128     def parse_args(self, args):
129         return self.option_parser.parse_args(args)
130
131     def execute(self, options, args, tool):
132         raise NotImplementedError, "subclasses must implement"
133
134
135 class BugsInCommitQueue(Command):
136     def __init__(self):
137         Command.__init__(self, 'Bugs in the commit queue')
138
139     def execute(self, options, args, tool):
140         bug_ids = tool.bugs.fetch_bug_ids_from_commit_queue()
141         for bug_id in bug_ids:
142             print "%s" % tool.bugs.bug_url_for_bug_id(bug_id)
143
144
145 class PatchesInCommitQueue(Command):
146     def __init__(self):
147         Command.__init__(self, 'Patches attached to bugs in the commit queue')
148
149     def execute(self, options, args, tool):
150         patches = tool.bugs.fetch_patches_from_commit_queue()
151         log("Patches in commit queue:")
152         for patch in patches:
153             print "%s" % patch['url']
154
155
156 class ReviewedPatchesOnBug(Command):
157     def __init__(self):
158         Command.__init__(self, 'r+\'d patches on a bug', 'BUGID')
159
160     def execute(self, options, args, tool):
161         bug_id = args[0]
162         patches_to_land = tool.bugs.fetch_reviewed_patches_from_bug(bug_id)
163         for patch in patches_to_land:
164             print "%s" % patch['url']
165
166
167 class ApplyPatchesFromBug(Command):
168     def __init__(self):
169         options = [
170             make_option("--no-update", action="store_false", dest="update", default=True, help="Don't update the working directory before applying patches"),
171             make_option("--force-clean", action="store_true", dest="force_clean", default=False, help="Clean working directory before applying patches (removes local changes and commits)"),
172             make_option("--no-clean", action="store_false", dest="clean", default=True, help="Don't check if the working directory is clean before applying patches"),
173             make_option("--local-commit", action="store_true", dest="local_commit", default=False, help="Make a local commit for each applied patch"),
174         ]
175         Command.__init__(self, 'Applies all patches on a bug to the local working directory without committing.', 'BUGID', options=options)
176
177     @staticmethod
178     def apply_patches(patches, scm, commit_each):
179         for patch in patches:
180             scm.apply_patch(patch)
181             if commit_each:
182                 commit_message = commit_message_for_this_commit(scm)
183                 scm.commit_locally_with_message(commit_message or patch['name'])
184
185     def execute(self, options, args, tool):
186         bug_id = args[0]
187         patches = tool.bugs.fetch_reviewed_patches_from_bug(bug_id)
188         os.chdir(tool.scm().checkout_root)
189         if options.clean:
190             tool.scm().ensure_clean_working_directory(options.force_clean)
191         if options.update:
192             tool.scm().update_webkit()
193         
194         if options.local_commit and not tool.scm().supports_local_commits():
195             error("--local-commit passed, but %s does not support local commits" % tool.scm().display_name())
196         
197         self.apply_patches(patches, tool.scm(), options.local_commit)
198
199 def bug_comment_from_commit_text(commit_text):
200     comment_lines = []
201     commit_lines = commit_text.splitlines()
202     for line in commit_lines:
203         comment_lines.append(line)
204         match = re.match("^Committed r(\d+)$", line)
205         if match:
206             revision = match.group(1)
207             comment_lines.append("http://trac.webkit.org/changeset/" + revision)
208             break
209     return "\n".join(comment_lines)
210
211
212 class LandAndUpdateBug(Command):
213     def __init__(self):
214         Command.__init__(self, 'Lands the current working directory diff and updates the bug.', 'BUGID')
215
216     def execute(self, options, args, tool):
217         bug_id = args[0]
218         os.chdir(tool.scm().checkout_root)
219         commit_message = commit_message_for_this_commit(tool.scm())
220         commit_log = tool.scm().commit_with_message(commit_message)
221         comment_text = bug_comment_from_commit_text(commit_log)
222         tool.bugs.close_bug_as_fixed(bug_id, comment_text)
223
224
225 class LandPatchesFromBugs(Command):
226     def __init__(self):
227         options = [
228             make_option("--no-update", action="store_false", dest="update", default=True, help="Don't update the working directory before applying patches"),
229             make_option("--force-clean", action="store_true", dest="force_clean", default=False, help="Clean working directory before applying patches (removes local changes and commits)"),
230             make_option("--no-clean", action="store_false", dest="clean", default=True, help="Don't check if the working directory is clean before applying patches"),
231             make_option("--no-build", action="store_false", dest="build", default=True, help="Commit without building first, implies --no-test."),
232             make_option("--no-test", action="store_false", dest="test", default=True, help="Commit without runnning run-webkit-tests."),
233         ]
234         Command.__init__(self, 'Lands all patches on a bug optionally testing them first', 'BUGID', options=options)
235
236     @staticmethod
237     def run_and_throw_if_fail(script_name):
238         build_webkit_process = subprocess.Popen(script_name)
239         return_code = build_webkit_process.wait()
240         if return_code:
241             raise ScriptError("%s failed with exit code %d" % (script_name, return_code))
242
243     @classmethod
244     def run_webkit_script(cls, script_name):
245         # We might need to pass scm into this function for scm.checkout_root
246         cls.run_and_throw_if_fail(os.path.join("WebKitTools", "Scripts", script_name))
247
248     @classmethod
249     def build_webkit(cls):
250         cls.run_webkit_script("build-webkit")
251
252     @classmethod
253     def run_webkit_tests(cls):
254         cls.run_webkit_script("run-webkit-tests")
255
256     @staticmethod
257     def setup_for_landing(scm, options):
258         os.chdir(scm.checkout_root)
259         scm.ensure_no_local_commits(options.force_clean)
260         if options.clean:
261             scm.ensure_clean_working_directory(options.force_clean)
262         if options.update:
263             scm.update_webkit()
264
265     @classmethod
266     def build_and_commit(cls, scm, options):
267         if options.build:
268             cls.build_webkit()
269             if options.test:
270                 cls.run_webkit_tests()
271         commit_message = commit_message_for_this_commit(scm)
272         commit_log = scm.commit_with_message(commit_message)
273         return bug_comment_from_commit_text(commit_log)
274
275     @classmethod
276     def land_patches(cls, bug_id, patches, options, tool):
277         try:
278             comment_text = ""
279             for patch in patches:
280                 tool.scm().apply_patch(patch)
281                 comment_text = cls.build_and_commit(tool.scm(), options)
282
283                 # If we're commiting more than one patch, update the bug as we go.
284                 if len(patches) > 1:
285                     tool.bugs.obsolete_attachment(patch['id'], comment_text)
286
287             if len(patches) > 1:
288                 comment_text = "All reviewed patches landed, closing."
289
290             tool.bugs.close_bug_as_fixed(bug_id, comment_text)
291         except ScriptError, e:
292             # We should add a comment to the bug, and r- the patch on failure
293             error(e)
294
295     def execute(self, options, args, tool):
296         if not len(args):
297             error("bug-id(s) required")
298
299         bugs_to_patches = {}
300         patch_count = 0
301         for bug_id in args:
302             patches = tool.bugs.fetch_reviewed_patches_from_bug(bug_id)
303             if not len(patches):
304                 exit("No reviewed patches found on %s" % bug_id)
305             patch_count += len(patches)
306             bugs_to_patches[bug_id] = patches
307
308         log("Landing %s from %s." % (pluralize("patch", patch_count), pluralize("bug", len(args))))
309         
310         self.setup_for_landing(tool.scm(), options)
311
312         for bug_id in args:
313             self.land_patches(bug_id, bugs_to_patches[bug_id], options, tool)
314
315 class CommitMessageForCurrentDiff(Command):
316     def __init__(self):
317         Command.__init__(self, 'Prints a commit message suitable for the uncommitted changes.')
318
319     def execute(self, options, args, tool):
320         os.chdir(tool.scm().checkout_root)
321         print "%s" % commit_message_for_this_commit(tool.scm())
322
323
324 class ObsoleteAttachmentsOnBug(Command):
325     def __init__(self):
326         Command.__init__(self, 'Marks all attachments on a bug as obsolete.', 'BUGID')
327
328     def execute(self, options, args, tool):
329         bug_id = args[0]
330         attachments = tool.bugs.fetch_attachments_from_bug(bug_id)
331         for attachment in attachments:
332             if not attachment['is_obsolete']:
333                 tool.bugs.obsolete_attachment(attachment['id'])
334
335
336 class PostDiffAsPatchToBug(Command):
337     def __init__(self):
338         options = [
339             make_option("--no-obsolete", action="store_false", dest="obsolete_patches", default=True, help="Do not obsolete old patches before posting this one."),
340             make_option("--no-review", action="store_false", dest="review", default=True, help="Do not mark the patch for review."),
341             make_option("-m", "--description", action="store", type="string", dest="description", help="Description string for the attachment (default: 'patch')"),
342         ]
343         Command.__init__(self, 'Attaches the current working directory diff to a bug as a patch file.', 'BUGID', options=options)
344
345     @staticmethod
346     def obsolete_patches_on_bug(bug_id, bugs):
347         patches = bugs.fetch_patches_from_bug(bug_id)
348         if len(patches):
349             log("Obsoleting %s on bug %s" % (pluralize('old patch', len(patches)), bug_id))
350             for patch in patches:
351                 bugs.obsolete_attachment(patch['id'])
352
353     def execute(self, options, args, tool):
354         bug_id = args[0]
355
356         if options.obsolete_patches:
357             self.obsolete_patches_on_bug(bug_id, tool.bugs)
358
359         diff_process = subprocess.Popen(tool.scm().create_patch_command(), stdout=subprocess.PIPE, shell=True)
360         diff_process.wait() # Make sure svn-create-patch is done before we continue.
361         
362         description = options.description or "patch"
363         tool.bugs.add_patch_to_bug(bug_id, diff_process.stdout, description, mark_for_review=options.review)
364
365
366 class PostCommitsAsPatchesToBug(Command):
367     def __init__(self):
368         options = [
369             make_option("--no-obsolete", action="store_false", dest="obsolete_patches", default=True, help="Do not obsolete old patches before posting new ones."),
370             make_option("--no-review", action="store_false", dest="review", default=True, help="Do not mark the patch for review."),
371         ]
372         Command.__init__(self, 'Attaches a range of local commits to a bug as patch files.', 'BUGID COMMITISH', options=options)
373
374     def execute(self, options, args, tool):
375         bug_id = args[0]
376         
377         if not tool.scm().supports_local_commits():
378             error(tool.scm().display_name() + " does not support local commits.")
379         
380         commit_ids = tool.scm().commit_ids_from_range_arguments(args[1:])
381         
382         if len(commit_ids) > 10:
383             error("Are you sure you want to attach %s to bug %s?" % (pluralize('patch', len(commit_ids)), bug_id))
384             # Could add a --patches-limit option.
385
386         if options.obsolete_patches:
387             PostDiffAsPatchToBug.obsolete_patches_on_bug(bug_id, tool.bugs)
388         
389         log("Attaching %s as patches to bug %s" % (pluralize('commit', len(commit_ids)), bug_id))
390         for commit_id in commit_ids:
391             commit_message = tool.scm().commit_message_for_commit(commit_id)
392             commit_lines = commit_message.splitlines()
393             
394             description = commit_lines[0]
395             comment_text = "\n".join(commit_lines[1:])
396             comment_text += "---\n"
397             comment_text += tool.scm().files_changed_summary_for_commit(commit_id)
398         
399             # This is a little bit of a hack, that we pass stdout as the patch file.
400             # We could alternatively make an in-memory file-like object with the patch contents.
401             diff_process = subprocess.Popen(tool.scm().show_diff_command_for_commit(commit_id), stdout=subprocess.PIPE, shell=True)
402             tool.bugs.add_patch_to_bug(bug_id, diff_process.stdout, description, comment_text, mark_for_review=options.review)
403
404
405 class NonWrappingEpilogIndentedHelpFormatter(IndentedHelpFormatter):
406     def __init__(self):
407         IndentedHelpFormatter.__init__(self)
408
409     # The standard IndentedHelpFormatter paragraph-wraps the epilog, killing our custom formatting.
410     def format_epilog(self, epilog):
411         if epilog:
412             return "\n" + epilog + "\n"
413         return ""
414
415 class BugzillaTool:
416     def __init__(self):
417         self.cached_scm = None
418         self.bugs = Bugzilla()
419         self.commands = [
420             { 'name' : 'bugs-to-commit', 'object' : BugsInCommitQueue() },
421             { 'name' : 'patches-to-commit', 'object' : PatchesInCommitQueue() },
422             { 'name' : 'reviewed-patches', 'object' : ReviewedPatchesOnBug() },
423             { 'name' : 'apply-patches', 'object' : ApplyPatchesFromBug() },
424             { 'name' : 'land-and-update', 'object' : LandAndUpdateBug() },
425             { 'name' : 'land-patches', 'object' : LandPatchesFromBugs() },
426             { 'name' : 'commit-message', 'object' : CommitMessageForCurrentDiff() },
427             { 'name' : 'obsolete-attachments', 'object' : ObsoleteAttachmentsOnBug() },
428             { 'name' : 'post-diff', 'object' : PostDiffAsPatchToBug() },
429             { 'name' : 'post-commits', 'object' : PostCommitsAsPatchesToBug() },
430         ]
431         
432         self.global_option_parser = OptionParser(usage=self.usage_line(), formatter=NonWrappingEpilogIndentedHelpFormatter(), epilog=self.commands_usage())
433         self.global_option_parser.add_option("--dry-run", action="store_true", dest="dryrun", help="do not touch remote servers", default=False)
434     
435     def scm(self):
436         # Lazily initialize SCM to not error-out before command line parsing (or when running non-scm commands).
437         original_cwd = os.path.abspath('.')
438         if not self.cached_scm:
439             self.cached_scm = detect_scm_system(original_cwd)
440         
441         if not self.cached_scm:
442             script_directory = os.path.abspath(sys.path[0])
443             webkit_directory = os.path.abspath(os.path.join(script_directory, "../.."))
444             self.cached_scm = detect_scm_system(webkit_directory)
445             if self.cached_scm:
446                 log("The current directory (%s) is not a WebKit checkout, using %s" % (original_cwd, webkit_directory))
447             else:
448                 error("FATAL: Failed to determine the SCM system for either %s or %s" % (original_cwd, webkit_directory))
449         
450         return self.cached_scm
451     
452     @staticmethod
453     def usage_line():
454         return "Usage: %prog [options] command [command-options] [command-arguments]"
455     
456     def commands_usage(self):
457         commands_text = "Commands:\n"
458         longest_name_length = 0
459         command_rows = []
460         for command in self.commands:
461             command_object = command['object']
462             command_name_and_args = command_object.name_with_arguments(command['name'])
463             command_rows.append({ 'name-and-args': command_name_and_args, 'object': command_object })
464             longest_name_length = max([longest_name_length, len(command_name_and_args)])
465         
466         # Use our own help formatter so as to indent enough.
467         formatter = IndentedHelpFormatter()
468         formatter.indent()
469         formatter.indent()
470         
471         for row in command_rows:
472             command_object = row['object']
473             commands_text += "  " + row['name-and-args'].ljust(longest_name_length + 3) + command_object.help_text + "\n"
474             commands_text += command_object.option_parser.format_option_help(formatter)
475         return commands_text
476
477     def handle_global_args(self, args):
478         (options, args) = self.global_option_parser.parse_args(args)
479         if len(args):
480             # We'll never hit this because split_args splits at the first arg without a leading '-'
481             self.global_option_parser.error("Extra arguments before command: " + args)
482         
483         if options.dryrun:
484             self.scm().dryrun = True
485             self.bugs.dryrun = True
486     
487     @staticmethod
488     def split_args(args):
489         # Assume the first argument which doesn't start with '-' is the command name.
490         command_index = 0
491         for arg in args:
492             if arg[0] != '-':
493                 break
494             command_index += 1
495         else:
496             return (args[:], None, [])
497
498         global_args = args[:command_index]
499         command = args[command_index]
500         command_args = args[command_index + 1:]
501         return (global_args, command, command_args)
502     
503     def command_by_name(self, command_name):
504         for command in self.commands:
505             if command_name == command['name']:
506                 return command
507         return None
508     
509     def main(self):
510         (global_args, command_name, args_after_command_name) = self.split_args(sys.argv[1:])
511         
512         # Handle --help, etc:
513         self.handle_global_args(global_args)
514         
515         if not command_name:
516             self.global_option_parser.error("No command specified")
517         
518         command = self.command_by_name(command_name)
519         if not command:
520             self.global_option_parser.error(command_name + " is not a recognized command")
521         
522         command_object = command['object']
523         (command_options, command_args) = command_object.parse_args(args_after_command_name)
524         return command_object.execute(command_options, command_args, self)
525
526
527 def main():
528     tool = BugzillaTool()
529     return tool.main()
530
531 if __name__ == "__main__":
532     main()