2009-12-02 Eric Seidel <eric@webkit.org>
[WebKit-https.git] / WebKitTools / Scripts / modules / commands / download.py
1 #!/usr/bin/env python
2 # Copyright (c) 2009, Google Inc. All rights reserved.
3 # Copyright (c) 2009 Apple Inc. All rights reserved.
4 #
5 # Redistribution and use in source and binary forms, with or without
6 # modification, are permitted provided that the following conditions are
7 # met:
8
9 #     * Redistributions of source code must retain the above copyright
10 # notice, this list of conditions and the following disclaimer.
11 #     * Redistributions in binary form must reproduce the above
12 # copyright notice, this list of conditions and the following disclaimer
13 # in the documentation and/or other materials provided with the
14 # distribution.
15 #     * Neither the name of Google Inc. nor the names of its
16 # contributors may be used to endorse or promote products derived from
17 # this software without specific prior written permission.
18
19 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30
31 import os
32
33 from optparse import make_option
34
35 from modules.bugzilla import parse_bug_id
36 from modules.buildsteps import BuildSteps
37 from modules.changelogs import ChangeLog
38 from modules.comments import bug_comment_from_commit_text
39 from modules.grammar import pluralize
40 from modules.landingsequence import LandingSequence, ConditionalLandingSequence
41 from modules.logging import error, log
42 from modules.multicommandtool import Command
43 from modules.scm import ScriptError
44
45
46 class BuildSequence(ConditionalLandingSequence):
47     def __init__(self, options, tool):
48         ConditionalLandingSequence.__init__(self, None, options, tool)
49
50     def run(self):
51         self.clean()
52         self.update()
53         self.build()
54
55
56 class Build(Command):
57     name = "build"
58     show_in_main_help = False
59     def __init__(self):
60         options = BuildSteps.cleaning_options()
61         options += BuildSteps.build_options()
62         options += BuildSteps.land_options()
63         Command.__init__(self, "Update working copy and build", "", options)
64
65     def execute(self, options, args, tool):
66         sequence = BuildSequence(options, tool)
67         sequence.run_and_handle_errors()
68
69
70 class ApplyAttachment(Command):
71     name = "apply-attachment"
72     show_in_main_help = True
73     def __init__(self):
74         options = WebKitApplyingScripts.apply_options()
75         options += BuildSteps.cleaning_options()
76         Command.__init__(self, "Apply an attachment to the local working directory", "ATTACHMENT_ID", options=options)
77
78     def execute(self, options, args, tool):
79         WebKitApplyingScripts.setup_for_patch_apply(tool, options)
80         attachment_id = args[0]
81         attachment = tool.bugs.fetch_attachment(attachment_id)
82         WebKitApplyingScripts.apply_patches_with_options(tool.scm(), [attachment], options)
83
84
85 class ApplyPatches(Command):
86     name = "apply-patches"
87     show_in_main_help = True
88     def __init__(self):
89         options = WebKitApplyingScripts.apply_options()
90         options += BuildSteps.cleaning_options()
91         Command.__init__(self, "Apply reviewed patches from provided bugs to the local working directory", "BUGID", options=options)
92
93     def execute(self, options, args, tool):
94         WebKitApplyingScripts.setup_for_patch_apply(tool, options)
95         bug_id = args[0]
96         patches = tool.bugs.fetch_reviewed_patches_from_bug(bug_id)
97         WebKitApplyingScripts.apply_patches_with_options(tool.scm(), patches, options)
98
99
100 class WebKitApplyingScripts:
101     @staticmethod
102     def apply_options():
103         return [
104             make_option("--no-update", action="store_false", dest="update", default=True, help="Don't update the working directory before applying patches"),
105             make_option("--local-commit", action="store_true", dest="local_commit", default=False, help="Make a local commit for each applied patch"),
106         ]
107
108     @staticmethod
109     def setup_for_patch_apply(tool, options):
110         tool.steps.clean_working_directory(tool.scm(), options, allow_local_commits=True)
111         if options.update:
112             tool.scm().update_webkit()
113
114     @staticmethod
115     def apply_patches_with_options(scm, patches, options):
116         if options.local_commit and not scm.supports_local_commits():
117             error("--local-commit passed, but %s does not support local commits" % scm.display_name())
118
119         for patch in patches:
120             log("Applying attachment %s from bug %s" % (patch["id"], patch["bug_id"]))
121             scm.apply_patch(patch)
122             if options.local_commit:
123                 commit_message = scm.commit_message_for_this_commit()
124                 scm.commit_locally_with_message(commit_message.message() or patch["name"])
125
126
127 class LandDiffSequence(ConditionalLandingSequence):
128     def __init__(self, patch, options, tool):
129         ConditionalLandingSequence.__init__(self, patch, options, tool)
130
131     def run(self):
132         self.build()
133         self.test()
134         commit_log = self.commit()
135         self.close_bug(commit_log)
136
137     def close_bug(self, commit_log):
138         comment_test = bug_comment_from_commit_text(self._tool.scm(), commit_log)
139         bug_id = self._patch["bug_id"]
140         if bug_id:
141             log("Updating bug %s" % bug_id)
142             if self._options.close_bug:
143                 self._tool.bugs.close_bug_as_fixed(bug_id, comment_test)
144             else:
145                 # FIXME: We should a smart way to figure out if the patch is attached
146                 # to the bug, and if so obsolete it.
147                 self._tool.bugs.post_comment_to_bug(bug_id, comment_test)
148         else:
149             log(comment_test)
150             log("No bug id provided.")
151
152
153 class LandDiff(Command):
154     name = "land-diff"
155     show_in_main_help = True
156     def __init__(self):
157         options = [
158             make_option("-r", "--reviewer", action="store", type="string", dest="reviewer", help="Update ChangeLogs to say Reviewed by REVIEWER."),
159         ]
160         options += BuildSteps.build_options()
161         options += BuildSteps.land_options()
162         Command.__init__(self, "Land the current working directory diff and updates the associated bug if any", "[BUGID]", options=options)
163
164     def guess_reviewer_from_bug(self, bugs, bug_id):
165         patches = bugs.fetch_reviewed_patches_from_bug(bug_id)
166         if len(patches) != 1:
167             log("%s on bug %s, cannot infer reviewer." % (pluralize("reviewed patch", len(patches)), bug_id))
168             return None
169         patch = patches[0]
170         reviewer = patch["reviewer"]
171         log("Guessing \"%s\" as reviewer from attachment %s on bug %s." % (reviewer, patch["id"], bug_id))
172         return reviewer
173
174     def update_changelogs_with_reviewer(self, reviewer, bug_id, tool):
175         if not reviewer:
176             if not bug_id:
177                 log("No bug id provided and --reviewer= not provided.  Not updating ChangeLogs with reviewer.")
178                 return
179             reviewer = self.guess_reviewer_from_bug(tool.bugs, bug_id)
180
181         if not reviewer:
182             log("Failed to guess reviewer from bug %s and --reviewer= not provided.  Not updating ChangeLogs with reviewer." % bug_id)
183             return
184
185         for changelog_path in tool.scm().modified_changelogs():
186             ChangeLog(changelog_path).set_reviewer(reviewer)
187
188     def execute(self, options, args, tool):
189         bug_id = (args and args[0]) or parse_bug_id(tool.scm().create_patch())
190
191         tool.steps.ensure_builders_are_green(tool.buildbot, options)
192
193         os.chdir(tool.scm().checkout_root)
194         self.update_changelogs_with_reviewer(options.reviewer, bug_id, tool)
195
196         fake_patch = {
197             "id": None,
198             "bug_id": bug_id
199         }
200
201         sequence = LandDiffSequence(fake_patch, options, tool)
202         sequence.run()
203
204
205 class AbstractPatchProcessingCommand(Command):
206     def __init__(self, help_text, args_description, options):
207         Command.__init__(self, help_text, args_description, options=options)
208
209     def _fetch_list_of_patches_to_process(self, options, args, tool):
210         raise NotImplementedError, "subclasses must implement"
211
212     def _prepare_to_process(self, options, args, tool):
213         raise NotImplementedError, "subclasses must implement"
214
215     @staticmethod
216     def _collect_patches_by_bug(patches):
217         bugs_to_patches = {}
218         for patch in patches:
219             bug_id = patch["bug_id"]
220             bugs_to_patches[bug_id] = bugs_to_patches.get(bug_id, []) + [patch]
221         return bugs_to_patches
222
223     def execute(self, options, args, tool):
224         self._prepare_to_process(options, args, tool)
225         patches = self._fetch_list_of_patches_to_process(options, args, tool)
226
227         # It's nice to print out total statistics.
228         bugs_to_patches = self._collect_patches_by_bug(patches)
229         log("Processing %s from %s." % (pluralize("patch", len(patches)), pluralize("bug", len(bugs_to_patches))))
230
231         for patch in patches:
232             self._process_patch(patch, options, args, tool)
233
234
235 class CheckStyleSequence(LandingSequence):
236     def __init__(self, patch, options, tool):
237         LandingSequence.__init__(self, patch, options, tool)
238
239     def run(self):
240         self.clean()
241         self.update()
242         self.apply_patch()
243         self.build()
244
245     def build(self):
246         # Instead of building, we check style.
247         self._tool.steps.check_style()
248
249
250 class CheckStyle(AbstractPatchProcessingCommand):
251     name = "check-style"
252     show_in_main_help = False
253     def __init__(self):
254         options = BuildSteps.cleaning_options()
255         options += BuildSteps.build_options()
256         AbstractPatchProcessingCommand.__init__(self, "Run check-webkit-style on the specified attachments", "ATTACHMENT_ID [ATTACHMENT_IDS]", options)
257
258     def _fetch_list_of_patches_to_process(self, options, args, tool):
259         return map(lambda patch_id: tool.bugs.fetch_attachment(patch_id), args)
260
261     def _prepare_to_process(self, options, args, tool):
262         pass
263
264     def _process_patch(self, patch, options, args, tool):
265         sequence = CheckStyleSequence(patch, options, tool)
266         sequence.run_and_handle_errors()
267
268
269 class BuildAttachmentSequence(LandingSequence):
270     def __init__(self, patch, options, tool):
271         LandingSequence.__init__(self, patch, options, tool)
272
273     def run(self):
274         self.clean()
275         self.update()
276         self.apply_patch()
277         self.build()
278
279
280 class BuildAttachment(AbstractPatchProcessingCommand):
281     name = "build-attachment"
282     show_in_main_help = False
283     def __init__(self):
284         options = BuildSteps.cleaning_options()
285         options += BuildSteps.build_options()
286         AbstractPatchProcessingCommand.__init__(self, "Apply and build patches from bugzilla", "ATTACHMENT_ID [ATTACHMENT_IDS]", options)
287
288     def _fetch_list_of_patches_to_process(self, options, args, tool):
289         return map(lambda patch_id: tool.bugs.fetch_attachment(patch_id), args)
290
291     def _prepare_to_process(self, options, args, tool):
292         # Check the tree status first so we can fail early.
293         tool.steps.ensure_builders_are_green(tool.buildbot, options)
294
295     def _process_patch(self, patch, options, args, tool):
296         sequence = BuildAttachmentSequence(patch, options, tool)
297         sequence.run_and_handle_errors()
298
299
300 class AbstractPatchLandingCommand(AbstractPatchProcessingCommand):
301     def __init__(self, help_text, args_description):
302         options = BuildSteps.cleaning_options()
303         options += BuildSteps.build_options()
304         options += BuildSteps.land_options()
305         AbstractPatchProcessingCommand.__init__(self, help_text, args_description, options)
306
307     def _prepare_to_process(self, options, args, tool):
308         # Check the tree status first so we can fail early.
309         tool.steps.ensure_builders_are_green(tool.buildbot, options)
310
311     def _process_patch(self, patch, options, args, tool):
312         sequence = ConditionalLandingSequence(patch, options, tool)
313         sequence.run_and_handle_errors()
314
315
316 class LandAttachment(AbstractPatchLandingCommand):
317     name = "land-attachment"
318     show_in_main_help = True
319     def __init__(self):
320         AbstractPatchLandingCommand.__init__(self, "Land patches from bugzilla, optionally building and testing them first", "ATTACHMENT_ID [ATTACHMENT_IDS]")
321
322     def _fetch_list_of_patches_to_process(self, options, args, tool):
323         return map(lambda patch_id: tool.bugs.fetch_attachment(patch_id), args)
324
325
326 class LandPatches(AbstractPatchLandingCommand):
327     name = "land-patches"
328     show_in_main_help = True
329     def __init__(self):
330         AbstractPatchLandingCommand.__init__(self, "Land all patches on the given bugs, optionally building and testing them first", "BUGID [BUGIDS]")
331
332     def _fetch_list_of_patches_to_process(self, options, args, tool):
333         all_patches = []
334         for bug_id in args:
335             patches = tool.bugs.fetch_reviewed_patches_from_bug(bug_id)
336             log("%s found on bug %s." % (pluralize("reviewed patch", len(patches)), bug_id))
337             all_patches += patches
338         return all_patches
339
340
341 # FIXME: Requires unit test.
342 class Rollout(Command):
343     name = "rollout"
344     show_in_main_help = True
345     def __init__(self):
346         options = BuildSteps.cleaning_options()
347         options += BuildSteps.build_options()
348         options += BuildSteps.land_options()
349         options.append(make_option("--complete-rollout", action="store_true", dest="complete_rollout", help="Commit the revert and re-open the original bug."))
350         Command.__init__(self, "Revert the given revision in the working copy and optionally commit the revert and re-open the original bug", "REVISION [BUGID]", options=options)
351
352     @staticmethod
353     def _create_changelogs_for_revert(tool, revision):
354         # First, discard the ChangeLog changes from the rollout.
355         changelog_paths = tool.scm().modified_changelogs()
356         tool.scm().revert_files(changelog_paths)
357
358         # Second, make new ChangeLog entries for this rollout.
359         # This could move to prepare-ChangeLog by adding a --revert= option.
360         tool.steps.prepare_changelog()
361         for changelog_path in changelog_paths:
362             ChangeLog(changelog_path).update_for_revert(revision)
363
364     @staticmethod
365     def _parse_bug_id_from_revision_diff(tool, revision):
366         original_diff = tool.scm().diff_for_revision(revision)
367         return parse_bug_id(original_diff)
368
369     @staticmethod
370     def _reopen_bug_after_rollout(tool, bug_id, comment_text):
371         if bug_id:
372             tool.bugs.reopen_bug(bug_id, comment_text)
373         else:
374             log(comment_text)
375             log("No bugs were updated or re-opened to reflect this rollout.")
376
377     def execute(self, options, args, tool):
378         revision = args[0]
379         bug_id = self._parse_bug_id_from_revision_diff(tool, revision)
380         if options.complete_rollout:
381             if bug_id:
382                 log("Will re-open bug %s after rollout." % bug_id)
383             else:
384                 log("Failed to parse bug number from diff.  No bugs will be updated/reopened after the rollout.")
385
386         tool.steps.clean_working_directory(tool.scm(), options)
387         tool.scm().update_webkit()
388         tool.scm().apply_reverse_diff(revision)
389         self._create_changelogs_for_revert(tool, revision)
390
391         # FIXME: Fully automated rollout is not 100% idiot-proof yet, so for now just log with instructions on how to complete the rollout.
392         # Once we trust rollout we will remove this option.
393         if not options.complete_rollout:
394             log("\nNOTE: Rollout support is experimental.\nPlease verify the rollout diff and use \"bugzilla-tool land-diff %s\" to commit the rollout." % bug_id)
395         else:
396             # FIXME: This function does not exist!!
397             # comment_text = WebKitLandingScripts.build_and_commit(tool.scm(), options)
398             raise ScriptError("OOPS! This option is not implemented (yet).")
399             self._reopen_bug_after_rollout(tool, bug_id, comment_text)