2009-12-02 Eric Seidel <eric@webkit.org>
[WebKit-https.git] / WebKitTools / Scripts / modules / commands / download.py
1 #!/usr/bin/env python
2 # Copyright (c) 2009, Google Inc. All rights reserved.
3 # Copyright (c) 2009 Apple Inc. All rights reserved.
4 #
5 # Redistribution and use in source and binary forms, with or without
6 # modification, are permitted provided that the following conditions are
7 # met:
8
9 #     * Redistributions of source code must retain the above copyright
10 # notice, this list of conditions and the following disclaimer.
11 #     * Redistributions in binary form must reproduce the above
12 # copyright notice, this list of conditions and the following disclaimer
13 # in the documentation and/or other materials provided with the
14 # distribution.
15 #     * Neither the name of Google Inc. nor the names of its
16 # contributors may be used to endorse or promote products derived from
17 # this software without specific prior written permission.
18
19 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30
31 import os
32
33 from optparse import make_option
34
35 from modules.bugzilla import parse_bug_id
36 from modules.buildsteps import BuildSteps
37 from modules.changelogs import ChangeLog
38 from modules.comments import bug_comment_from_commit_text
39 from modules.grammar import pluralize
40 from modules.landingsequence import LandingSequence, ConditionalLandingSequence
41 from modules.logging import error, log
42 from modules.multicommandtool import Command
43 from modules.scm import ScriptError
44
45
46 class BuildSequence(ConditionalLandingSequence):
47     def __init__(self, options, tool):
48         ConditionalLandingSequence.__init__(self, None, options, tool)
49
50     def run(self):
51         self.clean()
52         self.update()
53         self.build()
54
55
56 class Build(Command):
57     name = "build"
58     show_in_main_help = False
59     def __init__(self):
60         options = BuildSteps.cleaning_options()
61         options += BuildSteps.build_options()
62         options += BuildSteps.land_options()
63         Command.__init__(self, "Update working copy and build", "", options)
64
65     def execute(self, options, args, tool):
66         sequence = BuildSequence(options, tool)
67         sequence.run_and_handle_errors()
68
69
70 class ApplyAttachment(Command):
71     name = "apply-attachment"
72     show_in_main_help = True
73     def __init__(self):
74         options = WebKitApplyingScripts.apply_options()
75         options += BuildSteps.cleaning_options()
76         Command.__init__(self, "Apply an attachment to the local working directory", "ATTACHMENT_ID", options=options)
77
78     def execute(self, options, args, tool):
79         WebKitApplyingScripts.setup_for_patch_apply(tool, options)
80         attachment_id = args[0]
81         attachment = tool.bugs.fetch_attachment(attachment_id)
82         WebKitApplyingScripts.apply_patches_with_options(tool.scm(), [attachment], options)
83
84
85 class ApplyPatches(Command):
86     name = "apply-patches"
87     show_in_main_help = True
88     def __init__(self):
89         options = WebKitApplyingScripts.apply_options()
90         options += BuildSteps.cleaning_options()
91         Command.__init__(self, "Apply reviewed patches from provided bugs to the local working directory", "BUGID", options=options)
92
93     def execute(self, options, args, tool):
94         WebKitApplyingScripts.setup_for_patch_apply(tool, options)
95         bug_id = args[0]
96         patches = tool.bugs.fetch_reviewed_patches_from_bug(bug_id)
97         WebKitApplyingScripts.apply_patches_with_options(tool.scm(), patches, options)
98
99
100 class WebKitApplyingScripts:
101     @staticmethod
102     def apply_options():
103         return [
104             make_option("--no-update", action="store_false", dest="update", default=True, help="Don't update the working directory before applying patches"),
105             make_option("--local-commit", action="store_true", dest="local_commit", default=False, help="Make a local commit for each applied patch"),
106         ]
107
108     @staticmethod
109     def setup_for_patch_apply(tool, options):
110         tool.steps.clean_working_directory(tool.scm(), options, allow_local_commits=True)
111         if options.update:
112             tool.scm().update_webkit()
113
114     @staticmethod
115     def apply_patches_with_options(scm, patches, options):
116         if options.local_commit and not scm.supports_local_commits():
117             error("--local-commit passed, but %s does not support local commits" % scm.display_name())
118
119         for patch in patches:
120             log("Applying attachment %s from bug %s" % (patch["id"], patch["bug_id"]))
121             scm.apply_patch(patch)
122             if options.local_commit:
123                 commit_message = scm.commit_message_for_this_commit()
124                 scm.commit_locally_with_message(commit_message.message() or patch["name"])
125
126
127 class LandDiffSequence(ConditionalLandingSequence):
128     def __init__(self, patch, options, tool):
129         ConditionalLandingSequence.__init__(self, patch, options, tool)
130
131     def run(self):
132         self.build()
133         self.test()
134         commit_log = self.commit()
135         self.close_bug(commit_log)
136
137     def close_bug(self, commit_log):
138         comment_test = bug_comment_from_commit_text(self._tool.scm(), commit_log)
139         bug_id = self._patch["bug_id"]
140         if bug_id:
141             log("Updating bug %s" % bug_id)
142             if self._options.close_bug:
143                 self._tool.bugs.close_bug_as_fixed(bug_id, comment_test)
144             else:
145                 # FIXME: We should a smart way to figure out if the patch is attached
146                 # to the bug, and if so obsolete it.
147                 self._tool.bugs.post_comment_to_bug(bug_id, comment_test)
148         else:
149             log(comment_test)
150             log("No bug id provided.")
151
152
153 class LandDiff(Command):
154     name = "land-diff"
155     show_in_main_help = True
156     def __init__(self):
157         options = [
158             make_option("-r", "--reviewer", action="store", type="string", dest="reviewer", help="Update ChangeLogs to say Reviewed by REVIEWER."),
159         ]
160         options += BuildSteps.build_options()
161         options += BuildSteps.land_options()
162         Command.__init__(self, "Land the current working directory diff and updates the associated bug if any", "[BUGID]", options=options)
163
164     def guess_reviewer_from_bug(self, bugs, bug_id):
165         patches = bugs.fetch_reviewed_patches_from_bug(bug_id)
166         if len(patches) != 1:
167             log("%s on bug %s, cannot infer reviewer." % (pluralize("reviewed patch", len(patches)), bug_id))
168             return None
169         patch = patches[0]
170         reviewer = patch["reviewer"]
171         log("Guessing \"%s\" as reviewer from attachment %s on bug %s." % (reviewer, patch["id"], bug_id))
172         return reviewer
173
174     def update_changelogs_with_reviewer(self, reviewer, bug_id, tool):
175         if not reviewer:
176             if not bug_id:
177                 log("No bug id provided and --reviewer= not provided.  Not updating ChangeLogs with reviewer.")
178                 return
179             reviewer = self.guess_reviewer_from_bug(tool.bugs, bug_id)
180
181         if not reviewer:
182             log("Failed to guess reviewer from bug %s and --reviewer= not provided.  Not updating ChangeLogs with reviewer." % bug_id)
183             return
184
185         for changelog_path in tool.scm().modified_changelogs():
186             ChangeLog(changelog_path).set_reviewer(reviewer)
187
188     def execute(self, options, args, tool):
189         bug_id = (args and args[0]) or parse_bug_id(tool.scm().create_patch())
190
191         tool.steps.ensure_builders_are_green(tool.buildbot, options)
192
193         os.chdir(tool.scm().checkout_root)
194         self.update_changelogs_with_reviewer(options.reviewer, bug_id, tool)
195
196         fake_patch = {
197             "id": None,
198             "bug_id": bug_id
199         }
200
201         sequence = LandDiffSequence(fake_patch, options, tool)
202         sequence.run()
203
204
205 class AbstractPatchProcessingCommand(Command):
206     def __init__(self, help_text, args_description, options):
207         Command.__init__(self, help_text, args_description, options=options)
208
209     def _fetch_list_of_patches_to_process(self, options, args, tool):
210         raise NotImplementedError, "subclasses must implement"
211
212     def _prepare_to_process(self, options, args, tool):
213         raise NotImplementedError, "subclasses must implement"
214
215     @staticmethod
216     def _collect_patches_by_bug(patches):
217         bugs_to_patches = {}
218         for patch in patches:
219             bug_id = patch["bug_id"]
220             bugs_to_patches[bug_id] = bugs_to_patches.get(bug_id, []) + [patch]
221         return bugs_to_patches
222
223     def execute(self, options, args, tool):
224         self._prepare_to_process(options, args, tool)
225         patches = self._fetch_list_of_patches_to_process(options, args, tool)
226
227         # It's nice to print out total statistics.
228         bugs_to_patches = self._collect_patches_by_bug(patches)
229         log("Processing %s from %s." % (pluralize("patch", len(patches)), pluralize("bug", len(bugs_to_patches))))
230
231         for patch in patches:
232             self._process_patch(patch, options, args, tool)
233
234
235 class CheckStyleSequence(LandingSequence):
236     def __init__(self, patch, options, tool):
237         LandingSequence.__init__(self, patch, options, tool)
238
239     def run(self):
240         self.clean()
241         self.update()
242         self.apply_patch()
243         self.build()
244
245     def build(self):
246         # Instead of building, we check style.
247         self._tool.steps.check_style()
248
249
250 class CheckStyle(AbstractPatchProcessingCommand):
251     name = "check-style"
252     show_in_main_help = False
253     def __init__(self):
254         options = BuildSteps.cleaning_options()
255         options += BuildSteps.build_options()
256         AbstractPatchProcessingCommand.__init__(self, "Run check-webkit-style on the specified attachments", "ATTACHMENT_ID [ATTACHMENT_IDS]", options)
257
258     def _fetch_list_of_patches_to_process(self, options, args, tool):
259         return map(lambda patch_id: tool.bugs.fetch_attachment(patch_id), args)
260
261     def _prepare_to_process(self, options, args, tool):
262         pass
263
264     def _process_patch(self, patch, options, args, tool):
265         sequence = CheckStyleSequence(patch, options, tool)
266         sequence.run_and_handle_errors()
267
268
269 class BuildAttachmentSequence(ConditionalLandingSequence):
270     def __init__(self, patch, options, tool):
271         LandingSequence.__init__(self, patch, options, tool)
272
273     def run(self):
274         self.clean()
275         self.update()
276         self.apply_patch()
277         self.build()
278
279
280 class BuildAttachment(AbstractPatchProcessingCommand):
281     name = "build-attachment"
282     show_in_main_help = False
283     def __init__(self):
284         options = BuildSteps.cleaning_options()
285         options += BuildSteps.build_options()
286         options += BuildSteps.land_options()
287         AbstractPatchProcessingCommand.__init__(self, "Apply and build patches from bugzilla", "ATTACHMENT_ID [ATTACHMENT_IDS]", options)
288
289     def _fetch_list_of_patches_to_process(self, options, args, tool):
290         return map(lambda patch_id: tool.bugs.fetch_attachment(patch_id), args)
291
292     def _prepare_to_process(self, options, args, tool):
293         # Check the tree status first so we can fail early.
294         tool.steps.ensure_builders_are_green(tool.buildbot, options)
295
296     def _process_patch(self, patch, options, args, tool):
297         sequence = BuildAttachmentSequence(patch, options, tool)
298         sequence.run_and_handle_errors()
299
300
301 class AbstractPatchLandingCommand(AbstractPatchProcessingCommand):
302     def __init__(self, help_text, args_description):
303         options = BuildSteps.cleaning_options()
304         options += BuildSteps.build_options()
305         options += BuildSteps.land_options()
306         AbstractPatchProcessingCommand.__init__(self, help_text, args_description, options)
307
308     def _prepare_to_process(self, options, args, tool):
309         # Check the tree status first so we can fail early.
310         tool.steps.ensure_builders_are_green(tool.buildbot, options)
311
312     def _process_patch(self, patch, options, args, tool):
313         sequence = ConditionalLandingSequence(patch, options, tool)
314         sequence.run_and_handle_errors()
315
316
317 class LandAttachment(AbstractPatchLandingCommand):
318     name = "land-attachment"
319     show_in_main_help = True
320     def __init__(self):
321         AbstractPatchLandingCommand.__init__(self, "Land patches from bugzilla, optionally building and testing them first", "ATTACHMENT_ID [ATTACHMENT_IDS]")
322
323     def _fetch_list_of_patches_to_process(self, options, args, tool):
324         return map(lambda patch_id: tool.bugs.fetch_attachment(patch_id), args)
325
326
327 class LandPatches(AbstractPatchLandingCommand):
328     name = "land-patches"
329     show_in_main_help = True
330     def __init__(self):
331         AbstractPatchLandingCommand.__init__(self, "Land all patches on the given bugs, optionally building and testing them first", "BUGID [BUGIDS]")
332
333     def _fetch_list_of_patches_to_process(self, options, args, tool):
334         all_patches = []
335         for bug_id in args:
336             patches = tool.bugs.fetch_reviewed_patches_from_bug(bug_id)
337             log("%s found on bug %s." % (pluralize("reviewed patch", len(patches)), bug_id))
338             all_patches += patches
339         return all_patches
340
341
342 # FIXME: Requires unit test.
343 class Rollout(Command):
344     name = "rollout"
345     show_in_main_help = True
346     def __init__(self):
347         options = BuildSteps.cleaning_options()
348         options += BuildSteps.build_options()
349         options += BuildSteps.land_options()
350         options.append(make_option("--complete-rollout", action="store_true", dest="complete_rollout", help="Commit the revert and re-open the original bug."))
351         Command.__init__(self, "Revert the given revision in the working copy and optionally commit the revert and re-open the original bug", "REVISION [BUGID]", options=options)
352
353     @staticmethod
354     def _create_changelogs_for_revert(tool, revision):
355         # First, discard the ChangeLog changes from the rollout.
356         changelog_paths = tool.scm().modified_changelogs()
357         tool.scm().revert_files(changelog_paths)
358
359         # Second, make new ChangeLog entries for this rollout.
360         # This could move to prepare-ChangeLog by adding a --revert= option.
361         tool.steps.prepare_changelog()
362         for changelog_path in changelog_paths:
363             ChangeLog(changelog_path).update_for_revert(revision)
364
365     @staticmethod
366     def _parse_bug_id_from_revision_diff(tool, revision):
367         original_diff = tool.scm().diff_for_revision(revision)
368         return parse_bug_id(original_diff)
369
370     @staticmethod
371     def _reopen_bug_after_rollout(tool, bug_id, comment_text):
372         if bug_id:
373             tool.bugs.reopen_bug(bug_id, comment_text)
374         else:
375             log(comment_text)
376             log("No bugs were updated or re-opened to reflect this rollout.")
377
378     def execute(self, options, args, tool):
379         revision = args[0]
380         bug_id = self._parse_bug_id_from_revision_diff(tool, revision)
381         if options.complete_rollout:
382             if bug_id:
383                 log("Will re-open bug %s after rollout." % bug_id)
384             else:
385                 log("Failed to parse bug number from diff.  No bugs will be updated/reopened after the rollout.")
386
387         tool.steps.clean_working_directory(tool.scm(), options)
388         tool.scm().update_webkit()
389         tool.scm().apply_reverse_diff(revision)
390         self._create_changelogs_for_revert(tool, revision)
391
392         # FIXME: Fully automated rollout is not 100% idiot-proof yet, so for now just log with instructions on how to complete the rollout.
393         # Once we trust rollout we will remove this option.
394         if not options.complete_rollout:
395             log("\nNOTE: Rollout support is experimental.\nPlease verify the rollout diff and use \"bugzilla-tool land-diff %s\" to commit the rollout." % bug_id)
396         else:
397             # FIXME: This function does not exist!!
398             # comment_text = WebKitLandingScripts.build_and_commit(tool.scm(), options)
399             raise ScriptError("OOPS! This option is not implemented (yet).")
400             self._reopen_bug_after_rollout(tool, bug_id, comment_text)