483219a3f6d93b0647c998fb14bfc64f15b7deec
[WebKit-https.git] / WebKitTools / Scripts / modules / commands / download.py
1 #!/usr/bin/env python
2 # Copyright (c) 2009, Google Inc. All rights reserved.
3 # Copyright (c) 2009 Apple Inc. All rights reserved.
4 #
5 # Redistribution and use in source and binary forms, with or without
6 # modification, are permitted provided that the following conditions are
7 # met:
8
9 #     * Redistributions of source code must retain the above copyright
10 # notice, this list of conditions and the following disclaimer.
11 #     * Redistributions in binary form must reproduce the above
12 # copyright notice, this list of conditions and the following disclaimer
13 # in the documentation and/or other materials provided with the
14 # distribution.
15 #     * Neither the name of Google Inc. nor the names of its
16 # contributors may be used to endorse or promote products derived from
17 # this software without specific prior written permission.
18
19 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30
31 import os
32
33 from optparse import make_option
34
35 from modules.bugzilla import parse_bug_id
36 from modules.buildsteps import BuildSteps
37 from modules.changelogs import ChangeLog
38 from modules.comments import bug_comment_from_commit_text
39 from modules.grammar import pluralize
40 from modules.landingsequence import LandingSequence, ConditionalLandingSequence
41 from modules.logging import error, log
42 from modules.multicommandtool import Command
43 from modules.scm import ScriptError
44
45
46 class BuildSequence(ConditionalLandingSequence):
47     def __init__(self, options, tool):
48         ConditionalLandingSequence.__init__(self, None, options, tool)
49
50     def run(self):
51         self.clean()
52         self.update()
53         self.build()
54
55
56 class Build(Command):
57     name = "build"
58     show_in_main_help = False
59     def __init__(self):
60         options = BuildSteps.cleaning_options()
61         options += BuildSteps.build_options()
62         options += BuildSteps.land_options()
63         Command.__init__(self, "Update working copy and build", "", options)
64
65     def execute(self, options, args, tool):
66         sequence = BuildSequence(options, tool)
67         sequence.run_and_handle_errors()
68
69
70 class ApplyAttachment(Command):
71     name = "apply-attachment"
72     show_in_main_help = True
73     def __init__(self):
74         options = WebKitApplyingScripts.apply_options()
75         options += BuildSteps.cleaning_options()
76         Command.__init__(self, "Apply an attachment to the local working directory", "ATTACHMENT_ID", options=options)
77
78     def execute(self, options, args, tool):
79         WebKitApplyingScripts.setup_for_patch_apply(tool, options)
80         attachment_id = args[0]
81         attachment = tool.bugs.fetch_attachment(attachment_id)
82         WebKitApplyingScripts.apply_patches_with_options(tool.scm(), [attachment], options)
83
84
85 class ApplyPatches(Command):
86     name = "apply-patches"
87     show_in_main_help = True
88     def __init__(self):
89         options = WebKitApplyingScripts.apply_options()
90         options += BuildSteps.cleaning_options()
91         Command.__init__(self, "Apply reviewed patches from provided bugs to the local working directory", "BUGID", options=options)
92
93     def execute(self, options, args, tool):
94         WebKitApplyingScripts.setup_for_patch_apply(tool, options)
95         bug_id = args[0]
96         patches = tool.bugs.fetch_reviewed_patches_from_bug(bug_id)
97         WebKitApplyingScripts.apply_patches_with_options(tool.scm(), patches, options)
98
99
100 class WebKitApplyingScripts:
101     @staticmethod
102     def apply_options():
103         return [
104             make_option("--no-update", action="store_false", dest="update", default=True, help="Don't update the working directory before applying patches"),
105             make_option("--local-commit", action="store_true", dest="local_commit", default=False, help="Make a local commit for each applied patch"),
106         ]
107
108     @staticmethod
109     def setup_for_patch_apply(tool, options):
110         tool.steps.clean_working_directory(tool.scm(), options, allow_local_commits=True)
111         if options.update:
112             tool.steps.update()
113
114     @staticmethod
115     def apply_patches_with_options(scm, patches, options):
116         if options.local_commit and not scm.supports_local_commits():
117             error("--local-commit passed, but %s does not support local commits" % scm.display_name())
118
119         for patch in patches:
120             log("Applying attachment %s from bug %s" % (patch["id"], patch["bug_id"]))
121             scm.apply_patch(patch)
122             if options.local_commit:
123                 commit_message = scm.commit_message_for_this_commit()
124                 scm.commit_locally_with_message(commit_message.message() or patch["name"])
125
126
127 class LandDiffSequence(ConditionalLandingSequence):
128     def __init__(self, patch, options, tool):
129         ConditionalLandingSequence.__init__(self, patch, options, tool)
130
131     def run(self):
132         self.check_builders()
133         self.build()
134         self.test()
135         commit_log = self.commit()
136         self.close_bug(commit_log)
137
138     def close_bug(self, commit_log):
139         comment_test = bug_comment_from_commit_text(self._tool.scm(), commit_log)
140         bug_id = self._patch["bug_id"]
141         if bug_id:
142             log("Updating bug %s" % bug_id)
143             if self._options.close_bug:
144                 self._tool.bugs.close_bug_as_fixed(bug_id, comment_test)
145             else:
146                 # FIXME: We should a smart way to figure out if the patch is attached
147                 # to the bug, and if so obsolete it.
148                 self._tool.bugs.post_comment_to_bug(bug_id, comment_test)
149         else:
150             log(comment_test)
151             log("No bug id provided.")
152
153
154 class LandDiff(Command):
155     name = "land-diff"
156     show_in_main_help = True
157     def __init__(self):
158         options = [
159             make_option("-r", "--reviewer", action="store", type="string", dest="reviewer", help="Update ChangeLogs to say Reviewed by REVIEWER."),
160         ]
161         options += BuildSteps.build_options()
162         options += BuildSteps.land_options()
163         Command.__init__(self, "Land the current working directory diff and updates the associated bug if any", "[BUGID]", options=options)
164
165     def guess_reviewer_from_bug(self, bugs, bug_id):
166         patches = bugs.fetch_reviewed_patches_from_bug(bug_id)
167         if len(patches) != 1:
168             log("%s on bug %s, cannot infer reviewer." % (pluralize("reviewed patch", len(patches)), bug_id))
169             return None
170         patch = patches[0]
171         reviewer = patch["reviewer"]
172         log("Guessing \"%s\" as reviewer from attachment %s on bug %s." % (reviewer, patch["id"], bug_id))
173         return reviewer
174
175     def update_changelogs_with_reviewer(self, reviewer, bug_id, tool):
176         if not reviewer:
177             if not bug_id:
178                 log("No bug id provided and --reviewer= not provided.  Not updating ChangeLogs with reviewer.")
179                 return
180             reviewer = self.guess_reviewer_from_bug(tool.bugs, bug_id)
181
182         if not reviewer:
183             log("Failed to guess reviewer from bug %s and --reviewer= not provided.  Not updating ChangeLogs with reviewer." % bug_id)
184             return
185
186         for changelog_path in tool.scm().modified_changelogs():
187             ChangeLog(changelog_path).set_reviewer(reviewer)
188
189     def execute(self, options, args, tool):
190         bug_id = (args and args[0]) or parse_bug_id(tool.scm().create_patch())
191
192         tool.steps.ensure_builders_are_green(tool.buildbot, options)
193
194         os.chdir(tool.scm().checkout_root)
195         self.update_changelogs_with_reviewer(options.reviewer, bug_id, tool)
196
197         fake_patch = {
198             "id": None,
199             "bug_id": bug_id
200         }
201
202         sequence = LandDiffSequence(fake_patch, options, tool)
203         sequence.run()
204
205
206 class AbstractPatchProcessingCommand(Command):
207     def __init__(self, help_text, args_description, options):
208         Command.__init__(self, help_text, args_description, options=options)
209
210     def _fetch_list_of_patches_to_process(self, options, args, tool):
211         raise NotImplementedError, "subclasses must implement"
212
213     def _prepare_to_process(self, options, args, tool):
214         raise NotImplementedError, "subclasses must implement"
215
216     @staticmethod
217     def _collect_patches_by_bug(patches):
218         bugs_to_patches = {}
219         for patch in patches:
220             bug_id = patch["bug_id"]
221             bugs_to_patches[bug_id] = bugs_to_patches.get(bug_id, []) + [patch]
222         return bugs_to_patches
223
224     def execute(self, options, args, tool):
225         self._prepare_to_process(options, args, tool)
226         patches = self._fetch_list_of_patches_to_process(options, args, tool)
227
228         # It's nice to print out total statistics.
229         bugs_to_patches = self._collect_patches_by_bug(patches)
230         log("Processing %s from %s." % (pluralize("patch", len(patches)), pluralize("bug", len(bugs_to_patches))))
231
232         for patch in patches:
233             self._process_patch(patch, options, args, tool)
234
235
236 class CheckStyleSequence(LandingSequence):
237     def __init__(self, patch, options, tool):
238         LandingSequence.__init__(self, patch, options, tool)
239
240     def run(self):
241         self.clean()
242         self.update()
243         self.apply_patch()
244         self.build()
245
246     def build(self):
247         # Instead of building, we check style.
248         self._tool.steps.check_style()
249
250
251 class CheckStyle(AbstractPatchProcessingCommand):
252     name = "check-style"
253     show_in_main_help = False
254     def __init__(self):
255         options = BuildSteps.cleaning_options()
256         options += BuildSteps.build_options()
257         AbstractPatchProcessingCommand.__init__(self, "Run check-webkit-style on the specified attachments", "ATTACHMENT_ID [ATTACHMENT_IDS]", options)
258
259     def _fetch_list_of_patches_to_process(self, options, args, tool):
260         return map(lambda patch_id: tool.bugs.fetch_attachment(patch_id), args)
261
262     def _prepare_to_process(self, options, args, tool):
263         pass
264
265     def _process_patch(self, patch, options, args, tool):
266         sequence = CheckStyleSequence(patch, options, tool)
267         sequence.run_and_handle_errors()
268
269
270 class BuildAttachmentSequence(ConditionalLandingSequence):
271     def __init__(self, patch, options, tool):
272         LandingSequence.__init__(self, patch, options, tool)
273
274     def run(self):
275         self.clean()
276         self.update()
277         self.apply_patch()
278         self.build()
279
280
281 class BuildAttachment(AbstractPatchProcessingCommand):
282     name = "build-attachment"
283     show_in_main_help = False
284     def __init__(self):
285         options = BuildSteps.cleaning_options()
286         options += BuildSteps.build_options()
287         options += BuildSteps.land_options()
288         AbstractPatchProcessingCommand.__init__(self, "Apply and build patches from bugzilla", "ATTACHMENT_ID [ATTACHMENT_IDS]", options)
289
290     def _fetch_list_of_patches_to_process(self, options, args, tool):
291         return map(lambda patch_id: tool.bugs.fetch_attachment(patch_id), args)
292
293     def _prepare_to_process(self, options, args, tool):
294         # Check the tree status first so we can fail early.
295         tool.steps.ensure_builders_are_green(tool.buildbot, options)
296
297     def _process_patch(self, patch, options, args, tool):
298         sequence = BuildAttachmentSequence(patch, options, tool)
299         sequence.run_and_handle_errors()
300
301
302 class AbstractPatchLandingCommand(AbstractPatchProcessingCommand):
303     def __init__(self, help_text, args_description):
304         options = BuildSteps.cleaning_options()
305         options += BuildSteps.build_options()
306         options += BuildSteps.land_options()
307         AbstractPatchProcessingCommand.__init__(self, help_text, args_description, options)
308
309     def _prepare_to_process(self, options, args, tool):
310         # Check the tree status first so we can fail early.
311         tool.steps.ensure_builders_are_green(tool.buildbot, options)
312
313     def _process_patch(self, patch, options, args, tool):
314         sequence = ConditionalLandingSequence(patch, options, tool)
315         sequence.run_and_handle_errors()
316
317
318 class LandAttachment(AbstractPatchLandingCommand):
319     name = "land-attachment"
320     show_in_main_help = True
321     def __init__(self):
322         AbstractPatchLandingCommand.__init__(self, "Land patches from bugzilla, optionally building and testing them first", "ATTACHMENT_ID [ATTACHMENT_IDS]")
323
324     def _fetch_list_of_patches_to_process(self, options, args, tool):
325         return map(lambda patch_id: tool.bugs.fetch_attachment(patch_id), args)
326
327
328 class LandPatches(AbstractPatchLandingCommand):
329     name = "land-patches"
330     show_in_main_help = True
331     def __init__(self):
332         AbstractPatchLandingCommand.__init__(self, "Land all patches on the given bugs, optionally building and testing them first", "BUGID [BUGIDS]")
333
334     def _fetch_list_of_patches_to_process(self, options, args, tool):
335         all_patches = []
336         for bug_id in args:
337             patches = tool.bugs.fetch_reviewed_patches_from_bug(bug_id)
338             log("%s found on bug %s." % (pluralize("reviewed patch", len(patches)), bug_id))
339             all_patches += patches
340         return all_patches
341
342
343 # FIXME: Requires unit test.
344 class Rollout(Command):
345     name = "rollout"
346     show_in_main_help = True
347     def __init__(self):
348         options = BuildSteps.cleaning_options()
349         options += BuildSteps.build_options()
350         options += BuildSteps.land_options()
351         options.append(make_option("--complete-rollout", action="store_true", dest="complete_rollout", help="Commit the revert and re-open the original bug."))
352         Command.__init__(self, "Revert the given revision in the working copy and optionally commit the revert and re-open the original bug", "REVISION [BUGID]", options=options)
353
354     @staticmethod
355     def _create_changelogs_for_revert(tool, revision):
356         # First, discard the ChangeLog changes from the rollout.
357         changelog_paths = tool.scm().modified_changelogs()
358         tool.scm().revert_files(changelog_paths)
359
360         # Second, make new ChangeLog entries for this rollout.
361         # This could move to prepare-ChangeLog by adding a --revert= option.
362         tool.steps.prepare_changelog()
363         for changelog_path in changelog_paths:
364             ChangeLog(changelog_path).update_for_revert(revision)
365
366     @staticmethod
367     def _parse_bug_id_from_revision_diff(tool, revision):
368         original_diff = tool.scm().diff_for_revision(revision)
369         return parse_bug_id(original_diff)
370
371     @staticmethod
372     def _reopen_bug_after_rollout(tool, bug_id, comment_text):
373         if bug_id:
374             tool.bugs.reopen_bug(bug_id, comment_text)
375         else:
376             log(comment_text)
377             log("No bugs were updated or re-opened to reflect this rollout.")
378
379     def execute(self, options, args, tool):
380         revision = args[0]
381         bug_id = self._parse_bug_id_from_revision_diff(tool, revision)
382         if options.complete_rollout:
383             if bug_id:
384                 log("Will re-open bug %s after rollout." % bug_id)
385             else:
386                 log("Failed to parse bug number from diff.  No bugs will be updated/reopened after the rollout.")
387
388         tool.steps.clean_working_directory(tool.scm(), options)
389         tool.steps.update()
390         tool.scm().apply_reverse_diff(revision)
391         self._create_changelogs_for_revert(tool, revision)
392
393         # FIXME: Fully automated rollout is not 100% idiot-proof yet, so for now just log with instructions on how to complete the rollout.
394         # Once we trust rollout we will remove this option.
395         if not options.complete_rollout:
396             log("\nNOTE: Rollout support is experimental.\nPlease verify the rollout diff and use \"bugzilla-tool land-diff %s\" to commit the rollout." % bug_id)
397         else:
398             # FIXME: This function does not exist!!
399             # comment_text = WebKitLandingScripts.build_and_commit(tool.scm(), options)
400             raise ScriptError("OOPS! This option is not implemented (yet).")
401             self._reopen_bug_after_rollout(tool, bug_id, comment_text)