31af13387fea786f62891334ae8a3ae229de1cc3
[WebKit-https.git] / WebKitTools / Scripts / modules / commands / download.py
1 #!/usr/bin/env python
2 # Copyright (c) 2009, Google Inc. All rights reserved.
3 # Copyright (c) 2009 Apple Inc. All rights reserved.
4 #
5 # Redistribution and use in source and binary forms, with or without
6 # modification, are permitted provided that the following conditions are
7 # met:
8
9 #     * Redistributions of source code must retain the above copyright
10 # notice, this list of conditions and the following disclaimer.
11 #     * Redistributions in binary form must reproduce the above
12 # copyright notice, this list of conditions and the following disclaimer
13 # in the documentation and/or other materials provided with the
14 # distribution.
15 #     * Neither the name of Google Inc. nor the names of its
16 # contributors may be used to endorse or promote products derived from
17 # this software without specific prior written permission.
18
19 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30
31 # FIXME: Trim down this import list once we have unit tests.
32 import os
33 import re
34 import StringIO
35 import subprocess
36 import sys
37 import time
38
39 from datetime import datetime, timedelta
40 from optparse import make_option
41
42 from modules.bugzilla import Bugzilla, parse_bug_id
43 from modules.buildbot import BuildBot
44 from modules.buildsteps import BuildSteps
45 from modules.changelogs import ChangeLog
46 from modules.comments import bug_comment_from_commit_text
47 from modules.grammar import pluralize
48 from modules.landingsequence import LandingSequence, ConditionalLandingSequence
49 from modules.logging import error, log, tee
50 from modules.multicommandtool import MultiCommandTool, Command
51 from modules.patchcollection import PatchCollection
52 from modules.scm import CommitMessage, detect_scm_system, ScriptError, CheckoutNeedsUpdate
53 from modules.statusbot import StatusBot
54 from modules.webkitport import WebKitPort
55 from modules.workqueue import WorkQueue, WorkQueueDelegate
56
57
58 class BuildSequence(ConditionalLandingSequence):
59     def __init__(self, options, tool):
60         ConditionalLandingSequence.__init__(self, None, options, tool)
61
62     def run(self):
63         self.clean()
64         self.update()
65         self.build()
66
67
68 class Build(Command):
69     name = "build"
70     show_in_main_help = False
71     def __init__(self):
72         options = BuildSteps.cleaning_options()
73         options += BuildSteps.build_options()
74         options += BuildSteps.land_options()
75         Command.__init__(self, "Update working copy and build", "", options)
76
77     def execute(self, options, args, tool):
78         sequence = BuildSequence(options, tool)
79         sequence.run_and_handle_errors()
80
81
82 class ApplyAttachment(Command):
83     name = "apply-attachment"
84     show_in_main_help = True
85     def __init__(self):
86         options = WebKitApplyingScripts.apply_options()
87         options += BuildSteps.cleaning_options()
88         Command.__init__(self, "Apply an attachment to the local working directory", "ATTACHMENT_ID", options=options)
89
90     def execute(self, options, args, tool):
91         WebKitApplyingScripts.setup_for_patch_apply(tool, options)
92         attachment_id = args[0]
93         attachment = tool.bugs.fetch_attachment(attachment_id)
94         WebKitApplyingScripts.apply_patches_with_options(tool.scm(), [attachment], options)
95
96
97 class ApplyPatches(Command):
98     name = "apply-patches"
99     show_in_main_help = True
100     def __init__(self):
101         options = WebKitApplyingScripts.apply_options()
102         options += BuildSteps.cleaning_options()
103         Command.__init__(self, "Apply reviewed patches from provided bugs to the local working directory", "BUGID", options=options)
104
105     def execute(self, options, args, tool):
106         WebKitApplyingScripts.setup_for_patch_apply(tool, options)
107         bug_id = args[0]
108         patches = tool.bugs.fetch_reviewed_patches_from_bug(bug_id)
109         WebKitApplyingScripts.apply_patches_with_options(tool.scm(), patches, options)
110
111
112 class WebKitApplyingScripts:
113     @staticmethod
114     def apply_options():
115         return [
116             make_option("--no-update", action="store_false", dest="update", default=True, help="Don't update the working directory before applying patches"),
117             make_option("--local-commit", action="store_true", dest="local_commit", default=False, help="Make a local commit for each applied patch"),
118         ]
119
120     @staticmethod
121     def setup_for_patch_apply(tool, options):
122         tool.steps.clean_working_directory(tool.scm(), options, allow_local_commits=True)
123         if options.update:
124             tool.scm().update_webkit()
125
126     @staticmethod
127     def apply_patches_with_options(scm, patches, options):
128         if options.local_commit and not scm.supports_local_commits():
129             error("--local-commit passed, but %s does not support local commits" % scm.display_name())
130
131         for patch in patches:
132             log("Applying attachment %s from bug %s" % (patch["id"], patch["bug_id"]))
133             scm.apply_patch(patch)
134             if options.local_commit:
135                 commit_message = scm.commit_message_for_this_commit()
136                 scm.commit_locally_with_message(commit_message.message() or patch["name"])
137
138
139 class LandDiffSequence(ConditionalLandingSequence):
140     def __init__(self, patch, options, tool):
141         ConditionalLandingSequence.__init__(self, patch, options, tool)
142
143     def run(self):
144         self.build()
145         self.test()
146         commit_log = self.commit()
147         self.close_bug(commit_log)
148
149     def close_bug(self, commit_log):
150         comment_test = bug_comment_from_commit_text(self._tool.scm(), commit_log)
151         bug_id = self._patch["bug_id"]
152         if bug_id:
153             log("Updating bug %s" % bug_id)
154             if self._options.close_bug:
155                 self._tool.bugs.close_bug_as_fixed(bug_id, comment_test)
156             else:
157                 # FIXME: We should a smart way to figure out if the patch is attached
158                 # to the bug, and if so obsolete it.
159                 self._tool.bugs.post_comment_to_bug(bug_id, comment_test)
160         else:
161             log(comment_test)
162             log("No bug id provided.")
163
164
165 class LandDiff(Command):
166     name = "land-diff"
167     show_in_main_help = True
168     def __init__(self):
169         options = [
170             make_option("-r", "--reviewer", action="store", type="string", dest="reviewer", help="Update ChangeLogs to say Reviewed by REVIEWER."),
171         ]
172         options += BuildSteps.build_options()
173         options += BuildSteps.land_options()
174         Command.__init__(self, "Land the current working directory diff and updates the associated bug if any", "[BUGID]", options=options)
175
176     def guess_reviewer_from_bug(self, bugs, bug_id):
177         patches = bugs.fetch_reviewed_patches_from_bug(bug_id)
178         if len(patches) != 1:
179             log("%s on bug %s, cannot infer reviewer." % (pluralize("reviewed patch", len(patches)), bug_id))
180             return None
181         patch = patches[0]
182         reviewer = patch["reviewer"]
183         log("Guessing \"%s\" as reviewer from attachment %s on bug %s." % (reviewer, patch["id"], bug_id))
184         return reviewer
185
186     def update_changelogs_with_reviewer(self, reviewer, bug_id, tool):
187         if not reviewer:
188             if not bug_id:
189                 log("No bug id provided and --reviewer= not provided.  Not updating ChangeLogs with reviewer.")
190                 return
191             reviewer = self.guess_reviewer_from_bug(tool.bugs, bug_id)
192
193         if not reviewer:
194             log("Failed to guess reviewer from bug %s and --reviewer= not provided.  Not updating ChangeLogs with reviewer." % bug_id)
195             return
196
197         for changelog_path in tool.scm().modified_changelogs():
198             ChangeLog(changelog_path).set_reviewer(reviewer)
199
200     def execute(self, options, args, tool):
201         bug_id = (args and args[0]) or parse_bug_id(tool.scm().create_patch())
202
203         tool.steps.ensure_builders_are_green(tool.buildbot, options)
204
205         os.chdir(tool.scm().checkout_root)
206         self.update_changelogs_with_reviewer(options.reviewer, bug_id, tool)
207
208         fake_patch = {
209             "id": None,
210             "bug_id": bug_id
211         }
212
213         sequence = LandDiffSequence(fake_patch, options, tool)
214         sequence.run()
215
216
217 class AbstractPatchProcessingCommand(Command):
218     def __init__(self, help_text, args_description, options):
219         Command.__init__(self, help_text, args_description, options=options)
220
221     def _fetch_list_of_patches_to_process(self, options, args, tool):
222         raise NotImplementedError, "subclasses must implement"
223
224     def _prepare_to_process(self, options, args, tool):
225         raise NotImplementedError, "subclasses must implement"
226
227     @staticmethod
228     def _collect_patches_by_bug(patches):
229         bugs_to_patches = {}
230         for patch in patches:
231             bug_id = patch["bug_id"]
232             bugs_to_patches[bug_id] = bugs_to_patches.get(bug_id, []) + [patch]
233         return bugs_to_patches
234
235     def execute(self, options, args, tool):
236         self._prepare_to_process(options, args, tool)
237         patches = self._fetch_list_of_patches_to_process(options, args, tool)
238
239         # It's nice to print out total statistics.
240         bugs_to_patches = self._collect_patches_by_bug(patches)
241         log("Processing %s from %s." % (pluralize("patch", len(patches)), pluralize("bug", len(bugs_to_patches))))
242
243         for patch in patches:
244             self._process_patch(patch, options, args, tool)
245
246
247 class CheckStyleSequence(LandingSequence):
248     def __init__(self, patch, options, tool):
249         LandingSequence.__init__(self, patch, options, tool)
250
251     def run(self):
252         self.clean()
253         self.update()
254         self.apply_patch()
255         self.build()
256
257     def build(self):
258         # Instead of building, we check style.
259         self._tool.steps.check_style()
260
261
262 class CheckStyle(AbstractPatchProcessingCommand):
263     name = "check-style"
264     show_in_main_help = False
265     def __init__(self):
266         options = BuildSteps.cleaning_options()
267         options += BuildSteps.build_options()
268         AbstractPatchProcessingCommand.__init__(self, "Run check-webkit-style on the specified attachments", "ATTACHMENT_ID [ATTACHMENT_IDS]", options)
269
270     def _fetch_list_of_patches_to_process(self, options, args, tool):
271         return map(lambda patch_id: tool.bugs.fetch_attachment(patch_id), args)
272
273     def _prepare_to_process(self, options, args, tool):
274         pass
275
276     def _process_patch(self, patch, options, args, tool):
277         sequence = CheckStyleSequence(patch, options, tool)
278         sequence.run_and_handle_errors()
279
280
281 class BuildAttachmentSequence(LandingSequence):
282     def __init__(self, patch, options, tool):
283         LandingSequence.__init__(self, patch, options, tool)
284
285     def run(self):
286         self.clean()
287         self.update()
288         self.apply_patch()
289         self.build()
290
291
292 class BuildAttachment(AbstractPatchProcessingCommand):
293     name = "build-attachment"
294     show_in_main_help = False
295     def __init__(self):
296         options = BuildSteps.cleaning_options()
297         options += BuildSteps.build_options()
298         AbstractPatchProcessingCommand.__init__(self, "Apply and build patches from bugzilla", "ATTACHMENT_ID [ATTACHMENT_IDS]", options)
299
300     def _fetch_list_of_patches_to_process(self, options, args, tool):
301         return map(lambda patch_id: tool.bugs.fetch_attachment(patch_id), args)
302
303     def _prepare_to_process(self, options, args, tool):
304         # Check the tree status first so we can fail early.
305         tool.steps.ensure_builders_are_green(tool.buildbot, options)
306
307     def _process_patch(self, patch, options, args, tool):
308         sequence = BuildAttachmentSequence(patch, options, tool)
309         sequence.run_and_handle_errors()
310
311
312 class AbstractPatchLandingCommand(AbstractPatchProcessingCommand):
313     def __init__(self, help_text, args_description):
314         options = BuildSteps.cleaning_options()
315         options += BuildSteps.build_options()
316         options += BuildSteps.land_options()
317         AbstractPatchProcessingCommand.__init__(self, help_text, args_description, options)
318
319     def _prepare_to_process(self, options, args, tool):
320         # Check the tree status first so we can fail early.
321         tool.steps.ensure_builders_are_green(tool.buildbot, options)
322
323     def _process_patch(self, patch, options, args, tool):
324         sequence = ConditionalLandingSequence(patch, options, tool)
325         sequence.run_and_handle_errors()
326
327
328 class LandAttachment(AbstractPatchLandingCommand):
329     name = "land-attachment"
330     show_in_main_help = True
331     def __init__(self):
332         AbstractPatchLandingCommand.__init__(self, "Land patches from bugzilla, optionally building and testing them first", "ATTACHMENT_ID [ATTACHMENT_IDS]")
333
334     def _fetch_list_of_patches_to_process(self, options, args, tool):
335         return map(lambda patch_id: tool.bugs.fetch_attachment(patch_id), args)
336
337
338 class LandPatches(AbstractPatchLandingCommand):
339     name = "land-patches"
340     show_in_main_help = True
341     def __init__(self):
342         AbstractPatchLandingCommand.__init__(self, "Land all patches on the given bugs, optionally building and testing them first", "BUGID [BUGIDS]")
343
344     def _fetch_list_of_patches_to_process(self, options, args, tool):
345         all_patches = []
346         for bug_id in args:
347             patches = tool.bugs.fetch_reviewed_patches_from_bug(bug_id)
348             log("%s found on bug %s." % (pluralize("reviewed patch", len(patches)), bug_id))
349             all_patches += patches
350         return all_patches
351
352
353 # FIXME: Requires unit test.
354 class Rollout(Command):
355     name = "rollout"
356     show_in_main_help = True
357     def __init__(self):
358         options = BuildSteps.cleaning_options()
359         options += BuildSteps.build_options()
360         options += BuildSteps.land_options()
361         options.append(make_option("--complete-rollout", action="store_true", dest="complete_rollout", help="Commit the revert and re-open the original bug."))
362         Command.__init__(self, "Revert the given revision in the working copy and optionally commit the revert and re-open the original bug", "REVISION [BUGID]", options=options)
363
364     @staticmethod
365     def _create_changelogs_for_revert(tool, revision):
366         # First, discard the ChangeLog changes from the rollout.
367         changelog_paths = tool.scm().modified_changelogs()
368         tool.scm().revert_files(changelog_paths)
369
370         # Second, make new ChangeLog entries for this rollout.
371         # This could move to prepare-ChangeLog by adding a --revert= option.
372         tool.steps.prepare_changelog()
373         for changelog_path in changelog_paths:
374             ChangeLog(changelog_path).update_for_revert(revision)
375
376     @staticmethod
377     def _parse_bug_id_from_revision_diff(tool, revision):
378         original_diff = tool.scm().diff_for_revision(revision)
379         return parse_bug_id(original_diff)
380
381     @staticmethod
382     def _reopen_bug_after_rollout(tool, bug_id, comment_text):
383         if bug_id:
384             tool.bugs.reopen_bug(bug_id, comment_text)
385         else:
386             log(comment_text)
387             log("No bugs were updated or re-opened to reflect this rollout.")
388
389     def execute(self, options, args, tool):
390         revision = args[0]
391         bug_id = self._parse_bug_id_from_revision_diff(tool, revision)
392         if options.complete_rollout:
393             if bug_id:
394                 log("Will re-open bug %s after rollout." % bug_id)
395             else:
396                 log("Failed to parse bug number from diff.  No bugs will be updated/reopened after the rollout.")
397
398         tool.steps.clean_working_directory(tool.scm(), options)
399         tool.scm().update_webkit()
400         tool.scm().apply_reverse_diff(revision)
401         self._create_changelogs_for_revert(tool, revision)
402
403         # FIXME: Fully automated rollout is not 100% idiot-proof yet, so for now just log with instructions on how to complete the rollout.
404         # Once we trust rollout we will remove this option.
405         if not options.complete_rollout:
406             log("\nNOTE: Rollout support is experimental.\nPlease verify the rollout diff and use \"bugzilla-tool land-diff %s\" to commit the rollout." % bug_id)
407         else:
408             # FIXME: This function does not exist!!
409             # comment_text = WebKitLandingScripts.build_and_commit(tool.scm(), options)
410             raise ScriptError("OOPS! This option is not implemented (yet).")
411             self._reopen_bug_after_rollout(tool, bug_id, comment_text)