2009-11-27 Adam Barth <abarth@webkit.org>
[WebKit-https.git] / WebKitTools / Scripts / modules / commands / download.py
1 #!/usr/bin/env python
2 # Copyright (c) 2009, Google Inc. All rights reserved.
3 # Copyright (c) 2009 Apple Inc. All rights reserved.
4 #
5 # Redistribution and use in source and binary forms, with or without
6 # modification, are permitted provided that the following conditions are
7 # met:
8
9 #     * Redistributions of source code must retain the above copyright
10 # notice, this list of conditions and the following disclaimer.
11 #     * Redistributions in binary form must reproduce the above
12 # copyright notice, this list of conditions and the following disclaimer
13 # in the documentation and/or other materials provided with the
14 # distribution.
15 #     * Neither the name of Google Inc. nor the names of its
16 # contributors may be used to endorse or promote products derived from
17 # this software without specific prior written permission.
18
19 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30
31 # FIXME: Trim down this import list once we have unit tests.
32 import os
33 import re
34 import StringIO
35 import subprocess
36 import sys
37 import time
38
39 from datetime import datetime, timedelta
40 from optparse import make_option
41
42 from modules.bugzilla import Bugzilla, parse_bug_id
43 from modules.buildbot import BuildBot
44 from modules.buildsteps import BuildSteps
45 from modules.changelogs import ChangeLog
46 from modules.comments import bug_comment_from_commit_text
47 from modules.grammar import pluralize
48 from modules.landingsequence import LandingSequence, ConditionalLandingSequence
49 from modules.logging import error, log, tee
50 from modules.multicommandtool import MultiCommandTool, Command
51 from modules.patchcollection import PatchCollection
52 from modules.scm import CommitMessage, detect_scm_system, ScriptError, CheckoutNeedsUpdate
53 from modules.statusbot import StatusBot
54 from modules.webkitport import WebKitPort
55 from modules.workqueue import WorkQueue, WorkQueueDelegate
56
57
58 class BuildSequence(ConditionalLandingSequence):
59     def __init__(self, options, tool):
60         ConditionalLandingSequence.__init__(self, None, options, tool)
61
62     def run(self):
63         self.clean()
64         self.update()
65         self.build()
66
67
68 class Build(Command):
69     name = "build"
70     show_in_main_help = False
71     def __init__(self):
72         options = BuildSteps.cleaning_options()
73         options += BuildSteps.build_options()
74         options += BuildSteps.land_options()
75         Command.__init__(self, "Update working copy and build", "", options)
76
77     def execute(self, options, args, tool):
78         sequence = BuildSequence(options, tool)
79         sequence.run_and_handle_errors()
80
81
82 # FIXME: Requires unit test. Blocking issue: WebKitApplyingScripts
83 class ApplyAttachment(Command):
84     name = "apply-attachment"
85     show_in_main_help = True
86     def __init__(self):
87         options = WebKitApplyingScripts.apply_options()
88         options += BuildSteps.cleaning_options()
89         Command.__init__(self, "Apply an attachment to the local working directory", "ATTACHMENT_ID", options=options)
90
91     def execute(self, options, args, tool):
92         WebKitApplyingScripts.setup_for_patch_apply(tool, options)
93         attachment_id = args[0]
94         attachment = tool.bugs.fetch_attachment(attachment_id)
95         WebKitApplyingScripts.apply_patches_with_options(tool.scm(), [attachment], options)
96
97
98 # FIXME: Requires unit test. Blocking issue: WebKitApplyingScripts
99 class ApplyPatches(Command):
100     name = "apply-patches"
101     show_in_main_help = True
102     def __init__(self):
103         options = WebKitApplyingScripts.apply_options()
104         options += BuildSteps.cleaning_options()
105         Command.__init__(self, "Apply reviewed patches from provided bugs to the local working directory", "BUGID", options=options)
106
107     def execute(self, options, args, tool):
108         WebKitApplyingScripts.setup_for_patch_apply(tool, options)
109         bug_id = args[0]
110         patches = tool.bugs.fetch_reviewed_patches_from_bug(bug_id)
111         WebKitApplyingScripts.apply_patches_with_options(tool.scm(), patches, options)
112
113
114 class WebKitApplyingScripts:
115     @staticmethod
116     def apply_options():
117         return [
118             make_option("--no-update", action="store_false", dest="update", default=True, help="Don't update the working directory before applying patches"),
119             make_option("--local-commit", action="store_true", dest="local_commit", default=False, help="Make a local commit for each applied patch"),
120         ]
121
122     @staticmethod
123     def setup_for_patch_apply(tool, options):
124         tool.steps.clean_working_directory(tool.scm(), options, allow_local_commits=True)
125         if options.update:
126             tool.scm().update_webkit()
127
128     @staticmethod
129     def apply_patches_with_options(scm, patches, options):
130         if options.local_commit and not scm.supports_local_commits():
131             error("--local-commit passed, but %s does not support local commits" % scm.display_name())
132
133         for patch in patches:
134             log("Applying attachment %s from bug %s" % (patch["id"], patch["bug_id"]))
135             scm.apply_patch(patch)
136             if options.local_commit:
137                 commit_message = scm.commit_message_for_this_commit()
138                 scm.commit_locally_with_message(commit_message.message() or patch["name"])
139
140
141 class LandDiffSequence(ConditionalLandingSequence):
142     def __init__(self, patch, options, tool):
143         ConditionalLandingSequence.__init__(self, patch, options, tool)
144
145     def run(self):
146         self.build()
147         self.test()
148         commit_log = self.commit()
149         self.close_bug(commit_log)
150
151     def close_bug(self, commit_log):
152         comment_test = bug_comment_from_commit_text(self._tool.scm(), commit_log)
153         bug_id = self._patch["bug_id"]
154         if bug_id:
155             log("Updating bug %s" % bug_id)
156             if self._options.close_bug:
157                 self._tool.bugs.close_bug_as_fixed(bug_id, comment_test)
158             else:
159                 # FIXME: We should a smart way to figure out if the patch is attached
160                 # to the bug, and if so obsolete it.
161                 self._tool.bugs.post_comment_to_bug(bug_id, comment_test)
162         else:
163             log(comment_test)
164             log("No bug id provided.")
165
166
167 class LandDiff(Command):
168     name = "land-diff"
169     show_in_main_help = True
170     def __init__(self):
171         options = [
172             make_option("-r", "--reviewer", action="store", type="string", dest="reviewer", help="Update ChangeLogs to say Reviewed by REVIEWER."),
173         ]
174         options += BuildSteps.build_options()
175         options += BuildSteps.land_options()
176         Command.__init__(self, "Land the current working directory diff and updates the associated bug if any", "[BUGID]", options=options)
177
178     def guess_reviewer_from_bug(self, bugs, bug_id):
179         patches = bugs.fetch_reviewed_patches_from_bug(bug_id)
180         if len(patches) != 1:
181             log("%s on bug %s, cannot infer reviewer." % (pluralize("reviewed patch", len(patches)), bug_id))
182             return None
183         patch = patches[0]
184         reviewer = patch["reviewer"]
185         log("Guessing \"%s\" as reviewer from attachment %s on bug %s." % (reviewer, patch["id"], bug_id))
186         return reviewer
187
188     def update_changelogs_with_reviewer(self, reviewer, bug_id, tool):
189         if not reviewer:
190             if not bug_id:
191                 log("No bug id provided and --reviewer= not provided.  Not updating ChangeLogs with reviewer.")
192                 return
193             reviewer = self.guess_reviewer_from_bug(tool.bugs, bug_id)
194
195         if not reviewer:
196             log("Failed to guess reviewer from bug %s and --reviewer= not provided.  Not updating ChangeLogs with reviewer." % bug_id)
197             return
198
199         for changelog_path in tool.scm().modified_changelogs():
200             ChangeLog(changelog_path).set_reviewer(reviewer)
201
202     def execute(self, options, args, tool):
203         bug_id = (args and args[0]) or parse_bug_id(tool.scm().create_patch())
204
205         tool.steps.ensure_builders_are_green(tool.buildbot, options)
206
207         os.chdir(tool.scm().checkout_root)
208         self.update_changelogs_with_reviewer(options.reviewer, bug_id, tool)
209
210         fake_patch = {
211             "id": None,
212             "bug_id": bug_id
213         }
214
215         sequence = LandDiffSequence(fake_patch, options, tool)
216         sequence.run()
217
218
219 class AbstractPatchProcessingCommand(Command):
220     def __init__(self, help_text, args_description, options):
221         Command.__init__(self, help_text, args_description, options=options)
222
223     def _fetch_list_of_patches_to_process(self, options, args, tool):
224         raise NotImplementedError, "subclasses must implement"
225
226     def _prepare_to_process(self, options, args, tool):
227         raise NotImplementedError, "subclasses must implement"
228
229     @staticmethod
230     def _collect_patches_by_bug(patches):
231         bugs_to_patches = {}
232         for patch in patches:
233             bug_id = patch["bug_id"]
234             bugs_to_patches[bug_id] = bugs_to_patches.get(bug_id, []) + [patch]
235         return bugs_to_patches
236
237     def execute(self, options, args, tool):
238         self._prepare_to_process(options, args, tool)
239         patches = self._fetch_list_of_patches_to_process(options, args, tool)
240
241         # It's nice to print out total statistics.
242         bugs_to_patches = self._collect_patches_by_bug(patches)
243         log("Processing %s from %s." % (pluralize("patch", len(patches)), pluralize("bug", len(bugs_to_patches))))
244
245         for patch in patches:
246             self._process_patch(patch, options, args, tool)
247
248
249 class CheckStyleSequence(LandingSequence):
250     def __init__(self, patch, options, tool):
251         LandingSequence.__init__(self, patch, options, tool)
252
253     def run(self):
254         self.clean()
255         self.update()
256         self.apply_patch()
257         self.build()
258
259     def build(self):
260         # Instead of building, we check style.
261         self._tool.steps.check_style()
262
263
264 class CheckStyle(AbstractPatchProcessingCommand):
265     name = "check-style"
266     show_in_main_help = False
267     def __init__(self):
268         options = BuildSteps.cleaning_options()
269         options += BuildSteps.build_options()
270         AbstractPatchProcessingCommand.__init__(self, "Run check-webkit-style on the specified attachments", "ATTACHMENT_ID [ATTACHMENT_IDS]", options)
271
272     def _fetch_list_of_patches_to_process(self, options, args, tool):
273         return map(lambda patch_id: tool.bugs.fetch_attachment(patch_id), args)
274
275     def _prepare_to_process(self, options, args, tool):
276         pass
277
278     def _process_patch(self, patch, options, args, tool):
279         sequence = CheckStyleSequence(patch, options, tool)
280         sequence.run_and_handle_errors()
281
282
283 class BuildAttachmentSequence(LandingSequence):
284     def __init__(self, patch, options, tool):
285         LandingSequence.__init__(self, patch, options, tool)
286
287     def run(self):
288         self.clean()
289         self.update()
290         self.apply_patch()
291         self.build()
292
293
294 class BuildAttachment(AbstractPatchProcessingCommand):
295     name = "build-attachment"
296     show_in_main_help = False
297     def __init__(self):
298         options = BuildSteps.cleaning_options()
299         options += BuildSteps.build_options()
300         AbstractPatchProcessingCommand.__init__(self, "Apply and build patches from bugzilla", "ATTACHMENT_ID [ATTACHMENT_IDS]", options)
301
302     def _fetch_list_of_patches_to_process(self, options, args, tool):
303         return map(lambda patch_id: tool.bugs.fetch_attachment(patch_id), args)
304
305     def _prepare_to_process(self, options, args, tool):
306         # Check the tree status first so we can fail early.
307         tool.steps.ensure_builders_are_green(tool.buildbot, options)
308
309     def _process_patch(self, patch, options, args, tool):
310         sequence = BuildAttachmentSequence(patch, options, tool)
311         sequence.run_and_handle_errors()
312
313
314 class AbstractPatchLandingCommand(AbstractPatchProcessingCommand):
315     def __init__(self, help_text, args_description):
316         options = BuildSteps.cleaning_options()
317         options += BuildSteps.build_options()
318         options += BuildSteps.land_options()
319         AbstractPatchProcessingCommand.__init__(self, help_text, args_description, options)
320
321     def _prepare_to_process(self, options, args, tool):
322         # Check the tree status first so we can fail early.
323         tool.steps.ensure_builders_are_green(tool.buildbot, options)
324
325     def _process_patch(self, patch, options, args, tool):
326         sequence = ConditionalLandingSequence(patch, options, tool)
327         sequence.run_and_handle_errors()
328
329
330 class LandAttachment(AbstractPatchLandingCommand):
331     name = "land-attachment"
332     show_in_main_help = True
333     def __init__(self):
334         AbstractPatchLandingCommand.__init__(self, "Land patches from bugzilla, optionally building and testing them first", "ATTACHMENT_ID [ATTACHMENT_IDS]")
335
336     def _fetch_list_of_patches_to_process(self, options, args, tool):
337         return map(lambda patch_id: tool.bugs.fetch_attachment(patch_id), args)
338
339
340 class LandPatches(AbstractPatchLandingCommand):
341     name = "land-patches"
342     show_in_main_help = True
343     def __init__(self):
344         AbstractPatchLandingCommand.__init__(self, "Land all patches on the given bugs, optionally building and testing them first", "BUGID [BUGIDS]")
345
346     def _fetch_list_of_patches_to_process(self, options, args, tool):
347         all_patches = []
348         for bug_id in args:
349             patches = tool.bugs.fetch_reviewed_patches_from_bug(bug_id)
350             log("%s found on bug %s." % (pluralize("reviewed patch", len(patches)), bug_id))
351             all_patches += patches
352         return all_patches
353
354
355 # FIXME: Requires unit test.
356 class Rollout(Command):
357     name = "rollout"
358     show_in_main_help = True
359     def __init__(self):
360         options = BuildSteps.cleaning_options()
361         options += BuildSteps.build_options()
362         options += BuildSteps.land_options()
363         options.append(make_option("--complete-rollout", action="store_true", dest="complete_rollout", help="Commit the revert and re-open the original bug."))
364         Command.__init__(self, "Revert the given revision in the working copy and optionally commit the revert and re-open the original bug", "REVISION [BUGID]", options=options)
365
366     @staticmethod
367     def _create_changelogs_for_revert(tool, revision):
368         # First, discard the ChangeLog changes from the rollout.
369         changelog_paths = tool.scm().modified_changelogs()
370         tool.scm().revert_files(changelog_paths)
371
372         # Second, make new ChangeLog entries for this rollout.
373         # This could move to prepare-ChangeLog by adding a --revert= option.
374         tool.steps.prepare_changelog()
375         for changelog_path in changelog_paths:
376             ChangeLog(changelog_path).update_for_revert(revision)
377
378     @staticmethod
379     def _parse_bug_id_from_revision_diff(tool, revision):
380         original_diff = tool.scm().diff_for_revision(revision)
381         return parse_bug_id(original_diff)
382
383     @staticmethod
384     def _reopen_bug_after_rollout(tool, bug_id, comment_text):
385         if bug_id:
386             tool.bugs.reopen_bug(bug_id, comment_text)
387         else:
388             log(comment_text)
389             log("No bugs were updated or re-opened to reflect this rollout.")
390
391     def execute(self, options, args, tool):
392         revision = args[0]
393         bug_id = self._parse_bug_id_from_revision_diff(tool, revision)
394         if options.complete_rollout:
395             if bug_id:
396                 log("Will re-open bug %s after rollout." % bug_id)
397             else:
398                 log("Failed to parse bug number from diff.  No bugs will be updated/reopened after the rollout.")
399
400         tool.steps.clean_working_directory(tool.scm(), options)
401         tool.scm().update_webkit()
402         tool.scm().apply_reverse_diff(revision)
403         self._create_changelogs_for_revert(tool, revision)
404
405         # FIXME: Fully automated rollout is not 100% idiot-proof yet, so for now just log with instructions on how to complete the rollout.
406         # Once we trust rollout we will remove this option.
407         if not options.complete_rollout:
408             log("\nNOTE: Rollout support is experimental.\nPlease verify the rollout diff and use \"bugzilla-tool land-diff %s\" to commit the rollout." % bug_id)
409         else:
410             # FIXME: This function does not exist!!
411             # comment_text = WebKitLandingScripts.build_and_commit(tool.scm(), options)
412             raise ScriptError("OOPS! This option is not implemented (yet).")
413             self._reopen_bug_after_rollout(tool, bug_id, comment_text)