2009-11-21 Adam Barth <abarth@webkit.org>
[WebKit-https.git] / WebKitTools / Scripts / modules / commands / download.py
1 #!/usr/bin/env python
2 # Copyright (c) 2009, Google Inc. All rights reserved.
3 # Copyright (c) 2009 Apple Inc. All rights reserved.
4 #
5 # Redistribution and use in source and binary forms, with or without
6 # modification, are permitted provided that the following conditions are
7 # met:
8
9 #     * Redistributions of source code must retain the above copyright
10 # notice, this list of conditions and the following disclaimer.
11 #     * Redistributions in binary form must reproduce the above
12 # copyright notice, this list of conditions and the following disclaimer
13 # in the documentation and/or other materials provided with the
14 # distribution.
15 #     * Neither the name of Google Inc. nor the names of its
16 # contributors may be used to endorse or promote products derived from
17 # this software without specific prior written permission.
18
19 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30
31 # FIXME: Trim down this import list once we have unit tests.
32 import os
33 import re
34 import StringIO
35 import subprocess
36 import sys
37 import time
38
39 from datetime import datetime, timedelta
40 from optparse import make_option
41
42 from modules.bugzilla import Bugzilla, parse_bug_id
43 from modules.buildbot import BuildBot
44 from modules.changelogs import ChangeLog
45 from modules.comments import bug_comment_from_commit_text
46 from modules.grammar import pluralize
47 from modules.landingsequence import LandingSequence, ConditionalLandingSequence
48 from modules.logging import error, log, tee
49 from modules.multicommandtool import MultiCommandTool, Command
50 from modules.patchcollection import PatchCollection
51 from modules.scm import CommitMessage, detect_scm_system, ScriptError, CheckoutNeedsUpdate
52 from modules.statusbot import StatusBot
53 from modules.webkitlandingscripts import WebKitLandingScripts, commit_message_for_this_commit
54 from modules.webkitport import WebKitPort
55 from modules.workqueue import WorkQueue, WorkQueueDelegate
56
57
58 class BuildSequence(ConditionalLandingSequence):
59     def __init__(self, options, tool):
60         ConditionalLandingSequence.__init__(self, None, options, tool)
61
62     def run(self):
63         self.clean()
64         self.update()
65         self.build()
66
67
68 class Build(Command):
69     name = "build"
70     def __init__(self):
71         options = WebKitLandingScripts.cleaning_options()
72         options += WebKitLandingScripts.build_options()
73         options += WebKitLandingScripts.land_options()
74         Command.__init__(self, "Updates working copy and does a build.", "", options)
75
76     def execute(self, options, args, tool):
77         sequence = BuildSequence(options, tool)
78         sequence.run_and_handle_errors()
79
80
81 class ApplyAttachment(Command):
82     name = "apply-attachment"
83     def __init__(self):
84         options = WebKitApplyingScripts.apply_options() + WebKitLandingScripts.cleaning_options()
85         Command.__init__(self, "Applies an attachment to the local working directory.", "ATTACHMENT_ID", options=options)
86
87     def execute(self, options, args, tool):
88         WebKitApplyingScripts.setup_for_patch_apply(tool.scm(), options)
89         attachment_id = args[0]
90         attachment = tool.bugs.fetch_attachment(attachment_id)
91         WebKitApplyingScripts.apply_patches_with_options(tool.scm(), [attachment], options)
92
93
94 class ApplyPatches(Command):
95     name = "apply-patches"
96     def __init__(self):
97         options = WebKitApplyingScripts.apply_options() + WebKitLandingScripts.cleaning_options()
98         Command.__init__(self, "Applies all patches on a bug to the local working directory.", "BUGID", options=options)
99
100     def execute(self, options, args, tool):
101         WebKitApplyingScripts.setup_for_patch_apply(tool.scm(), options)
102         bug_id = args[0]
103         patches = tool.bugs.fetch_reviewed_patches_from_bug(bug_id)
104         WebKitApplyingScripts.apply_patches_with_options(tool.scm(), patches, options)
105
106
107 class WebKitApplyingScripts:
108     @staticmethod
109     def apply_options():
110         return [
111             make_option("--no-update", action="store_false", dest="update", default=True, help="Don't update the working directory before applying patches"),
112             make_option("--local-commit", action="store_true", dest="local_commit", default=False, help="Make a local commit for each applied patch"),
113         ]
114
115     @staticmethod
116     def setup_for_patch_apply(scm, options):
117         WebKitLandingScripts.prepare_clean_working_directory(scm, options, allow_local_commits=True)
118         if options.update:
119             scm.update_webkit()
120
121     @staticmethod
122     def apply_patches_with_options(scm, patches, options):
123         if options.local_commit and not scm.supports_local_commits():
124             error("--local-commit passed, but %s does not support local commits" % scm.display_name())
125
126         for patch in patches:
127             log("Applying attachment %s from bug %s" % (patch["id"], patch["bug_id"]))
128             scm.apply_patch(patch)
129             if options.local_commit:
130                 commit_message = commit_message_for_this_commit(scm)
131                 scm.commit_locally_with_message(commit_message.message() or patch["name"])
132
133
134 class LandDiffSequence(ConditionalLandingSequence):
135     def __init__(self, patch, options, tool):
136         ConditionalLandingSequence.__init__(self, patch, options, tool)
137
138     def run(self):
139         self.build()
140         self.test()
141         commit_log = self.commit()
142         self.close_bug(commit_log)
143
144     def close_bug(self, commit_log):
145         comment_test = bug_comment_from_commit_text(self._tool.scm(), commit_log)
146         bug_id = self._patch["bug_id"]
147         if bug_id:
148             log("Updating bug %s" % bug_id)
149             if self._options.close_bug:
150                 self._tool.bugs.close_bug_as_fixed(bug_id, comment_test)
151             else:
152                 # FIXME: We should a smart way to figure out if the patch is attached
153                 # to the bug, and if so obsolete it.
154                 self._tool.bugs.post_comment_to_bug(bug_id, comment_test)
155         else:
156             log(comment_test)
157             log("No bug id provided.")
158
159
160 class LandDiff(Command):
161     name = "land-diff"
162     def __init__(self):
163         options = [
164             make_option("-r", "--reviewer", action="store", type="string", dest="reviewer", help="Update ChangeLogs to say Reviewed by REVIEWER."),
165         ]
166         options += WebKitLandingScripts.build_options()
167         options += WebKitLandingScripts.land_options()
168         Command.__init__(self, "Lands the current working directory diff and updates the bug if provided.", "[BUGID]", options=options)
169
170     def guess_reviewer_from_bug(self, bugs, bug_id):
171         patches = bugs.fetch_reviewed_patches_from_bug(bug_id)
172         if len(patches) != 1:
173             log("%s on bug %s, cannot infer reviewer." % (pluralize("reviewed patch", len(patches)), bug_id))
174             return None
175         patch = patches[0]
176         reviewer = patch["reviewer"]
177         log("Guessing \"%s\" as reviewer from attachment %s on bug %s." % (reviewer, patch["id"], bug_id))
178         return reviewer
179
180     def update_changelogs_with_reviewer(self, reviewer, bug_id, tool):
181         if not reviewer:
182             if not bug_id:
183                 log("No bug id provided and --reviewer= not provided.  Not updating ChangeLogs with reviewer.")
184                 return
185             reviewer = self.guess_reviewer_from_bug(tool.bugs, bug_id)
186
187         if not reviewer:
188             log("Failed to guess reviewer from bug %s and --reviewer= not provided.  Not updating ChangeLogs with reviewer." % bug_id)
189             return
190
191         for changelog_path in tool.scm().modified_changelogs():
192             ChangeLog(changelog_path).set_reviewer(reviewer)
193
194     def execute(self, options, args, tool):
195         bug_id = (args and args[0]) or parse_bug_id(tool.scm().create_patch())
196
197         WebKitLandingScripts.ensure_builders_are_green(tool.buildbot, options)
198
199         os.chdir(tool.scm().checkout_root)
200         self.update_changelogs_with_reviewer(options.reviewer, bug_id, tool)
201
202         fake_patch = {
203             "id": None,
204             "bug_id": bug_id
205         }
206
207         sequence = LandDiffSequence(fake_patch, options, tool)
208         sequence.run()
209
210
211 class AbstractPatchProcessingCommand(Command):
212     def __init__(self, help_text, args_description, options):
213         Command.__init__(self, help_text, args_description, options=options)
214
215     def _fetch_list_of_patches_to_process(self, options, args, tool):
216         raise NotImplementedError, "subclasses must implement"
217
218     def _prepare_to_process(self, options, args, tool):
219         raise NotImplementedError, "subclasses must implement"
220
221     @staticmethod
222     def _collect_patches_by_bug(patches):
223         bugs_to_patches = {}
224         for patch in patches:
225             bug_id = patch["bug_id"]
226             bugs_to_patches[bug_id] = bugs_to_patches.get(bug_id, []).append(patch)
227         return bugs_to_patches
228
229     def execute(self, options, args, tool):
230         if not args:
231             error("%s required" % self.argument_names)
232
233         self._prepare_to_process(options, args, tool)
234         patches = self._fetch_list_of_patches_to_process(options, args, tool)
235
236         # It's nice to print out total statistics.
237         bugs_to_patches = self._collect_patches_by_bug(patches)
238         log("Processing %s from %s." % (pluralize("patch", len(patches)), pluralize("bug", len(bugs_to_patches))))
239
240         for patch in patches:
241             self._process_patch(patch, options, args, tool)
242
243
244 class CheckStyleSequence(LandingSequence):
245     def __init__(self, patch, options, tool):
246         ConditionalLandingSequence.__init__(self, patch, options, tool)
247
248     def run(self):
249         self.clean()
250         self.update()
251         self.apply_patch()
252         self.build()
253
254     def build(self):
255         # Instead of building, we check style.
256         WebKitLandingScripts.run_webkit_script("check-webkit-style")
257
258
259 class CheckStyle(AbstractPatchProcessingCommand):
260     name = "check-style"
261     def __init__(self):
262         options = WebKitLandingScripts.cleaning_options()
263         AbstractPatchProcessingCommand.__init__(self, "Runs check-webkit-style on the specified attachments.", "ATTACHMENT_ID [ATTACHMENT_IDS]", options)
264
265     def _fetch_list_of_patches_to_process(self, options, args, tool):
266         return map(lambda patch_id: tool.bugs.fetch_attachment(patch_id), args)
267
268     def _prepare_to_process(self, options, args, tool):
269         pass
270
271     def _process_patch(self, patch, options, args, tool):
272         sequence = CheckStyleSequence(patch, options, tool)
273         sequence.run_and_handle_errors()
274
275
276 class BuildAttachmentSequence(LandingSequence):
277     def __init__(self, patch, options, tool):
278         LandingSequence.__init__(self, patch, options, tool)
279
280     def run(self):
281         self.clean()
282         self.update()
283         self.apply_patch()
284         self.build()
285
286
287 class BuildAttachment(AbstractPatchProcessingCommand):
288     name = "build-attachment"
289     def __init__(self):
290         options = WebKitLandingScripts.cleaning_options()
291         options += WebKitLandingScripts.build_options()
292         AbstractPatchProcessingCommand.__init__(self, "Builds patches from bugzilla", "ATTACHMENT_ID [ATTACHMENT_IDS]", options)
293
294     def _fetch_list_of_patches_to_process(self, options, args, tool):
295         return map(lambda patch_id: tool.bugs.fetch_attachment(patch_id), args)
296
297     def _prepare_to_process(self, options, args, tool):
298         # Check the tree status first so we can fail early.
299         WebKitLandingScripts.ensure_builders_are_green(tool.buildbot, options)
300
301     def _process_patch(self, patch, options, args, tool):
302         sequence = BuildAttachmentSequence(patch, options, tool)
303         sequence.run_and_handle_errors()
304
305
306 class AbstractPatchLandingCommand(AbstractPatchProcessingCommand):
307     def __init__(self, help_text, args_description):
308         options = WebKitLandingScripts.cleaning_options()
309         options += WebKitLandingScripts.build_options()
310         options += WebKitLandingScripts.land_options()
311         AbstractPatchProcessingCommand.__init__(self, help_text, args_description, options)
312
313     def _prepare_to_process(self, options, args, tool):
314         # Check the tree status first so we can fail early.
315         WebKitLandingScripts.ensure_builders_are_green(tool.buildbot, options)
316
317     def _process_patch(self, patch, options, args, tool):
318         sequence = ConditionalLandingSequence(patch, options, tool)
319         sequence.run_and_handle_errors()
320
321
322 class LandAttachment(AbstractPatchLandingCommand):
323     name = "land-attachment"
324     def __init__(self):
325         AbstractPatchLandingCommand.__init__(self, "Lands patches from bugzilla, optionally building and testing them first", "ATTACHMENT_ID [ATTACHMENT_IDS]")
326
327     def _fetch_list_of_patches_to_process(self, options, args, tool):
328         return map(lambda patch_id: tool.bugs.fetch_attachment(patch_id), args)
329
330
331 class LandPatches(AbstractPatchLandingCommand):
332     name = "land-patches"
333     def __init__(self):
334         AbstractPatchLandingCommand.__init__(self, "Lands all patches on the given bugs, optionally building and testing them first", "BUGID [BUGIDS]")
335
336     def _fetch_list_of_patches_to_process(self, options, args, tool):
337         all_patches = []
338         for bug_id in args:
339             patches = tool.bugs.fetch_reviewed_patches_from_bug(bug_id)
340             log("%s found on bug %s." % (pluralize("reviewed patch", len(patches)), bug_id))
341             all_patches += patches
342         return all_patches
343
344
345 class Rollout(Command):
346     name = "rollout"
347     def __init__(self):
348         options = WebKitLandingScripts.cleaning_options()
349         options += WebKitLandingScripts.build_options()
350         options += WebKitLandingScripts.land_options()
351         options.append(make_option("--complete-rollout", action="store_true", dest="complete_rollout", help="Experimental support for complete unsupervised rollouts, including re-opening the bug.  Not recommended."))
352         Command.__init__(self, "Reverts the given revision and commits the revert and re-opens the original bug.", "REVISION [BUGID]", options=options)
353
354     @staticmethod
355     def _create_changelogs_for_revert(scm, revision):
356         # First, discard the ChangeLog changes from the rollout.
357         changelog_paths = scm.modified_changelogs()
358         scm.revert_files(changelog_paths)
359
360         # Second, make new ChangeLog entries for this rollout.
361         # This could move to prepare-ChangeLog by adding a --revert= option.
362         WebKitLandingScripts.run_webkit_script("prepare-ChangeLog")
363         for changelog_path in changelog_paths:
364             ChangeLog(changelog_path).update_for_revert(revision)
365
366     @staticmethod
367     def _parse_bug_id_from_revision_diff(tool, revision):
368         original_diff = tool.scm().diff_for_revision(revision)
369         return parse_bug_id(original_diff)
370
371     @staticmethod
372     def _reopen_bug_after_rollout(tool, bug_id, comment_text):
373         if bug_id:
374             tool.bugs.reopen_bug(bug_id, comment_text)
375         else:
376             log(comment_text)
377             log("No bugs were updated or re-opened to reflect this rollout.")
378
379     def execute(self, options, args, tool):
380         if not args:
381             error("REVISION is required, see --help.")
382         revision = args[0]
383         bug_id = self._parse_bug_id_from_revision_diff(tool, revision)
384         if options.complete_rollout:
385             if bug_id:
386                 log("Will re-open bug %s after rollout." % bug_id)
387             else:
388                 log("Failed to parse bug number from diff.  No bugs will be updated/reopened after the rollout.")
389
390         WebKitLandingScripts.prepare_clean_working_directory(tool.scm(), options)
391         tool.scm().update_webkit()
392         tool.scm().apply_reverse_diff(revision)
393         self._create_changelogs_for_revert(tool.scm(), revision)
394
395         # FIXME: Fully automated rollout is not 100% idiot-proof yet, so for now just log with instructions on how to complete the rollout.
396         # Once we trust rollout we will remove this option.
397         if not options.complete_rollout:
398             log("\nNOTE: Rollout support is experimental.\nPlease verify the rollout diff and use \"bugzilla-tool land-diff %s\" to commit the rollout." % bug_id)
399         else:
400             comment_text = WebKitLandingScripts.build_and_commit(tool.scm(), options)
401             self._reopen_bug_after_rollout(tool, bug_id, comment_text)