2009-11-20 Adam Barth <abarth@webkit.org>
[WebKit-https.git] / WebKitTools / Scripts / modules / commands / download.py
1 #!/usr/bin/env python
2 # Copyright (c) 2009, Google Inc. All rights reserved.
3 # Copyright (c) 2009 Apple Inc. All rights reserved.
4 #
5 # Redistribution and use in source and binary forms, with or without
6 # modification, are permitted provided that the following conditions are
7 # met:
8
9 #     * Redistributions of source code must retain the above copyright
10 # notice, this list of conditions and the following disclaimer.
11 #     * Redistributions in binary form must reproduce the above
12 # copyright notice, this list of conditions and the following disclaimer
13 # in the documentation and/or other materials provided with the
14 # distribution.
15 #     * Neither the name of Google Inc. nor the names of its
16 # contributors may be used to endorse or promote products derived from
17 # this software without specific prior written permission.
18
19 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30
31 # FIXME: Trim down this import list once we have unit tests.
32 import os
33 import re
34 import StringIO
35 import subprocess
36 import sys
37 import time
38
39 from datetime import datetime, timedelta
40 from optparse import make_option
41
42 from modules.bugzilla import Bugzilla, parse_bug_id
43 from modules.buildbot import BuildBot
44 from modules.changelogs import ChangeLog
45 from modules.comments import bug_comment_from_commit_text
46 from modules.grammar import pluralize
47 from modules.landingsequence import LandingSequence, ConditionalLandingSequence
48 from modules.logging import error, log, tee
49 from modules.multicommandtool import MultiCommandTool, Command
50 from modules.patchcollection import PatchCollection
51 from modules.scm import CommitMessage, detect_scm_system, ScriptError, CheckoutNeedsUpdate
52 from modules.statusbot import StatusBot
53 from modules.webkitlandingscripts import WebKitLandingScripts, commit_message_for_this_commit
54 from modules.webkitport import WebKitPort
55 from modules.workqueue import WorkQueue, WorkQueueDelegate
56
57 class CheckStyle(Command):
58     name = "check-style"
59     def __init__(self):
60         options = WebKitLandingScripts.cleaning_options()
61         Command.__init__(self, "Runs check-webkit-style on the specified attachment", "ATTACHMENT_ID", options=options)
62
63     @classmethod
64     def check_style(cls, patch, options, tool):
65         tool.scm().update_webkit()
66         log("Checking style for patch %s from bug %s." % (patch["id"], patch["bug_id"]))
67         try:
68             # FIXME: check-webkit-style shouldn't really have to apply the patch to check the style.
69             tool.scm().apply_patch(patch)
70             WebKitLandingScripts.run_webkit_script("check-webkit-style")
71         except ScriptError, e:
72             log("Patch %s from bug %s failed to apply and check style." % (patch["id"], patch["bug_id"]))
73             log(e.output)
74
75         # This is safe because in order to get here the working directory had to be
76         # clean at the beginning.  Clean it out again before we exit.
77         tool.scm().ensure_clean_working_directory(force_clean=True)
78
79     def execute(self, options, args, tool):
80         attachment_id = args[0]
81         attachment = tool.bugs.fetch_attachment(attachment_id)
82
83         WebKitLandingScripts.prepare_clean_working_directory(tool.scm(), options)
84         self.check_style(attachment, options, tool)
85
86
87 class BuildSequence(ConditionalLandingSequence):
88     def __init__(self, options, tool):
89         ConditionalLandingSequence.__init__(self, None, options, tool)
90
91     def run(self):
92         self.clean()
93         self.update()
94         self.build()
95
96
97 class Build(Command):
98     name = "build"
99     def __init__(self):
100         options = WebKitLandingScripts.cleaning_options()
101         options += WebKitLandingScripts.build_options()
102         options += WebKitLandingScripts.land_options()
103         Command.__init__(self, "Updates working copy and does a build.", "", options)
104
105     def execute(self, options, args, tool):
106         sequence = BuildSequence(options, tool)
107         sequence.run_and_handle_errors()
108
109
110 class ApplyAttachment(Command):
111     name = "apply-attachment"
112     def __init__(self):
113         options = WebKitApplyingScripts.apply_options() + WebKitLandingScripts.cleaning_options()
114         Command.__init__(self, "Applies an attachment to the local working directory.", "ATTACHMENT_ID", options=options)
115
116     def execute(self, options, args, tool):
117         WebKitApplyingScripts.setup_for_patch_apply(tool.scm(), options)
118         attachment_id = args[0]
119         attachment = tool.bugs.fetch_attachment(attachment_id)
120         WebKitApplyingScripts.apply_patches_with_options(tool.scm(), [attachment], options)
121
122
123 class ApplyPatches(Command):
124     name = "apply-patches"
125     def __init__(self):
126         options = WebKitApplyingScripts.apply_options() + WebKitLandingScripts.cleaning_options()
127         Command.__init__(self, "Applies all patches on a bug to the local working directory.", "BUGID", options=options)
128
129     def execute(self, options, args, tool):
130         WebKitApplyingScripts.setup_for_patch_apply(tool.scm(), options)
131         bug_id = args[0]
132         patches = tool.bugs.fetch_reviewed_patches_from_bug(bug_id)
133         WebKitApplyingScripts.apply_patches_with_options(tool.scm(), patches, options)
134
135
136 class WebKitApplyingScripts:
137     @staticmethod
138     def apply_options():
139         return [
140             make_option("--no-update", action="store_false", dest="update", default=True, help="Don't update the working directory before applying patches"),
141             make_option("--local-commit", action="store_true", dest="local_commit", default=False, help="Make a local commit for each applied patch"),
142         ]
143
144     @staticmethod
145     def setup_for_patch_apply(scm, options):
146         WebKitLandingScripts.prepare_clean_working_directory(scm, options, allow_local_commits=True)
147         if options.update:
148             scm.update_webkit()
149
150     @staticmethod
151     def apply_patches_with_options(scm, patches, options):
152         if options.local_commit and not scm.supports_local_commits():
153             error("--local-commit passed, but %s does not support local commits" % scm.display_name())
154
155         for patch in patches:
156             log("Applying attachment %s from bug %s" % (patch["id"], patch["bug_id"]))
157             scm.apply_patch(patch)
158             if options.local_commit:
159                 commit_message = commit_message_for_this_commit(scm)
160                 scm.commit_locally_with_message(commit_message.message() or patch["name"])
161
162
163 class LandDiffSequence(ConditionalLandingSequence):
164     def __init__(self, patch, options, tool):
165         ConditionalLandingSequence.__init__(self, patch, options, tool)
166
167     def run(self):
168         self.build()
169         self.test()
170         commit_log = self.commit()
171         self.close_bug(commit_log)
172
173     def close_bug(self, commit_log):
174         comment_test = bug_comment_from_commit_text(self._tool.scm(), commit_log)
175         bug_id = self._patch["bug_id"]
176         if bug_id:
177             log("Updating bug %s" % bug_id)
178             if self._options.close_bug:
179                 self._tool.bugs.close_bug_as_fixed(bug_id, comment_test)
180             else:
181                 # FIXME: We should a smart way to figure out if the patch is attached
182                 # to the bug, and if so obsolete it.
183                 self._tool.bugs.post_comment_to_bug(bug_id, comment_test)
184         else:
185             log(comment_test)
186             log("No bug id provided.")
187
188
189 class LandDiff(Command):
190     name = "land-diff"
191     def __init__(self):
192         options = [
193             make_option("-r", "--reviewer", action="store", type="string", dest="reviewer", help="Update ChangeLogs to say Reviewed by REVIEWER."),
194         ]
195         options += WebKitLandingScripts.build_options()
196         options += WebKitLandingScripts.land_options()
197         Command.__init__(self, "Lands the current working directory diff and updates the bug if provided.", "[BUGID]", options=options)
198
199     def guess_reviewer_from_bug(self, bugs, bug_id):
200         patches = bugs.fetch_reviewed_patches_from_bug(bug_id)
201         if len(patches) != 1:
202             log("%s on bug %s, cannot infer reviewer." % (pluralize("reviewed patch", len(patches)), bug_id))
203             return None
204         patch = patches[0]
205         reviewer = patch["reviewer"]
206         log("Guessing \"%s\" as reviewer from attachment %s on bug %s." % (reviewer, patch["id"], bug_id))
207         return reviewer
208
209     def update_changelogs_with_reviewer(self, reviewer, bug_id, tool):
210         if not reviewer:
211             if not bug_id:
212                 log("No bug id provided and --reviewer= not provided.  Not updating ChangeLogs with reviewer.")
213                 return
214             reviewer = self.guess_reviewer_from_bug(tool.bugs, bug_id)
215
216         if not reviewer:
217             log("Failed to guess reviewer from bug %s and --reviewer= not provided.  Not updating ChangeLogs with reviewer." % bug_id)
218             return
219
220         for changelog_path in tool.scm().modified_changelogs():
221             ChangeLog(changelog_path).set_reviewer(reviewer)
222
223     def execute(self, options, args, tool):
224         bug_id = (args and args[0]) or parse_bug_id(tool.scm().create_patch())
225
226         WebKitLandingScripts.ensure_builders_are_green(tool.buildbot, options)
227
228         os.chdir(tool.scm().checkout_root)
229         self.update_changelogs_with_reviewer(options.reviewer, bug_id, tool)
230
231         fake_patch = {
232             "id": None,
233             "bug_id": bug_id
234         }
235
236         sequence = LandDiffSequence(fake_patch, options, tool)
237         sequence.run()
238
239
240 class AbstractPatchProcessingCommand(Command):
241     def __init__(self, help_text, args_description, options):
242         Command.__init__(self, help_text, args_description, options=options)
243
244     def _fetch_list_of_patches_to_process(self, options, args, tool):
245         raise NotImplementedError, "subclasses must implement"
246
247     def _prepare_to_process(self, options, args, tool):
248         raise NotImplementedError, "subclasses must implement"
249
250     @staticmethod
251     def _collect_patches_by_bug(patches):
252         bugs_to_patches = {}
253         for patch in patches:
254             bug_id = patch["bug_id"]
255             bugs_to_patches[bug_id] = bugs_to_patches.get(bug_id, []).append(patch)
256         return bugs_to_patches
257
258     def execute(self, options, args, tool):
259         if not args:
260             error("%s required" % self.argument_names)
261
262         self._prepare_to_process(options, args, tool)
263         patches = self._fetch_list_of_patches_to_process(options, args, tool)
264
265         # It's nice to print out total statistics.
266         bugs_to_patches = self._collect_patches_by_bug(patches)
267         log("Processing %s from %s." % (pluralize("patch", len(patches)), pluralize("bug", len(bugs_to_patches))))
268
269         for patch in patches:
270             self._process_patch(patch, options, args, tool)
271
272
273 class BuildAttachmentSequence(LandingSequence):
274     def __init__(self, patch, options, tool):
275         LandingSequence.__init__(self, patch, options, tool)
276
277     def run(self):
278         self.clean()
279         self.update()
280         self.apply_patch()
281         self.build()
282
283
284 class BuildAttachment(AbstractPatchProcessingCommand):
285     name = "build-attachment"
286     def __init__(self):
287         options = WebKitLandingScripts.cleaning_options()
288         options += WebKitLandingScripts.build_options()
289         AbstractPatchProcessingCommand.__init__(self, "Builds patches from bugzilla", "ATTACHMENT_ID [ATTACHMENT_IDS]", options)
290
291     def _fetch_list_of_patches_to_process(self, options, args, tool):
292         return map(lambda patch_id: tool.bugs.fetch_attachment(patch_id), args)
293
294     def _prepare_to_process(self, options, args, tool):
295         # Check the tree status first so we can fail early.
296         WebKitLandingScripts.ensure_builders_are_green(tool.buildbot, options)
297
298     def _process_patch(self, patch, options, args, tool):
299         sequence = BuildAttachmentSequence(patch, options, tool)
300         sequence.run_and_handle_errors()
301
302
303 class AbstractPatchLandingCommand(AbstractPatchProcessingCommand):
304     def __init__(self, help_text, args_description):
305         options = WebKitLandingScripts.cleaning_options()
306         options += WebKitLandingScripts.build_options()
307         options += WebKitLandingScripts.land_options()
308         AbstractPatchProcessingCommand.__init__(self, help_text, args_description, options)
309
310     def _prepare_to_process(self, options, args, tool):
311         # Check the tree status first so we can fail early.
312         WebKitLandingScripts.ensure_builders_are_green(tool.buildbot, options)
313
314     def _process_patch(self, patch, options, args, tool):
315         sequence = ConditionalLandingSequence(patch, options, tool)
316         sequence.run_and_handle_errors()
317
318
319 class LandAttachment(AbstractPatchLandingCommand):
320     name = "land-attachment"
321     def __init__(self):
322         AbstractPatchLandingCommand.__init__(self, "Lands patches from bugzilla, optionally building and testing them first", "ATTACHMENT_ID [ATTACHMENT_IDS]")
323
324     def _fetch_list_of_patches_to_process(self, options, args, tool):
325         return map(lambda patch_id: tool.bugs.fetch_attachment(patch_id), args)
326
327
328 class LandPatches(AbstractPatchLandingCommand):
329     name = "land-patches"
330     def __init__(self):
331         AbstractPatchLandingCommand.__init__(self, "Lands all patches on the given bugs, optionally building and testing them first", "BUGID [BUGIDS]")
332
333     def _fetch_list_of_patches_to_process(self, options, args, tool):
334         all_patches = []
335         for bug_id in args:
336             patches = tool.bugs.fetch_reviewed_patches_from_bug(bug_id)
337             log("%s found on bug %s." % (pluralize("reviewed patch", len(patches)), bug_id))
338             all_patches += patches
339         return all_patches
340
341
342 class Rollout(Command):
343     name = "rollout"
344     def __init__(self):
345         options = WebKitLandingScripts.cleaning_options()
346         options += WebKitLandingScripts.build_options()
347         options += WebKitLandingScripts.land_options()
348         options.append(make_option("--complete-rollout", action="store_true", dest="complete_rollout", help="Experimental support for complete unsupervised rollouts, including re-opening the bug.  Not recommended."))
349         Command.__init__(self, "Reverts the given revision and commits the revert and re-opens the original bug.", "REVISION [BUGID]", options=options)
350
351     @staticmethod
352     def _create_changelogs_for_revert(scm, revision):
353         # First, discard the ChangeLog changes from the rollout.
354         changelog_paths = scm.modified_changelogs()
355         scm.revert_files(changelog_paths)
356
357         # Second, make new ChangeLog entries for this rollout.
358         # This could move to prepare-ChangeLog by adding a --revert= option.
359         WebKitLandingScripts.run_webkit_script("prepare-ChangeLog")
360         for changelog_path in changelog_paths:
361             ChangeLog(changelog_path).update_for_revert(revision)
362
363     @staticmethod
364     def _parse_bug_id_from_revision_diff(tool, revision):
365         original_diff = tool.scm().diff_for_revision(revision)
366         return parse_bug_id(original_diff)
367
368     @staticmethod
369     def _reopen_bug_after_rollout(tool, bug_id, comment_text):
370         if bug_id:
371             tool.bugs.reopen_bug(bug_id, comment_text)
372         else:
373             log(comment_text)
374             log("No bugs were updated or re-opened to reflect this rollout.")
375
376     def execute(self, options, args, tool):
377         if not args:
378             error("REVISION is required, see --help.")
379         revision = args[0]
380         bug_id = self._parse_bug_id_from_revision_diff(tool, revision)
381         if options.complete_rollout:
382             if bug_id:
383                 log("Will re-open bug %s after rollout." % bug_id)
384             else:
385                 log("Failed to parse bug number from diff.  No bugs will be updated/reopened after the rollout.")
386
387         WebKitLandingScripts.prepare_clean_working_directory(tool.scm(), options)
388         tool.scm().update_webkit()
389         tool.scm().apply_reverse_diff(revision)
390         self._create_changelogs_for_revert(tool.scm(), revision)
391
392         # FIXME: Fully automated rollout is not 100% idiot-proof yet, so for now just log with instructions on how to complete the rollout.
393         # Once we trust rollout we will remove this option.
394         if not options.complete_rollout:
395             log("\nNOTE: Rollout support is experimental.\nPlease verify the rollout diff and use \"bugzilla-tool land-diff %s\" to commit the rollout." % bug_id)
396         else:
397             comment_text = WebKitLandingScripts.build_and_commit(tool.scm(), options)
398             self._reopen_bug_after_rollout(tool, bug_id, comment_text)