WebKitTools/Scripts/modules/commands/download.py

   1 #!/usr/bin/env python
   2 # Copyright (c) 2009, Google Inc. All rights reserved.
   3 # Copyright (c) 2009 Apple Inc. All rights reserved.
   4 #
   5 # Redistribution and use in source and binary forms, with or without
   6 # modification, are permitted provided that the following conditions are
   7 # met:
   8 #
   9 #     * Redistributions of source code must retain the above copyright
  10 # notice, this list of conditions and the following disclaimer.
  11 #     * Redistributions in binary form must reproduce the above
  12 # copyright notice, this list of conditions and the following disclaimer
  13 # in the documentation and/or other materials provided with the
  14 # distribution.
  15 #     * Neither the name of Google Inc. nor the names of its
  16 # contributors may be used to endorse or promote products derived from
  17 # this software without specific prior written permission.
  18 #
  19 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  20 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  21 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  22 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  23 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  24 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  25 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  26 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  27 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  28 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  29 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  30
  31 # FIXME: Trim down this import list once we have unit tests.
  32 import os
  33 import re
  34 import StringIO
  35 import subprocess
  36 import sys
  37 import time
  38
  39 from datetime import datetime, timedelta
  40 from optparse import make_option
  41
  42 from modules.bugzilla import Bugzilla, parse_bug_id
  43 from modules.buildbot import BuildBot
  44 from modules.changelogs import ChangeLog
  45 from modules.comments import bug_comment_from_commit_text
  46 from modules.grammar import pluralize
  47 from modules.landingsequence import LandingSequence, ConditionalLandingSequence
  48 from modules.logging import error, log, tee
  49 from modules.multicommandtool import MultiCommandTool, Command
  50 from modules.patchcollection import PatchCollection
  51 from modules.scm import CommitMessage, detect_scm_system, ScriptError, CheckoutNeedsUpdate
  52 from modules.statusbot import StatusBot
  53 from modules.webkitlandingscripts import WebKitLandingScripts, commit_message_for_this_commit
  54 from modules.webkitport import WebKitPort
  55 from modules.workqueue import WorkQueue, WorkQueueDelegate
  56
  57 class CheckStyle(Command):
  58     name = "check-style"
  59     def __init__(self):
  60         options = WebKitLandingScripts.cleaning_options()
  61         Command.__init__(self, "Runs check-webkit-style on the specified attachment", "ATTACHMENT_ID", options=options)
  62
  63     @classmethod
  64     def check_style(cls, patch, options, tool):
  65         tool.scm().update_webkit()
  66         log("Checking style for patch %s from bug %s." % (patch["id"], patch["bug_id"]))
  67         try:
  68             # FIXME: check-webkit-style shouldn't really have to apply the patch to check the style.
  69             tool.scm().apply_patch(patch)
  70             WebKitLandingScripts.run_webkit_script("check-webkit-style")
  71         except ScriptError, e:
  72             log("Patch %s from bug %s failed to apply and check style." % (patch["id"], patch["bug_id"]))
  73             log(e.output)
  74
  75         # This is safe because in order to get here the working directory had to be
  76         # clean at the beginning.  Clean it out again before we exit.
  77         tool.scm().ensure_clean_working_directory(force_clean=True)
  78
  79     def execute(self, options, args, tool):
  80         attachment_id = args[0]
  81         attachment = tool.bugs.fetch_attachment(attachment_id)
  82
  83         WebKitLandingScripts.prepare_clean_working_directory(tool.scm(), options)
  84         self.check_style(attachment, options, tool)
  85
  86
  87 class BuildSequence(ConditionalLandingSequence):
  88     def __init__(self, options, tool):
  89         ConditionalLandingSequence.__init__(self, None, options, tool)
  90
  91     def run(self):
  92         self.clean()
  93         self.update()
  94         self.build()
  95
  96
  97 class Build(Command):
  98     name = "build"
  99     def __init__(self):
 100         options = WebKitLandingScripts.cleaning_options()
 101         options += WebKitLandingScripts.build_options()
 102         options += WebKitLandingScripts.land_options()
 103         Command.__init__(self, "Updates working copy and does a build.", "", options)
 104
 105     def execute(self, options, args, tool):
 106         sequence = BuildSequence(options, tool)
 107         sequence.run_and_handle_errors()
 108
 109
 110 class ApplyAttachment(Command):
 111     name = "apply-attachment"
 112     def __init__(self):
 113         options = WebKitApplyingScripts.apply_options() + WebKitLandingScripts.cleaning_options()
 114         Command.__init__(self, "Applies an attachment to the local working directory.", "ATTACHMENT_ID", options=options)
 115
 116     def execute(self, options, args, tool):
 117         WebKitApplyingScripts.setup_for_patch_apply(tool.scm(), options)
 118         attachment_id = args[0]
 119         attachment = tool.bugs.fetch_attachment(attachment_id)
 120         WebKitApplyingScripts.apply_patches_with_options(tool.scm(), [attachment], options)
 121
 122
 123 class ApplyPatches(Command):
 124     name = "apply-patches"
 125     def __init__(self):
 126         options = WebKitApplyingScripts.apply_options() + WebKitLandingScripts.cleaning_options()
 127         Command.__init__(self, "Applies all patches on a bug to the local working directory.", "BUGID", options=options)
 128
 129     def execute(self, options, args, tool):
 130         WebKitApplyingScripts.setup_for_patch_apply(tool.scm(), options)
 131         bug_id = args[0]
 132         patches = tool.bugs.fetch_reviewed_patches_from_bug(bug_id)
 133         WebKitApplyingScripts.apply_patches_with_options(tool.scm(), patches, options)
 134
 135
 136 class WebKitApplyingScripts:
 137     @staticmethod
 138     def apply_options():
 139         return [
 140             make_option("--no-update", action="store_false", dest="update", default=True, help="Don't update the working directory before applying patches"),
 141             make_option("--local-commit", action="store_true", dest="local_commit", default=False, help="Make a local commit for each applied patch"),
 142         ]
 143
 144     @staticmethod
 145     def setup_for_patch_apply(scm, options):
 146         WebKitLandingScripts.prepare_clean_working_directory(scm, options, allow_local_commits=True)
 147         if options.update:
 148             scm.update_webkit()
 149
 150     @staticmethod
 151     def apply_patches_with_options(scm, patches, options):
 152         if options.local_commit and not scm.supports_local_commits():
 153             error("--local-commit passed, but %s does not support local commits" % scm.display_name())
 154
 155         for patch in patches:
 156             log("Applying attachment %s from bug %s" % (patch["id"], patch["bug_id"]))
 157             scm.apply_patch(patch)
 158             if options.local_commit:
 159                 commit_message = commit_message_for_this_commit(scm)
 160                 scm.commit_locally_with_message(commit_message.message() or patch["name"])
 161
 162
 163 class LandDiffSequence(ConditionalLandingSequence):
 164     def __init__(self, patch, options, tool):
 165         ConditionalLandingSequence.__init__(self, patch, options, tool)
 166
 167     def run(self):
 168         self.build()
 169         self.test()
 170         commit_log = self.commit()
 171         self.close_bug(commit_log)
 172
 173     def close_bug(self, commit_log):
 174         comment_test = bug_comment_from_commit_text(self._tool.scm(), commit_log)
 175         bug_id = self._patch["bug_id"]
 176         if bug_id:
 177             log("Updating bug %s" % bug_id)
 178             if self._options.close_bug:
 179                 self._tool.bugs.close_bug_as_fixed(bug_id, comment_test)
 180             else:
 181                 # FIXME: We should a smart way to figure out if the patch is attached
 182                 # to the bug, and if so obsolete it.
 183                 self._tool.bugs.post_comment_to_bug(bug_id, comment_test)
 184         else:
 185             log(comment_test)
 186             log("No bug id provided.")
 187
 188
 189 class LandDiff(Command):
 190     name = "land-diff"
 191     def __init__(self):
 192         options = [
 193             make_option("-r", "--reviewer", action="store", type="string", dest="reviewer", help="Update ChangeLogs to say Reviewed by REVIEWER."),
 194         ]
 195         options += WebKitLandingScripts.build_options()
 196         options += WebKitLandingScripts.land_options()
 197         Command.__init__(self, "Lands the current working directory diff and updates the bug if provided.", "[BUGID]", options=options)
 198
 199     def guess_reviewer_from_bug(self, bugs, bug_id):
 200         patches = bugs.fetch_reviewed_patches_from_bug(bug_id)
 201         if len(patches) != 1:
 202             log("%s on bug %s, cannot infer reviewer." % (pluralize("reviewed patch", len(patches)), bug_id))
 203             return None
 204         patch = patches[0]
 205         reviewer = patch["reviewer"]
 206         log("Guessing \"%s\" as reviewer from attachment %s on bug %s." % (reviewer, patch["id"], bug_id))
 207         return reviewer
 208
 209     def update_changelogs_with_reviewer(self, reviewer, bug_id, tool):
 210         if not reviewer:
 211             if not bug_id:
 212                 log("No bug id provided and --reviewer= not provided.  Not updating ChangeLogs with reviewer.")
 213                 return
 214             reviewer = self.guess_reviewer_from_bug(tool.bugs, bug_id)
 215
 216         if not reviewer:
 217             log("Failed to guess reviewer from bug %s and --reviewer= not provided.  Not updating ChangeLogs with reviewer." % bug_id)
 218             return
 219
 220         for changelog_path in tool.scm().modified_changelogs():
 221             ChangeLog(changelog_path).set_reviewer(reviewer)
 222
 223     def execute(self, options, args, tool):
 224         bug_id = (args and args[0]) or parse_bug_id(tool.scm().create_patch())
 225
 226         WebKitLandingScripts.ensure_builders_are_green(tool.buildbot, options)
 227
 228         os.chdir(tool.scm().checkout_root)
 229         self.update_changelogs_with_reviewer(options.reviewer, bug_id, tool)
 230
 231         fake_patch = {
 232             "id": None,
 233             "bug_id": bug_id
 234         }
 235
 236         sequence = LandDiffSequence(fake_patch, options, tool)
 237         sequence.run()
 238
 239
 240 class AbstractPatchProcessingCommand(Command):
 241     def __init__(self, help_text, args_description, options):
 242         Command.__init__(self, help_text, args_description, options=options)
 243
 244     def _fetch_list_of_patches_to_process(self, options, args, tool):
 245         raise NotImplementedError, "subclasses must implement"
 246
 247     def _prepare_to_process(self, options, args, tool):
 248         raise NotImplementedError, "subclasses must implement"
 249
 250     @staticmethod
 251     def _collect_patches_by_bug(patches):
 252         bugs_to_patches = {}
 253         for patch in patches:
 254             bug_id = patch["bug_id"]
 255             bugs_to_patches[bug_id] = bugs_to_patches.get(bug_id, []).append(patch)
 256         return bugs_to_patches
 257
 258     def execute(self, options, args, tool):
 259         if not args:
 260             error("%s required" % self.argument_names)
 261
 262         self._prepare_to_process(options, args, tool)
 263         patches = self._fetch_list_of_patches_to_process(options, args, tool)
 264
 265         # It's nice to print out total statistics.
 266         bugs_to_patches = self._collect_patches_by_bug(patches)
 267         log("Processing %s from %s." % (pluralize("patch", len(patches)), pluralize("bug", len(bugs_to_patches))))
 268
 269         for patch in patches:
 270             self._process_patch(patch, options, args, tool)
 271
 272
 273 class BuildAttachmentSequence(LandingSequence):
 274     def __init__(self, patch, options, tool):
 275         LandingSequence.__init__(self, patch, options, tool)
 276
 277     def run(self):
 278         self.clean()
 279         self.update()
 280         self.apply_patch()
 281         self.build()
 282
 283
 284 class BuildAttachment(AbstractPatchProcessingCommand):
 285     name = "build-attachment"
 286     def __init__(self):
 287         options = WebKitLandingScripts.cleaning_options()
 288         options += WebKitLandingScripts.build_options()
 289         AbstractPatchProcessingCommand.__init__(self, "Builds patches from bugzilla", "ATTACHMENT_ID [ATTACHMENT_IDS]", options)
 290
 291     def _fetch_list_of_patches_to_process(self, options, args, tool):
 292         return map(lambda patch_id: tool.bugs.fetch_attachment(patch_id), args)
 293
 294     def _prepare_to_process(self, options, args, tool):
 295         # Check the tree status first so we can fail early.
 296         WebKitLandingScripts.ensure_builders_are_green(tool.buildbot, options)
 297
 298     def _process_patch(self, patch, options, args, tool):
 299         sequence = BuildAttachmentSequence(patch, options, tool)
 300         sequence.run_and_handle_errors()
 301
 302
 303 class AbstractPatchLandingCommand(AbstractPatchProcessingCommand):
 304     def __init__(self, help_text, args_description):
 305         options = WebKitLandingScripts.cleaning_options()
 306         options += WebKitLandingScripts.build_options()
 307         options += WebKitLandingScripts.land_options()
 308         AbstractPatchProcessingCommand.__init__(self, help_text, args_description, options)
 309
 310     def _prepare_to_process(self, options, args, tool):
 311         # Check the tree status first so we can fail early.
 312         WebKitLandingScripts.ensure_builders_are_green(tool.buildbot, options)
 313
 314     def _process_patch(self, patch, options, args, tool):
 315         sequence = ConditionalLandingSequence(patch, options, tool)
 316         sequence.run_and_handle_errors()
 317
 318
 319 class LandAttachment(AbstractPatchLandingCommand):
 320     name = "land-attachment"
 321     def __init__(self):
 322         AbstractPatchLandingCommand.__init__(self, "Lands patches from bugzilla, optionally building and testing them first", "ATTACHMENT_ID [ATTACHMENT_IDS]")
 323
 324     def _fetch_list_of_patches_to_process(self, options, args, tool):
 325         return map(lambda patch_id: tool.bugs.fetch_attachment(patch_id), args)
 326
 327
 328 class LandPatches(AbstractPatchLandingCommand):
 329     name = "land-patches"
 330     def __init__(self):
 331         AbstractPatchLandingCommand.__init__(self, "Lands all patches on the given bugs, optionally building and testing them first", "BUGID [BUGIDS]")
 332
 333     def _fetch_list_of_patches_to_process(self, options, args, tool):
 334         all_patches = []
 335         for bug_id in args:
 336             patches = tool.bugs.fetch_reviewed_patches_from_bug(bug_id)
 337             log("%s found on bug %s." % (pluralize("reviewed patch", len(patches)), bug_id))
 338             all_patches += patches
 339         return all_patches
 340
 341
 342 class Rollout(Command):
 343     name = "rollout"
 344     def __init__(self):
 345         options = WebKitLandingScripts.cleaning_options()
 346         options += WebKitLandingScripts.build_options()
 347         options += WebKitLandingScripts.land_options()
 348         options.append(make_option("--complete-rollout", action="store_true", dest="complete_rollout", help="Experimental support for complete unsupervised rollouts, including re-opening the bug.  Not recommended."))
 349         Command.__init__(self, "Reverts the given revision and commits the revert and re-opens the original bug.", "REVISION [BUGID]", options=options)
 350
 351     @staticmethod
 352     def _create_changelogs_for_revert(scm, revision):
 353         # First, discard the ChangeLog changes from the rollout.
 354         changelog_paths = scm.modified_changelogs()
 355         scm.revert_files(changelog_paths)
 356
 357         # Second, make new ChangeLog entries for this rollout.
 358         # This could move to prepare-ChangeLog by adding a --revert= option.
 359         WebKitLandingScripts.run_webkit_script("prepare-ChangeLog")
 360         for changelog_path in changelog_paths:
 361             ChangeLog(changelog_path).update_for_revert(revision)
 362
 363     @staticmethod
 364     def _parse_bug_id_from_revision_diff(tool, revision):
 365         original_diff = tool.scm().diff_for_revision(revision)
 366         return parse_bug_id(original_diff)
 367
 368     @staticmethod
 369     def _reopen_bug_after_rollout(tool, bug_id, comment_text):
 370         if bug_id:
 371             tool.bugs.reopen_bug(bug_id, comment_text)
 372         else:
 373             log(comment_text)
 374             log("No bugs were updated or re-opened to reflect this rollout.")
 375
 376     def execute(self, options, args, tool):
 377         if not args:
 378             error("REVISION is required, see --help.")
 379         revision = args[0]
 380         bug_id = self._parse_bug_id_from_revision_diff(tool, revision)
 381         if options.complete_rollout:
 382             if bug_id:
 383                 log("Will re-open bug %s after rollout." % bug_id)
 384             else:
 385                 log("Failed to parse bug number from diff.  No bugs will be updated/reopened after the rollout.")
 386
 387         WebKitLandingScripts.prepare_clean_working_directory(tool.scm(), options)
 388         tool.scm().update_webkit()
 389         tool.scm().apply_reverse_diff(revision)
 390         self._create_changelogs_for_revert(tool.scm(), revision)
 391
 392         # FIXME: Fully automated rollout is not 100% idiot-proof yet, so for now just log with instructions on how to complete the rollout.
 393         # Once we trust rollout we will remove this option.
 394         if not options.complete_rollout:
 395             log("\nNOTE: Rollout support is experimental.\nPlease verify the rollout diff and use \"bugzilla-tool land-diff %s\" to commit the rollout." % bug_id)
 396         else:
 397             comment_text = WebKitLandingScripts.build_and_commit(tool.scm(), options)
 398             self._reopen_bug_after_rollout(tool, bug_id, comment_text)