WebKitTools/Scripts/modules/commands/download.py

   1 #!/usr/bin/env python
   2 # Copyright (c) 2009, Google Inc. All rights reserved.
   3 # Copyright (c) 2009 Apple Inc. All rights reserved.
   4 #
   5 # Redistribution and use in source and binary forms, with or without
   6 # modification, are permitted provided that the following conditions are
   7 # met:
   8 #
   9 #     * Redistributions of source code must retain the above copyright
  10 # notice, this list of conditions and the following disclaimer.
  11 #     * Redistributions in binary form must reproduce the above
  12 # copyright notice, this list of conditions and the following disclaimer
  13 # in the documentation and/or other materials provided with the
  14 # distribution.
  15 #     * Neither the name of Google Inc. nor the names of its
  16 # contributors may be used to endorse or promote products derived from
  17 # this software without specific prior written permission.
  18 #
  19 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  20 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  21 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  22 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  23 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  24 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  25 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  26 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  27 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  28 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  29 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  30
  31 # FIXME: Trim down this import list once we have unit tests.
  32 import os
  33 import re
  34 import StringIO
  35 import subprocess
  36 import sys
  37 import time
  38
  39 from datetime import datetime, timedelta
  40 from optparse import make_option
  41
  42 from modules.bugzilla import Bugzilla, parse_bug_id
  43 from modules.buildbot import BuildBot
  44 from modules.buildsteps import BuildSteps
  45 from modules.changelogs import ChangeLog
  46 from modules.comments import bug_comment_from_commit_text
  47 from modules.grammar import pluralize
  48 from modules.landingsequence import LandingSequence, ConditionalLandingSequence
  49 from modules.logging import error, log, tee
  50 from modules.multicommandtool import MultiCommandTool, Command
  51 from modules.patchcollection import PatchCollection
  52 from modules.scm import CommitMessage, detect_scm_system, ScriptError, CheckoutNeedsUpdate
  53 from modules.statusbot import StatusBot
  54 from modules.webkitport import WebKitPort
  55 from modules.workqueue import WorkQueue, WorkQueueDelegate
  56
  57
  58 class BuildSequence(ConditionalLandingSequence):
  59     def __init__(self, options, tool):
  60         ConditionalLandingSequence.__init__(self, None, options, tool)
  61
  62     def run(self):
  63         self.clean()
  64         self.update()
  65         self.build()
  66
  67
  68 class Build(Command):
  69     name = "build"
  70     show_in_main_help = False
  71     def __init__(self):
  72         options = BuildSteps.cleaning_options()
  73         options += BuildSteps.build_options()
  74         options += BuildSteps.land_options()
  75         Command.__init__(self, "Update working copy and build", "", options)
  76
  77     def execute(self, options, args, tool):
  78         sequence = BuildSequence(options, tool)
  79         sequence.run_and_handle_errors()
  80
  81
  82 # FIXME: Requires unit test. Blocking issue: WebKitApplyingScripts
  83 class ApplyAttachment(Command):
  84     name = "apply-attachment"
  85     show_in_main_help = True
  86     def __init__(self):
  87         options = WebKitApplyingScripts.apply_options()
  88         options += BuildSteps.cleaning_options()
  89         Command.__init__(self, "Apply an attachment to the local working directory", "ATTACHMENT_ID", options=options)
  90
  91     def execute(self, options, args, tool):
  92         WebKitApplyingScripts.setup_for_patch_apply(tool, options)
  93         attachment_id = args[0]
  94         attachment = tool.bugs.fetch_attachment(attachment_id)
  95         WebKitApplyingScripts.apply_patches_with_options(tool.scm(), [attachment], options)
  96
  97
  98 # FIXME: Requires unit test. Blocking issue: WebKitApplyingScripts
  99 class ApplyPatches(Command):
 100     name = "apply-patches"
 101     show_in_main_help = True
 102     def __init__(self):
 103         options = WebKitApplyingScripts.apply_options()
 104         options += BuildSteps.cleaning_options()
 105         Command.__init__(self, "Apply reviewed patches from provided bugs to the local working directory", "BUGID", options=options)
 106
 107     def execute(self, options, args, tool):
 108         WebKitApplyingScripts.setup_for_patch_apply(tool, options)
 109         bug_id = args[0]
 110         patches = tool.bugs.fetch_reviewed_patches_from_bug(bug_id)
 111         WebKitApplyingScripts.apply_patches_with_options(tool.scm(), patches, options)
 112
 113
 114 class WebKitApplyingScripts:
 115     @staticmethod
 116     def apply_options():
 117         return [
 118             make_option("--no-update", action="store_false", dest="update", default=True, help="Don't update the working directory before applying patches"),
 119             make_option("--local-commit", action="store_true", dest="local_commit", default=False, help="Make a local commit for each applied patch"),
 120         ]
 121
 122     @staticmethod
 123     def setup_for_patch_apply(tool, options):
 124         tool.steps.clean_working_directory(tool.scm(), options, allow_local_commits=True)
 125         if options.update:
 126             tool.scm().update_webkit()
 127
 128     @staticmethod
 129     def apply_patches_with_options(scm, patches, options):
 130         if options.local_commit and not scm.supports_local_commits():
 131             error("--local-commit passed, but %s does not support local commits" % scm.display_name())
 132
 133         for patch in patches:
 134             log("Applying attachment %s from bug %s" % (patch["id"], patch["bug_id"]))
 135             scm.apply_patch(patch)
 136             if options.local_commit:
 137                 commit_message = scm.commit_message_for_this_commit()
 138                 scm.commit_locally_with_message(commit_message.message() or patch["name"])
 139
 140
 141 class LandDiffSequence(ConditionalLandingSequence):
 142     def __init__(self, patch, options, tool):
 143         ConditionalLandingSequence.__init__(self, patch, options, tool)
 144
 145     def run(self):
 146         self.build()
 147         self.test()
 148         commit_log = self.commit()
 149         self.close_bug(commit_log)
 150
 151     def close_bug(self, commit_log):
 152         comment_test = bug_comment_from_commit_text(self._tool.scm(), commit_log)
 153         bug_id = self._patch["bug_id"]
 154         if bug_id:
 155             log("Updating bug %s" % bug_id)
 156             if self._options.close_bug:
 157                 self._tool.bugs.close_bug_as_fixed(bug_id, comment_test)
 158             else:
 159                 # FIXME: We should a smart way to figure out if the patch is attached
 160                 # to the bug, and if so obsolete it.
 161                 self._tool.bugs.post_comment_to_bug(bug_id, comment_test)
 162         else:
 163             log(comment_test)
 164             log("No bug id provided.")
 165
 166
 167 class LandDiff(Command):
 168     name = "land-diff"
 169     show_in_main_help = True
 170     def __init__(self):
 171         options = [
 172             make_option("-r", "--reviewer", action="store", type="string", dest="reviewer", help="Update ChangeLogs to say Reviewed by REVIEWER."),
 173         ]
 174         options += BuildSteps.build_options()
 175         options += BuildSteps.land_options()
 176         Command.__init__(self, "Land the current working directory diff and updates the associated bug if any", "[BUGID]", options=options)
 177
 178     def guess_reviewer_from_bug(self, bugs, bug_id):
 179         patches = bugs.fetch_reviewed_patches_from_bug(bug_id)
 180         if len(patches) != 1:
 181             log("%s on bug %s, cannot infer reviewer." % (pluralize("reviewed patch", len(patches)), bug_id))
 182             return None
 183         patch = patches[0]
 184         reviewer = patch["reviewer"]
 185         log("Guessing \"%s\" as reviewer from attachment %s on bug %s." % (reviewer, patch["id"], bug_id))
 186         return reviewer
 187
 188     def update_changelogs_with_reviewer(self, reviewer, bug_id, tool):
 189         if not reviewer:
 190             if not bug_id:
 191                 log("No bug id provided and --reviewer= not provided.  Not updating ChangeLogs with reviewer.")
 192                 return
 193             reviewer = self.guess_reviewer_from_bug(tool.bugs, bug_id)
 194
 195         if not reviewer:
 196             log("Failed to guess reviewer from bug %s and --reviewer= not provided.  Not updating ChangeLogs with reviewer." % bug_id)
 197             return
 198
 199         for changelog_path in tool.scm().modified_changelogs():
 200             ChangeLog(changelog_path).set_reviewer(reviewer)
 201
 202     def execute(self, options, args, tool):
 203         bug_id = (args and args[0]) or parse_bug_id(tool.scm().create_patch())
 204
 205         tool.steps.ensure_builders_are_green(tool.buildbot, options)
 206
 207         os.chdir(tool.scm().checkout_root)
 208         self.update_changelogs_with_reviewer(options.reviewer, bug_id, tool)
 209
 210         fake_patch = {
 211             "id": None,
 212             "bug_id": bug_id
 213         }
 214
 215         sequence = LandDiffSequence(fake_patch, options, tool)
 216         sequence.run()
 217
 218
 219 class AbstractPatchProcessingCommand(Command):
 220     def __init__(self, help_text, args_description, options):
 221         Command.__init__(self, help_text, args_description, options=options)
 222
 223     def _fetch_list_of_patches_to_process(self, options, args, tool):
 224         raise NotImplementedError, "subclasses must implement"
 225
 226     def _prepare_to_process(self, options, args, tool):
 227         raise NotImplementedError, "subclasses must implement"
 228
 229     @staticmethod
 230     def _collect_patches_by_bug(patches):
 231         bugs_to_patches = {}
 232         for patch in patches:
 233             bug_id = patch["bug_id"]
 234             bugs_to_patches[bug_id] = bugs_to_patches.get(bug_id, []) + [patch]
 235         return bugs_to_patches
 236
 237     def execute(self, options, args, tool):
 238         self._prepare_to_process(options, args, tool)
 239         patches = self._fetch_list_of_patches_to_process(options, args, tool)
 240
 241         # It's nice to print out total statistics.
 242         bugs_to_patches = self._collect_patches_by_bug(patches)
 243         log("Processing %s from %s." % (pluralize("patch", len(patches)), pluralize("bug", len(bugs_to_patches))))
 244
 245         for patch in patches:
 246             self._process_patch(patch, options, args, tool)
 247
 248
 249 class CheckStyleSequence(LandingSequence):
 250     def __init__(self, patch, options, tool):
 251         LandingSequence.__init__(self, patch, options, tool)
 252
 253     def run(self):
 254         self.clean()
 255         self.update()
 256         self.apply_patch()
 257         self.build()
 258
 259     def build(self):
 260         # Instead of building, we check style.
 261         self._tool.steps.check_style()
 262
 263
 264 class CheckStyle(AbstractPatchProcessingCommand):
 265     name = "check-style"
 266     show_in_main_help = False
 267     def __init__(self):
 268         options = BuildSteps.cleaning_options()
 269         options += BuildSteps.build_options()
 270         AbstractPatchProcessingCommand.__init__(self, "Run check-webkit-style on the specified attachments", "ATTACHMENT_ID [ATTACHMENT_IDS]", options)
 271
 272     def _fetch_list_of_patches_to_process(self, options, args, tool):
 273         return map(lambda patch_id: tool.bugs.fetch_attachment(patch_id), args)
 274
 275     def _prepare_to_process(self, options, args, tool):
 276         pass
 277
 278     def _process_patch(self, patch, options, args, tool):
 279         sequence = CheckStyleSequence(patch, options, tool)
 280         sequence.run_and_handle_errors()
 281
 282
 283 class BuildAttachmentSequence(LandingSequence):
 284     def __init__(self, patch, options, tool):
 285         LandingSequence.__init__(self, patch, options, tool)
 286
 287     def run(self):
 288         self.clean()
 289         self.update()
 290         self.apply_patch()
 291         self.build()
 292
 293
 294 class BuildAttachment(AbstractPatchProcessingCommand):
 295     name = "build-attachment"
 296     show_in_main_help = False
 297     def __init__(self):
 298         options = BuildSteps.cleaning_options()
 299         options += BuildSteps.build_options()
 300         AbstractPatchProcessingCommand.__init__(self, "Apply and build patches from bugzilla", "ATTACHMENT_ID [ATTACHMENT_IDS]", options)
 301
 302     def _fetch_list_of_patches_to_process(self, options, args, tool):
 303         return map(lambda patch_id: tool.bugs.fetch_attachment(patch_id), args)
 304
 305     def _prepare_to_process(self, options, args, tool):
 306         # Check the tree status first so we can fail early.
 307         tool.steps.ensure_builders_are_green(tool.buildbot, options)
 308
 309     def _process_patch(self, patch, options, args, tool):
 310         sequence = BuildAttachmentSequence(patch, options, tool)
 311         sequence.run_and_handle_errors()
 312
 313
 314 class AbstractPatchLandingCommand(AbstractPatchProcessingCommand):
 315     def __init__(self, help_text, args_description):
 316         options = BuildSteps.cleaning_options()
 317         options += BuildSteps.build_options()
 318         options += BuildSteps.land_options()
 319         AbstractPatchProcessingCommand.__init__(self, help_text, args_description, options)
 320
 321     def _prepare_to_process(self, options, args, tool):
 322         # Check the tree status first so we can fail early.
 323         tool.steps.ensure_builders_are_green(tool.buildbot, options)
 324
 325     def _process_patch(self, patch, options, args, tool):
 326         sequence = ConditionalLandingSequence(patch, options, tool)
 327         sequence.run_and_handle_errors()
 328
 329
 330 class LandAttachment(AbstractPatchLandingCommand):
 331     name = "land-attachment"
 332     show_in_main_help = True
 333     def __init__(self):
 334         AbstractPatchLandingCommand.__init__(self, "Land patches from bugzilla, optionally building and testing them first", "ATTACHMENT_ID [ATTACHMENT_IDS]")
 335
 336     def _fetch_list_of_patches_to_process(self, options, args, tool):
 337         return map(lambda patch_id: tool.bugs.fetch_attachment(patch_id), args)
 338
 339
 340 class LandPatches(AbstractPatchLandingCommand):
 341     name = "land-patches"
 342     show_in_main_help = True
 343     def __init__(self):
 344         AbstractPatchLandingCommand.__init__(self, "Land all patches on the given bugs, optionally building and testing them first", "BUGID [BUGIDS]")
 345
 346     def _fetch_list_of_patches_to_process(self, options, args, tool):
 347         all_patches = []
 348         for bug_id in args:
 349             patches = tool.bugs.fetch_reviewed_patches_from_bug(bug_id)
 350             log("%s found on bug %s." % (pluralize("reviewed patch", len(patches)), bug_id))
 351             all_patches += patches
 352         return all_patches
 353
 354
 355 # FIXME: Requires unit test.
 356 class Rollout(Command):
 357     name = "rollout"
 358     show_in_main_help = True
 359     def __init__(self):
 360         options = BuildSteps.cleaning_options()
 361         options += BuildSteps.build_options()
 362         options += BuildSteps.land_options()
 363         options.append(make_option("--complete-rollout", action="store_true", dest="complete_rollout", help="Commit the revert and re-open the original bug."))
 364         Command.__init__(self, "Revert the given revision in the working copy and optionally commit the revert and re-open the original bug", "REVISION [BUGID]", options=options)
 365
 366     @staticmethod
 367     def _create_changelogs_for_revert(tool, revision):
 368         # First, discard the ChangeLog changes from the rollout.
 369         changelog_paths = tool.scm().modified_changelogs()
 370         tool.scm().revert_files(changelog_paths)
 371
 372         # Second, make new ChangeLog entries for this rollout.
 373         # This could move to prepare-ChangeLog by adding a --revert= option.
 374         tool.steps.prepare_changelog()
 375         for changelog_path in changelog_paths:
 376             ChangeLog(changelog_path).update_for_revert(revision)
 377
 378     @staticmethod
 379     def _parse_bug_id_from_revision_diff(tool, revision):
 380         original_diff = tool.scm().diff_for_revision(revision)
 381         return parse_bug_id(original_diff)
 382
 383     @staticmethod
 384     def _reopen_bug_after_rollout(tool, bug_id, comment_text):
 385         if bug_id:
 386             tool.bugs.reopen_bug(bug_id, comment_text)
 387         else:
 388             log(comment_text)
 389             log("No bugs were updated or re-opened to reflect this rollout.")
 390
 391     def execute(self, options, args, tool):
 392         revision = args[0]
 393         bug_id = self._parse_bug_id_from_revision_diff(tool, revision)
 394         if options.complete_rollout:
 395             if bug_id:
 396                 log("Will re-open bug %s after rollout." % bug_id)
 397             else:
 398                 log("Failed to parse bug number from diff.  No bugs will be updated/reopened after the rollout.")
 399
 400         tool.steps.clean_working_directory(tool.scm(), options)
 401         tool.scm().update_webkit()
 402         tool.scm().apply_reverse_diff(revision)
 403         self._create_changelogs_for_revert(tool, revision)
 404
 405         # FIXME: Fully automated rollout is not 100% idiot-proof yet, so for now just log with instructions on how to complete the rollout.
 406         # Once we trust rollout we will remove this option.
 407         if not options.complete_rollout:
 408             log("\nNOTE: Rollout support is experimental.\nPlease verify the rollout diff and use \"bugzilla-tool land-diff %s\" to commit the rollout." % bug_id)
 409         else:
 410             # FIXME: This function does not exist!!
 411             # comment_text = WebKitLandingScripts.build_and_commit(tool.scm(), options)
 412             raise ScriptError("OOPS! This option is not implemented (yet).")
 413             self._reopen_bug_after_rollout(tool, bug_id, comment_text)