14b55bc3f7c9c10b5df60fc1af87d7ca39e3bc94
[WebKit-https.git] / Tools / Scripts / webkitpy / tool / commands / download.py
1 # Copyright (c) 2009, 2011 Google Inc. All rights reserved.
2 # Copyright (c) 2009, 2017 Apple Inc. All rights reserved.
3 #
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
6 # met:
7 #
8 #     * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 #     * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
13 # distribution.
14 #     * Neither the name of Google Inc. nor the names of its
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
17 #
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30 import logging
31
32 from webkitpy.tool import steps
33
34 from webkitpy.common.checkout.changelog import ChangeLog
35 from webkitpy.common.config import urls
36 from webkitpy.common.system.executive import ScriptError
37 from webkitpy.tool.commands.abstractsequencedcommand import AbstractSequencedCommand
38 from webkitpy.tool.commands.stepsequence import StepSequence
39 from webkitpy.tool.comments import bug_comment_from_commit_text
40 from webkitpy.tool.grammar import pluralize
41 from webkitpy.tool.multicommandtool import Command
42
43 _log = logging.getLogger(__name__)
44
45
46 class Clean(AbstractSequencedCommand):
47     name = "clean"
48     help_text = "Clean the working copy"
49     steps = [
50         steps.DiscardLocalChanges,
51     ]
52
53     def _prepare_state(self, options, args, tool):
54         options.force_clean = True
55
56
57 class Update(AbstractSequencedCommand):
58     name = "update"
59     help_text = "Update working copy (used internally)"
60     steps = [
61         steps.DiscardLocalChanges,
62         steps.Update,
63     ]
64
65
66 class Build(AbstractSequencedCommand):
67     name = "build"
68     help_text = "Update working copy and build"
69     steps = [
70         steps.DiscardLocalChanges,
71         steps.Update,
72         steps.Build,
73     ]
74
75     def _prepare_state(self, options, args, tool):
76         options.build = True
77
78
79 class BuildAndTest(AbstractSequencedCommand):
80     name = "build-and-test"
81     help_text = "Update working copy, build, and run the tests"
82     steps = [
83         steps.DiscardLocalChanges,
84         steps.Update,
85         steps.Build,
86         steps.RunTests,
87     ]
88
89
90 class CheckPatchRelevance(AbstractSequencedCommand):
91     name = "check-patch-relevance"
92     help_text = "Check if this patch needs to be tested"
93     steps = [
94         steps.CheckPatchRelevance,
95     ]
96
97
98 class Land(AbstractSequencedCommand):
99     name = "land"
100     help_text = "Land the current working directory diff and updates the associated bug if any"
101     argument_names = "[BUGID]"
102     show_in_main_help = True
103     steps = [
104         steps.AddSvnMimetypeForPng,
105         steps.UpdateChangeLogsWithReviewer,
106         steps.ValidateReviewer,
107         steps.ValidateChangeLogs,  # We do this after UpdateChangeLogsWithReviewer to avoid not having to cache the diff twice.
108         steps.Build,
109         steps.RunTests,
110         steps.Commit,
111         steps.CloseBugForLandDiff,
112     ]
113     long_help = """land commits the current working copy diff (just as svn or git commit would).
114 land will NOT build and run the tests before committing, but you can use the --build option for that.
115 If a bug id is provided, or one can be found in the ChangeLog land will update the bug after committing."""
116
117     def _prepare_state(self, options, args, tool):
118         changed_files = self._tool.scm().changed_files(options.git_commit)
119         return {
120             "changed_files": changed_files,
121             "bug_id": (args and args[0]) or tool.checkout().bug_id_for_this_commit(options.git_commit, changed_files),
122         }
123
124
125 class LandCowhand(AbstractSequencedCommand):
126     # Gender-blind term for cowboy, see: http://en.wiktionary.org/wiki/cowhand
127     name = "land-cowhand"
128     help_text = "Prepares a ChangeLog and lands the current working directory diff."
129     steps = [
130         steps.PrepareChangeLog,
131         steps.EditChangeLog,
132         steps.CheckStyle,
133         steps.ConfirmDiff,
134         steps.Build,
135         steps.RunTests,
136         steps.Commit,
137         steps.CloseBugForLandDiff,
138     ]
139
140     def _prepare_state(self, options, args, tool):
141         options.check_style_filter = "-changelog"
142
143
144 class LandCowboy(LandCowhand):
145     name = "land-cowboy"
146
147     def _prepare_state(self, options, args, tool):
148         _log.warning("land-cowboy is deprecated, use land-cowhand instead.")
149         LandCowhand._prepare_state(self, options, args, tool)
150
151
152 class CheckStyleLocal(AbstractSequencedCommand):
153     name = "check-style-local"
154     help_text = "Run check-webkit-style on the current working directory diff"
155     steps = [
156         steps.CheckStyle,
157     ]
158
159
160 class AbstractPatchProcessingCommand(Command):
161     # Subclasses must implement the methods below.  We don't declare them here
162     # because we want to be able to implement them with mix-ins.
163     #
164     # pylint: disable=E1101
165     # def _fetch_list_of_patches_to_process(self, options, args, tool):
166     # def _prepare_to_process(self, options, args, tool):
167     # def _process_patch(self, options, args, tool):
168
169     @staticmethod
170     def _collect_patches_by_bug(patches):
171         bugs_to_patches = {}
172         for patch in patches:
173             bugs_to_patches[patch.bug_id()] = bugs_to_patches.get(patch.bug_id(), []) + [patch]
174         return bugs_to_patches
175
176     def execute(self, options, args, tool):
177         self._prepare_to_process(options, args, tool)
178         patches = self._fetch_list_of_patches_to_process(options, args, tool)
179
180         # It's nice to print out total statistics.
181         bugs_to_patches = self._collect_patches_by_bug(patches)
182         _log.info("Processing %s from %s." % (pluralize(len(patches), "patch"), pluralize(len(bugs_to_patches), "bug")))
183
184         for patch in patches:
185             self._process_patch(patch, options, args, tool)
186
187
188 class AbstractPatchSequencingCommand(AbstractPatchProcessingCommand):
189     prepare_steps = None
190     main_steps = None
191
192     def __init__(self):
193         options = []
194         self._prepare_sequence = StepSequence(self.prepare_steps)
195         self._main_sequence = StepSequence(self.main_steps)
196         options = sorted(set(self._prepare_sequence.options() + self._main_sequence.options()))
197         AbstractPatchProcessingCommand.__init__(self, options)
198
199     def _prepare_to_process(self, options, args, tool):
200         try:
201             self.state = self._prepare_state(options, args, tool)
202         except ScriptError, e:
203             _log.error(e.message_with_output())
204             self._exit(e.exit_code or 2)
205         self._prepare_sequence.run_and_handle_errors(tool, options, self.state)
206
207     def _process_patch(self, patch, options, args, tool):
208         state = {}
209         state.update(self.state or {})
210         state["patch"] = patch
211         self._main_sequence.run_and_handle_errors(tool, options, state)
212
213     def _prepare_state(self, options, args, tool):
214         return None
215
216
217 class ProcessAttachmentsMixin(object):
218     def _fetch_list_of_patches_to_process(self, options, args, tool):
219         return map(lambda patch_id: tool.bugs.fetch_attachment(patch_id), args)
220
221
222 class ProcessBugsMixin(object):
223     def _fetch_list_of_patches_to_process(self, options, args, tool):
224         all_patches = []
225         for bug_id in args:
226             patches = tool.bugs.fetch_bug(bug_id).reviewed_patches()
227             _log.info("%s found on bug %s." % (pluralize(len(patches), "reviewed patch"), bug_id))
228             all_patches += patches
229         if not all_patches:
230             _log.info("No reviewed patches found, looking for unreviewed patches.")
231             for bug_id in args:
232                 patches = tool.bugs.fetch_bug(bug_id).patches()
233                 _log.info("%s found on bug %s." % (pluralize(len(patches), "patch"), bug_id))
234                 all_patches += patches
235         return all_patches
236
237
238 class ProcessURLsMixin(object):
239     def _fetch_list_of_patches_to_process(self, options, args, tool):
240         all_patches = []
241         for url in args:
242             bug_id = urls.parse_bug_id(url)
243             if bug_id:
244                 patches = tool.bugs.fetch_bug(bug_id).patches()
245                 _log.info("%s found on bug %s." % (pluralize(len(patches), "patch"), bug_id))
246                 all_patches += patches
247
248             attachment_id = urls.parse_attachment_id(url)
249             if attachment_id:
250                 all_patches += tool.bugs.fetch_attachment(attachment_id)
251
252         return all_patches
253
254
255 class CheckStyle(AbstractPatchSequencingCommand, ProcessAttachmentsMixin):
256     name = "check-style"
257     help_text = "Run check-webkit-style on the specified attachments"
258     argument_names = "ATTACHMENT_ID [ATTACHMENT_IDS]"
259     main_steps = [
260         steps.DiscardLocalChanges,
261         steps.Update,
262         steps.ApplyPatch,
263         steps.CheckStyle,
264     ]
265
266
267 class BuildAttachment(AbstractPatchSequencingCommand, ProcessAttachmentsMixin):
268     name = "build-attachment"
269     help_text = "Apply and build patches from bugzilla"
270     argument_names = "ATTACHMENT_ID [ATTACHMENT_IDS]"
271     main_steps = [
272         steps.DiscardLocalChanges,
273         steps.Update,
274         steps.ApplyPatch,
275         steps.Build,
276     ]
277
278
279 class BuildAndTestAttachment(AbstractPatchSequencingCommand, ProcessAttachmentsMixin):
280     name = "build-and-test-attachment"
281     help_text = "Apply, build, and test patches from bugzilla"
282     argument_names = "ATTACHMENT_ID [ATTACHMENT_IDS]"
283     main_steps = [
284         steps.DiscardLocalChanges,
285         steps.Update,
286         steps.ApplyPatch,
287         steps.Build,
288         steps.RunTests,
289     ]
290
291
292 class AbstractPatchApplyingCommand(AbstractPatchSequencingCommand):
293     prepare_steps = [
294         steps.EnsureLocalCommitIfNeeded,
295         steps.CleanWorkingDirectory,
296         steps.Update,
297     ]
298     main_steps = [
299         steps.ApplyPatchWithLocalCommit,
300     ]
301     long_help = """Updates the working copy.
302 Downloads and applies the patches, creating local commits if necessary."""
303
304
305 class ApplyAttachment(AbstractPatchApplyingCommand, ProcessAttachmentsMixin):
306     name = "apply-attachment"
307     help_text = "Apply an attachment to the local working directory"
308     argument_names = "ATTACHMENT_ID [ATTACHMENT_IDS]"
309     show_in_main_help = True
310
311
312 class ApplyFromBug(AbstractPatchApplyingCommand, ProcessBugsMixin):
313     name = "apply-from-bug"
314     help_text = "Apply reviewed patches from provided bugs to the local working directory"
315     argument_names = "BUGID [BUGIDS]"
316     show_in_main_help = True
317
318
319 class ApplyWatchList(AbstractPatchSequencingCommand, ProcessAttachmentsMixin):
320     name = "apply-watchlist"
321     help_text = "Applies the watchlist to the specified attachments"
322     argument_names = "ATTACHMENT_ID [ATTACHMENT_IDS]"
323     main_steps = [
324         steps.DiscardLocalChanges,
325         steps.Update,
326         steps.ApplyPatch,
327         steps.ApplyWatchList,
328     ]
329     long_help = """"Applies the watchlist to the specified attachments.
330 Downloads the attachment, applies it locally, runs the watchlist against it, and updates the bug with the result."""
331
332
333 class AbstractPatchLandingCommand(AbstractPatchSequencingCommand):
334     main_steps = [
335         steps.DiscardLocalChanges,
336         steps.Update,
337         steps.ApplyPatch,
338         steps.ValidateChangeLogs,
339         steps.ValidateReviewer,
340         steps.Build,
341         steps.RunTests,
342         steps.Commit,
343         steps.ClosePatch,
344         steps.CloseBug,
345     ]
346     long_help = """Checks to make sure builders are green.
347 Updates the working copy.
348 Applies the patch.
349 Builds.
350 Runs the layout tests.
351 Commits the patch.
352 Clears the flags on the patch.
353 Closes the bug if no patches are marked for review."""
354
355
356 class LandAttachment(AbstractPatchLandingCommand, ProcessAttachmentsMixin):
357     name = "land-attachment"
358     help_text = "Land patches from bugzilla, optionally building and testing them first"
359     argument_names = "ATTACHMENT_ID [ATTACHMENT_IDS]"
360     show_in_main_help = True
361
362
363 class LandFromBug(AbstractPatchLandingCommand, ProcessBugsMixin):
364     name = "land-from-bug"
365     help_text = "Land all patches on the given bugs, optionally building and testing them first"
366     argument_names = "BUGID [BUGIDS]"
367     show_in_main_help = True
368
369
370 class LandFromURL(AbstractPatchLandingCommand, ProcessURLsMixin):
371     name = "land-from-url"
372     help_text = "Land all patches on the given URLs, optionally building and testing them first"
373     argument_names = "URL [URLS]"
374
375
376 class ValidateChangelog(AbstractSequencedCommand):
377     name = "validate-changelog"
378     help_text = "Validate that the ChangeLogs and reviewers look reasonable"
379     long_help = """Examines the current diff to see whether the ChangeLogs
380 and the reviewers listed in the ChangeLogs look reasonable.
381 """
382     steps = [
383         steps.ValidateChangeLogs,
384         steps.ValidateReviewer,
385     ]
386
387
388 class AbstractRolloutPrepCommand(AbstractSequencedCommand):
389     argument_names = "REVISION [REVISIONS] REASON"
390
391     def _commit_info(self, revision):
392         commit_info = self._tool.checkout().commit_info_for_revision(revision)
393         if commit_info and commit_info.bug_id():
394             # Note: Don't print a bug URL here because it will confuse the
395             #       SheriffBot because the SheriffBot just greps the output
396             #       of create-rollout for bug URLs.  It should do better
397             #       parsing instead.
398             _log.info("Preparing rollout for bug %s." % commit_info.bug_id())
399         else:
400             _log.info("Unable to parse bug number from diff.")
401         return commit_info
402
403     def _prepare_state(self, options, args, tool):
404         revision_list = []
405         description_list = []
406         bug_id_list = []
407         for revision in str(args[0]).split():
408             if revision.isdigit():
409                 revision_list.append(int(revision))
410             else:
411                 raise ScriptError(message="Invalid svn revision number: " + revision)
412         revision_list.sort()
413
414         earliest_revision = revision_list[0]
415         state = {
416             "revision": earliest_revision,
417             "revision_list": revision_list,
418             "reason": args[1],
419             "bug_id": None,
420             "bug_id_list": bug_id_list,
421             "description_list": description_list,
422         }
423         for revision in revision_list:
424             commit_info = self._commit_info(revision)
425             if commit_info:
426                 # We use the earliest revision for the bug info
427                 if revision == earliest_revision:
428                     state["bug_blocked"] = commit_info.bug_id()
429                     cc_list = sorted([party.bugzilla_email()
430                             for party in commit_info.responsible_parties()
431                             if party.bugzilla_email()])
432                     # FIXME: We should used the list as the canonical representation.
433                     state["bug_cc"] = ",".join(cc_list)
434                 description_list.append(commit_info.bug_description())
435                 bug_id_list.append(commit_info.bug_id())
436             else:
437                 description_list.append(None)
438                 bug_id_list.append(None)
439         return state
440
441
442 class PrepareRollout(AbstractRolloutPrepCommand):
443     name = "prepare-rollout"
444     help_text = "Revert the given revision(s) in the working copy and prepare ChangeLogs with revert reason"
445     long_help = """Updates the working copy.
446 Applies the inverse diff for the provided revision(s).
447 Creates an appropriate rollout ChangeLog, including a trac link and bug link.
448 """
449     steps = [
450         steps.DiscardLocalChanges,
451         steps.Update,
452         steps.RevertRevision,
453         steps.PrepareChangeLogForRevert,
454     ]
455
456
457 class CreateRollout(AbstractRolloutPrepCommand):
458     name = "create-rollout"
459     help_text = "Creates a bug to track the broken SVN revision(s) and uploads a rollout patch."
460     steps = [
461         steps.DiscardLocalChanges,
462         steps.Update,
463         steps.RevertRevision,
464         steps.CreateBug,
465         steps.PrepareChangeLogForRevert,
466         steps.PostDiffForRevert,
467     ]
468
469     def _prepare_state(self, options, args, tool):
470         state = AbstractRolloutPrepCommand._prepare_state(self, options, args, tool)
471         state["bug_title"] = "REGRESSION(r%s): %s" % (state["revision"], state["reason"])
472         state["bug_description"] = "%s broke the build:\n%s" % (urls.view_revision_url(state["revision"]), state["reason"])
473         # FIXME: If we had more context here, we could link to other open bugs
474         #        that mention the test that regressed.
475         if options.parent_command == "sheriff-bot":
476             state["bug_description"] += """
477
478 This is an automatic bug report generated by webkitbot. If this bug
479 report was created because of a flaky test, please file a bug for the flaky
480 test (if we don't already have one on file) and dup this bug against that bug
481 so that we can track how often these flaky tests fail.
482 """
483         return state
484
485
486 class Rollout(AbstractRolloutPrepCommand):
487     name = "rollout"
488     show_in_main_help = True
489     help_text = "Revert the given revision(s) in the working copy and optionally commit the revert and re-open the original bug"
490     long_help = """Updates the working copy.
491 Applies the inverse diff for the provided revision.
492 Creates an appropriate rollout ChangeLog, including a trac link and bug link.
493 Opens the generated ChangeLogs in $EDITOR.
494 Shows the prepared diff for confirmation.
495 Commits the revert and updates the bug (including re-opening the bug if necessary)."""
496     steps = [
497         steps.DiscardLocalChanges,
498         steps.Update,
499         steps.RevertRevision,
500         steps.PrepareChangeLogForRevert,
501         steps.EditChangeLog,
502         steps.ConfirmDiff,
503         steps.Build,
504         steps.Commit,
505         steps.ReopenBugAfterRollout,
506     ]