[XCode] webkit-patch should run sort-Xcode-project-file
[WebKit-https.git] / Tools / Scripts / webkitpy / tool / commands / download.py
1 # Copyright (c) 2009, 2011 Google Inc. All rights reserved.
2 # Copyright (c) 2009, 2017 Apple Inc. All rights reserved.
3 #
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
6 # met:
7 #
8 #     * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 #     * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
13 # distribution.
14 #     * Neither the name of Google Inc. nor the names of its
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
17 #
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30 import logging
31
32 from webkitpy.tool import steps
33
34 from webkitpy.common.checkout.changelog import ChangeLog
35 from webkitpy.common.config import urls
36 from webkitpy.common.system.executive import ScriptError
37 from webkitpy.tool.commands.abstractsequencedcommand import AbstractSequencedCommand
38 from webkitpy.tool.commands.stepsequence import StepSequence
39 from webkitpy.tool.comments import bug_comment_from_commit_text
40 from webkitpy.tool.grammar import pluralize
41 from webkitpy.tool.multicommandtool import Command
42
43 _log = logging.getLogger(__name__)
44
45
46 class Clean(AbstractSequencedCommand):
47     name = "clean"
48     help_text = "Clean the working copy"
49     steps = [
50         steps.DiscardLocalChanges,
51     ]
52
53     def _prepare_state(self, options, args, tool):
54         options.force_clean = True
55
56
57 class Update(AbstractSequencedCommand):
58     name = "update"
59     help_text = "Update working copy (used internally)"
60     steps = [
61         steps.DiscardLocalChanges,
62         steps.Update,
63     ]
64
65
66 class Build(AbstractSequencedCommand):
67     name = "build"
68     help_text = "Update working copy and build"
69     steps = [
70         steps.DiscardLocalChanges,
71         steps.Update,
72         steps.Build,
73     ]
74
75     def _prepare_state(self, options, args, tool):
76         options.build = True
77
78
79 class BuildAndTest(AbstractSequencedCommand):
80     name = "build-and-test"
81     help_text = "Update working copy, build, and run the tests"
82     steps = [
83         steps.DiscardLocalChanges,
84         steps.Update,
85         steps.Build,
86         steps.RunTests,
87     ]
88
89
90 class CheckPatchRelevance(AbstractSequencedCommand):
91     name = "check-patch-relevance"
92     help_text = "Check if this patch needs to be tested"
93     steps = [
94         steps.CheckPatchRelevance,
95     ]
96
97
98 class Land(AbstractSequencedCommand):
99     name = "land"
100     help_text = "Land the current working directory diff and updates the associated bug if any"
101     argument_names = "[BUGID]"
102     show_in_main_help = True
103     steps = [
104         steps.AddSvnMimetypeForPng,
105         steps.UpdateChangeLogsWithReviewer,
106         steps.ValidateReviewer,
107         steps.ValidateChangeLogs,  # We do this after UpdateChangeLogsWithReviewer to avoid not having to cache the diff twice.
108         steps.Build,
109         steps.RunTests,
110         steps.Commit,
111         steps.CloseBugForLandDiff,
112     ]
113     long_help = """land commits the current working copy diff (just as svn or git commit would).
114 land will NOT build and run the tests before committing, but you can use the --build option for that.
115 If a bug id is provided, or one can be found in the ChangeLog land will update the bug after committing."""
116
117     def _prepare_state(self, options, args, tool):
118         changed_files = self._tool.scm().changed_files(options.git_commit)
119         return {
120             "changed_files": changed_files,
121             "bug_id": (args and args[0]) or tool.checkout().bug_id_for_this_commit(options.git_commit, changed_files),
122         }
123
124
125 class LandCowhand(AbstractSequencedCommand):
126     # Gender-blind term for cowboy, see: http://en.wiktionary.org/wiki/cowhand
127     name = "land-cowhand"
128     help_text = "Prepares a ChangeLog and lands the current working directory diff."
129     steps = [
130         steps.SortXcodeProjectFiles,
131         steps.PrepareChangeLog,
132         steps.EditChangeLog,
133         steps.CheckStyle,
134         steps.ConfirmDiff,
135         steps.Build,
136         steps.RunTests,
137         steps.Commit,
138         steps.CloseBugForLandDiff,
139     ]
140
141     def _prepare_state(self, options, args, tool):
142         options.check_style_filter = "-changelog"
143
144
145 class LandCowboy(LandCowhand):
146     name = "land-cowboy"
147
148     def _prepare_state(self, options, args, tool):
149         _log.warning("land-cowboy is deprecated, use land-cowhand instead.")
150         LandCowhand._prepare_state(self, options, args, tool)
151
152
153 class CheckStyleLocal(AbstractSequencedCommand):
154     name = "check-style-local"
155     help_text = "Run check-webkit-style on the current working directory diff"
156     steps = [
157         steps.CheckStyle,
158     ]
159
160
161 class AbstractPatchProcessingCommand(Command):
162     # Subclasses must implement the methods below.  We don't declare them here
163     # because we want to be able to implement them with mix-ins.
164     #
165     # pylint: disable=E1101
166     # def _fetch_list_of_patches_to_process(self, options, args, tool):
167     # def _prepare_to_process(self, options, args, tool):
168     # def _process_patch(self, options, args, tool):
169
170     @staticmethod
171     def _collect_patches_by_bug(patches):
172         bugs_to_patches = {}
173         for patch in patches:
174             bugs_to_patches[patch.bug_id()] = bugs_to_patches.get(patch.bug_id(), []) + [patch]
175         return bugs_to_patches
176
177     def execute(self, options, args, tool):
178         self._prepare_to_process(options, args, tool)
179         patches = self._fetch_list_of_patches_to_process(options, args, tool)
180
181         # It's nice to print out total statistics.
182         bugs_to_patches = self._collect_patches_by_bug(patches)
183         _log.info("Processing %s from %s." % (pluralize(len(patches), "patch"), pluralize(len(bugs_to_patches), "bug")))
184
185         for patch in patches:
186             self._process_patch(patch, options, args, tool)
187
188
189 class AbstractPatchSequencingCommand(AbstractPatchProcessingCommand):
190     prepare_steps = None
191     main_steps = None
192
193     def __init__(self):
194         options = []
195         self._prepare_sequence = StepSequence(self.prepare_steps)
196         self._main_sequence = StepSequence(self.main_steps)
197         options = sorted(set(self._prepare_sequence.options() + self._main_sequence.options()))
198         AbstractPatchProcessingCommand.__init__(self, options)
199
200     def _prepare_to_process(self, options, args, tool):
201         try:
202             self.state = self._prepare_state(options, args, tool)
203         except ScriptError, e:
204             _log.error(e.message_with_output())
205             self._exit(e.exit_code or 2)
206         self._prepare_sequence.run_and_handle_errors(tool, options, self.state)
207
208     def _process_patch(self, patch, options, args, tool):
209         state = {}
210         state.update(self.state or {})
211         state["patch"] = patch
212         self._main_sequence.run_and_handle_errors(tool, options, state)
213
214     def _prepare_state(self, options, args, tool):
215         return None
216
217
218 class ProcessAttachmentsMixin(object):
219     def _fetch_list_of_patches_to_process(self, options, args, tool):
220         return map(lambda patch_id: tool.bugs.fetch_attachment(patch_id), args)
221
222
223 class ProcessBugsMixin(object):
224     def _fetch_list_of_patches_to_process(self, options, args, tool):
225         all_patches = []
226         for bug_id in args:
227             patches = tool.bugs.fetch_bug(bug_id).reviewed_patches()
228             _log.info("%s found on bug %s." % (pluralize(len(patches), "reviewed patch"), bug_id))
229             all_patches += patches
230         if not all_patches:
231             _log.info("No reviewed patches found, looking for unreviewed patches.")
232             for bug_id in args:
233                 patches = tool.bugs.fetch_bug(bug_id).patches()
234                 _log.info("%s found on bug %s." % (pluralize(len(patches), "patch"), bug_id))
235                 all_patches += patches
236         return all_patches
237
238
239 class ProcessURLsMixin(object):
240     def _fetch_list_of_patches_to_process(self, options, args, tool):
241         all_patches = []
242         for url in args:
243             bug_id = urls.parse_bug_id(url)
244             if bug_id:
245                 patches = tool.bugs.fetch_bug(bug_id).patches()
246                 _log.info("%s found on bug %s." % (pluralize(len(patches), "patch"), bug_id))
247                 all_patches += patches
248
249             attachment_id = urls.parse_attachment_id(url)
250             if attachment_id:
251                 all_patches += tool.bugs.fetch_attachment(attachment_id)
252
253         return all_patches
254
255
256 class CheckStyle(AbstractPatchSequencingCommand, ProcessAttachmentsMixin):
257     name = "check-style"
258     help_text = "Run check-webkit-style on the specified attachments"
259     argument_names = "ATTACHMENT_ID [ATTACHMENT_IDS]"
260     main_steps = [
261         steps.DiscardLocalChanges,
262         steps.Update,
263         steps.ApplyPatch,
264         steps.CheckStyle,
265     ]
266
267
268 class BuildAttachment(AbstractPatchSequencingCommand, ProcessAttachmentsMixin):
269     name = "build-attachment"
270     help_text = "Apply and build patches from bugzilla"
271     argument_names = "ATTACHMENT_ID [ATTACHMENT_IDS]"
272     main_steps = [
273         steps.DiscardLocalChanges,
274         steps.Update,
275         steps.ApplyPatch,
276         steps.Build,
277     ]
278
279
280 class BuildAndTestAttachment(AbstractPatchSequencingCommand, ProcessAttachmentsMixin):
281     name = "build-and-test-attachment"
282     help_text = "Apply, build, and test patches from bugzilla"
283     argument_names = "ATTACHMENT_ID [ATTACHMENT_IDS]"
284     main_steps = [
285         steps.DiscardLocalChanges,
286         steps.Update,
287         steps.ApplyPatch,
288         steps.Build,
289         steps.RunTests,
290     ]
291
292
293 class AbstractPatchApplyingCommand(AbstractPatchSequencingCommand):
294     prepare_steps = [
295         steps.EnsureLocalCommitIfNeeded,
296         steps.CleanWorkingDirectory,
297         steps.Update,
298     ]
299     main_steps = [
300         steps.ApplyPatchWithLocalCommit,
301     ]
302     long_help = """Updates the working copy.
303 Downloads and applies the patches, creating local commits if necessary."""
304
305
306 class ApplyAttachment(AbstractPatchApplyingCommand, ProcessAttachmentsMixin):
307     name = "apply-attachment"
308     help_text = "Apply an attachment to the local working directory"
309     argument_names = "ATTACHMENT_ID [ATTACHMENT_IDS]"
310     show_in_main_help = True
311
312
313 class ApplyFromBug(AbstractPatchApplyingCommand, ProcessBugsMixin):
314     name = "apply-from-bug"
315     help_text = "Apply reviewed patches from provided bugs to the local working directory"
316     argument_names = "BUGID [BUGIDS]"
317     show_in_main_help = True
318
319
320 class ApplyWatchList(AbstractPatchSequencingCommand, ProcessAttachmentsMixin):
321     name = "apply-watchlist"
322     help_text = "Applies the watchlist to the specified attachments"
323     argument_names = "ATTACHMENT_ID [ATTACHMENT_IDS]"
324     main_steps = [
325         steps.DiscardLocalChanges,
326         steps.Update,
327         steps.ApplyPatch,
328         steps.ApplyWatchList,
329     ]
330     long_help = """"Applies the watchlist to the specified attachments.
331 Downloads the attachment, applies it locally, runs the watchlist against it, and updates the bug with the result."""
332
333
334 class AbstractPatchLandingCommand(AbstractPatchSequencingCommand):
335     main_steps = [
336         steps.DiscardLocalChanges,
337         steps.Update,
338         steps.ApplyPatch,
339         steps.ValidateChangeLogs,
340         steps.ValidateReviewer,
341         steps.Build,
342         steps.RunTests,
343         steps.Commit,
344         steps.ClosePatch,
345         steps.CloseBug,
346     ]
347     long_help = """Checks to make sure builders are green.
348 Updates the working copy.
349 Applies the patch.
350 Builds.
351 Runs the layout tests.
352 Commits the patch.
353 Clears the flags on the patch.
354 Closes the bug if no patches are marked for review."""
355
356
357 class LandAttachment(AbstractPatchLandingCommand, ProcessAttachmentsMixin):
358     name = "land-attachment"
359     help_text = "Land patches from bugzilla, optionally building and testing them first"
360     argument_names = "ATTACHMENT_ID [ATTACHMENT_IDS]"
361     show_in_main_help = True
362
363
364 class LandFromBug(AbstractPatchLandingCommand, ProcessBugsMixin):
365     name = "land-from-bug"
366     help_text = "Land all patches on the given bugs, optionally building and testing them first"
367     argument_names = "BUGID [BUGIDS]"
368     show_in_main_help = True
369
370
371 class LandFromURL(AbstractPatchLandingCommand, ProcessURLsMixin):
372     name = "land-from-url"
373     help_text = "Land all patches on the given URLs, optionally building and testing them first"
374     argument_names = "URL [URLS]"
375
376
377 class ValidateChangelog(AbstractSequencedCommand):
378     name = "validate-changelog"
379     help_text = "Validate that the ChangeLogs and reviewers look reasonable"
380     long_help = """Examines the current diff to see whether the ChangeLogs
381 and the reviewers listed in the ChangeLogs look reasonable.
382 """
383     steps = [
384         steps.ValidateChangeLogs,
385         steps.ValidateReviewer,
386     ]
387
388
389 class AbstractRolloutPrepCommand(AbstractSequencedCommand):
390     argument_names = "REVISION [REVISIONS] REASON"
391
392     def _commit_info(self, revision):
393         commit_info = self._tool.checkout().commit_info_for_revision(revision)
394         if commit_info and commit_info.bug_id():
395             # Note: Don't print a bug URL here because it will confuse the
396             #       SheriffBot because the SheriffBot just greps the output
397             #       of create-rollout for bug URLs.  It should do better
398             #       parsing instead.
399             _log.info("Preparing rollout for bug %s." % commit_info.bug_id())
400         else:
401             _log.info("Unable to parse bug number from diff.")
402         return commit_info
403
404     def _prepare_state(self, options, args, tool):
405         revision_list = []
406         description_list = []
407         bug_id_list = []
408         for revision in str(args[0]).split():
409             if revision.isdigit():
410                 revision_list.append(int(revision))
411             else:
412                 raise ScriptError(message="Invalid svn revision number: " + revision)
413         revision_list.sort()
414
415         earliest_revision = revision_list[0]
416         state = {
417             "revision": earliest_revision,
418             "revision_list": revision_list,
419             "reason": args[1],
420             "bug_id": None,
421             "bug_id_list": bug_id_list,
422             "description_list": description_list,
423         }
424         for revision in revision_list:
425             commit_info = self._commit_info(revision)
426             if commit_info:
427                 # We use the earliest revision for the bug info
428                 if revision == earliest_revision:
429                     state["bug_blocked"] = commit_info.bug_id()
430                     cc_list = sorted([party.bugzilla_email()
431                             for party in commit_info.responsible_parties()
432                             if party.bugzilla_email()])
433                     # FIXME: We should used the list as the canonical representation.
434                     state["bug_cc"] = ",".join(cc_list)
435                 description_list.append(commit_info.bug_description())
436                 bug_id_list.append(commit_info.bug_id())
437             else:
438                 description_list.append(None)
439                 bug_id_list.append(None)
440         return state
441
442
443 class PrepareRollout(AbstractRolloutPrepCommand):
444     name = "prepare-rollout"
445     help_text = "Revert the given revision(s) in the working copy and prepare ChangeLogs with revert reason"
446     long_help = """Updates the working copy.
447 Applies the inverse diff for the provided revision(s).
448 Creates an appropriate rollout ChangeLog, including a trac link and bug link.
449 """
450     steps = [
451         steps.DiscardLocalChanges,
452         steps.Update,
453         steps.RevertRevision,
454         steps.PrepareChangeLogForRevert,
455     ]
456
457
458 class CreateRollout(AbstractRolloutPrepCommand):
459     name = "create-rollout"
460     help_text = "Creates a bug to track the broken SVN revision(s) and uploads a rollout patch."
461     steps = [
462         steps.DiscardLocalChanges,
463         steps.Update,
464         steps.RevertRevision,
465         steps.CreateBug,
466         steps.PrepareChangeLogForRevert,
467         steps.PostDiffForRevert,
468     ]
469
470     def _prepare_state(self, options, args, tool):
471         state = AbstractRolloutPrepCommand._prepare_state(self, options, args, tool)
472         state["bug_title"] = "REGRESSION(r%s): %s" % (state["revision"], state["reason"])
473         state["bug_description"] = "%s broke the build:\n%s" % (urls.view_revision_url(state["revision"]), state["reason"])
474         # FIXME: If we had more context here, we could link to other open bugs
475         #        that mention the test that regressed.
476         if options.parent_command == "sheriff-bot":
477             state["bug_description"] += """
478
479 This is an automatic bug report generated by webkitbot. If this bug
480 report was created because of a flaky test, please file a bug for the flaky
481 test (if we don't already have one on file) and dup this bug against that bug
482 so that we can track how often these flaky tests fail.
483 """
484         return state
485
486
487 class Rollout(AbstractRolloutPrepCommand):
488     name = "rollout"
489     show_in_main_help = True
490     help_text = "Revert the given revision(s) in the working copy and optionally commit the revert and re-open the original bug"
491     long_help = """Updates the working copy.
492 Applies the inverse diff for the provided revision.
493 Creates an appropriate rollout ChangeLog, including a trac link and bug link.
494 Opens the generated ChangeLogs in $EDITOR.
495 Shows the prepared diff for confirmation.
496 Commits the revert and updates the bug (including re-opening the bug if necessary)."""
497     steps = [
498         steps.DiscardLocalChanges,
499         steps.Update,
500         steps.RevertRevision,
501         steps.PrepareChangeLogForRevert,
502         steps.EditChangeLog,
503         steps.ConfirmDiff,
504         steps.Build,
505         steps.Commit,
506         steps.ReopenBugAfterRollout,
507     ]