[JSC] Int52Rep(DoubleRepAnyIntUse) should not call operation function
[WebKit-https.git] / Tools / Scripts / webkitpy / tool / commands / download.py
1 # Copyright (c) 2009, 2011 Google Inc. All rights reserved.
2 # Copyright (c) 2009, 2017-2018 Apple Inc. All rights reserved.
3 #
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
6 # met:
7 #
8 #     * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 #     * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
13 # distribution.
14 #     * Neither the name of Google Inc. nor the names of its
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
17 #
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30 import logging
31
32 from webkitpy.tool import steps
33
34 from webkitpy.common.checkout.changelog import ChangeLog
35 from webkitpy.common.config import urls
36 from webkitpy.common.net.bugzilla import Bugzilla
37 from webkitpy.common.system.executive import ScriptError
38 from webkitpy.tool.commands.abstractsequencedcommand import AbstractSequencedCommand
39 from webkitpy.tool.commands.stepsequence import StepSequence
40 from webkitpy.tool.comments import bug_comment_from_commit_text
41 from webkitpy.tool.grammar import pluralize
42 from webkitpy.tool.multicommandtool import Command
43
44 _log = logging.getLogger(__name__)
45
46
47 class Clean(AbstractSequencedCommand):
48     name = "clean"
49     help_text = "Clean the working copy"
50     steps = [
51         steps.DiscardLocalChanges,
52     ]
53
54     def _prepare_state(self, options, args, tool):
55         options.force_clean = True
56
57
58 class Update(AbstractSequencedCommand):
59     name = "update"
60     help_text = "Update working copy (used internally)"
61     steps = [
62         steps.DiscardLocalChanges,
63         steps.Update,
64     ]
65
66
67 class Build(AbstractSequencedCommand):
68     name = "build"
69     help_text = "Update working copy and build"
70     steps = [
71         steps.DiscardLocalChanges,
72         steps.Update,
73         steps.Build,
74     ]
75
76     def _prepare_state(self, options, args, tool):
77         options.build = True
78
79
80 class BuildAndTest(AbstractSequencedCommand):
81     name = "build-and-test"
82     help_text = "Update working copy, build, and run the tests"
83     steps = [
84         steps.DiscardLocalChanges,
85         steps.Update,
86         steps.Build,
87         steps.RunTests,
88     ]
89
90
91 class CheckPatchRelevance(AbstractSequencedCommand):
92     name = "check-patch-relevance"
93     help_text = "Check if this patch needs to be tested"
94     steps = [
95         steps.CheckPatchRelevance,
96     ]
97
98
99 class Land(AbstractSequencedCommand):
100     name = "land"
101     help_text = "Land the current working directory diff and updates the associated bug if any"
102     argument_names = "[BUGID]"
103     show_in_main_help = True
104     steps = [
105         steps.AddSvnMimetypeForPng,
106         steps.UpdateChangeLogsWithReviewer,
107         steps.ValidateReviewer,
108         steps.ValidateChangeLogs,  # We do this after UpdateChangeLogsWithReviewer to avoid not having to cache the diff twice.
109         steps.Build,
110         steps.RunTests,
111         steps.Commit,
112         steps.CloseBugForLandDiff,
113     ]
114     long_help = """land commits the current working copy diff (just as svn or git commit would).
115 land will NOT build and run the tests before committing, but you can use the --build option for that.
116 If a bug id is provided, or one can be found in the ChangeLog land will update the bug after committing."""
117
118     def _prepare_state(self, options, args, tool):
119         changed_files = self._tool.scm().changed_files(options.git_commit)
120         return {
121             "changed_files": changed_files,
122             "bug_id": (args and args[0]) or tool.checkout().bug_id_for_this_commit(options.git_commit, changed_files),
123         }
124
125
126 class LandCowhand(AbstractSequencedCommand):
127     # Gender-blind term for cowboy, see: http://en.wiktionary.org/wiki/cowhand
128     name = "land-cowhand"
129     help_text = "Prepares a ChangeLog and lands the current working directory diff."
130     steps = [
131         steps.SortXcodeProjectFiles,
132         steps.PrepareChangeLog,
133         steps.EditChangeLog,
134         steps.CheckStyle,
135         steps.ConfirmDiff,
136         steps.Build,
137         steps.RunTests,
138         steps.Commit,
139         steps.CloseBugForLandDiff,
140     ]
141
142     def _prepare_state(self, options, args, tool):
143         options.check_style_filter = "-changelog"
144
145
146 class LandCowboy(LandCowhand):
147     name = "land-cowboy"
148
149     def _prepare_state(self, options, args, tool):
150         _log.warning("land-cowboy is deprecated, use land-cowhand instead.")
151         LandCowhand._prepare_state(self, options, args, tool)
152
153
154 class CheckStyleLocal(AbstractSequencedCommand):
155     name = "check-style-local"
156     help_text = "Run check-webkit-style on the current working directory diff"
157     steps = [
158         steps.CheckStyle,
159     ]
160
161
162 class AbstractPatchProcessingCommand(Command):
163     # Subclasses must implement the methods below.  We don't declare them here
164     # because we want to be able to implement them with mix-ins.
165     #
166     # pylint: disable=E1101
167     # def _fetch_list_of_patches_to_process(self, options, args, tool):
168     # def _prepare_to_process(self, options, args, tool):
169     # def _process_patch(self, options, args, tool):
170
171     @staticmethod
172     def _collect_patches_by_bug(patches):
173         bugs_to_patches = {}
174         for patch in patches:
175             bugs_to_patches[patch.bug_id()] = bugs_to_patches.get(patch.bug_id(), []) + [patch]
176         return bugs_to_patches
177
178     def execute(self, options, args, tool):
179         self._prepare_to_process(options, args, tool)
180         patches = self._fetch_list_of_patches_to_process(options, args, tool)
181
182         # It's nice to print out total statistics.
183         bugs_to_patches = self._collect_patches_by_bug(patches)
184         _log.info("Processing %s from %s." % (pluralize(len(patches), "patch"), pluralize(len(bugs_to_patches), "bug")))
185
186         for patch in patches:
187             self._process_patch(patch, options, args, tool)
188
189
190 class AbstractPatchSequencingCommand(AbstractPatchProcessingCommand):
191     prepare_steps = None
192     main_steps = None
193
194     def __init__(self):
195         options = []
196         self._prepare_sequence = StepSequence(self.prepare_steps)
197         self._main_sequence = StepSequence(self.main_steps)
198         options = sorted(set(self._prepare_sequence.options() + self._main_sequence.options()))
199         AbstractPatchProcessingCommand.__init__(self, options)
200
201     def _prepare_to_process(self, options, args, tool):
202         try:
203             self.state = self._prepare_state(options, args, tool)
204         except ScriptError as e:
205             _log.error(e.message_with_output(output_limit=5000))
206             self._exit(e.exit_code or 2)
207         self._prepare_sequence.run_and_handle_errors(tool, options, self.state)
208
209     def _process_patch(self, patch, options, args, tool):
210         state = {}
211         state.update(self.state or {})
212         state["patch"] = patch
213         self._main_sequence.run_and_handle_errors(tool, options, state)
214
215     def _prepare_state(self, options, args, tool):
216         return None
217
218
219 class ProcessAttachmentsMixin(object):
220     def _fetch_list_of_patches_to_process(self, options, args, tool):
221         patches = []
222         for patch_id in args:
223             try:
224                 patch = tool.bugs.fetch_attachment(patch_id, throw_on_access_error=True)
225             except Bugzilla.AccessError as e:
226                 if e.error_code == Bugzilla.AccessError.NOT_PERMITTED:
227                     patch = tool.status_server.fetch_attachment(patch_id)
228             if patch:
229                 patches.append(patch)
230         return patches
231
232
233 class ProcessBugsMixin(object):
234     def _fetch_list_of_patches_to_process(self, options, args, tool):
235         all_patches = []
236         for bug_id in args:
237             bug = tool.bugs.fetch_bug(bug_id)
238             if not bug:
239                 continue
240             patches = bug.reviewed_patches()
241             _log.info("%s found on bug %s." % (pluralize(len(patches), "reviewed patch"), bug_id))
242             all_patches += patches
243         if not all_patches:
244             _log.info("No reviewed patches found, looking for unreviewed patches.")
245             for bug_id in args:
246                 bug = tool.bugs.fetch_bug(bug_id)
247                 if not bug:
248                     continue
249                 patches = bug.patches()
250                 _log.info("%s found on bug %s." % (pluralize(len(patches), "patch"), bug_id))
251                 all_patches += patches
252         return all_patches
253
254
255 class ProcessURLsMixin(object):
256     def _fetch_list_of_patches_to_process(self, options, args, tool):
257         all_patches = []
258         for url in args:
259             bug_id = urls.parse_bug_id(url)
260             if bug_id:
261                 patches = tool.bugs.fetch_bug(bug_id).patches()
262                 _log.info("%s found on bug %s." % (pluralize(len(patches), "patch"), bug_id))
263                 all_patches += patches
264
265             attachment_id = urls.parse_attachment_id(url)
266             if attachment_id:
267                 all_patches += tool.bugs.fetch_attachment(attachment_id)
268
269         return all_patches
270
271
272 class CheckStyle(AbstractPatchSequencingCommand, ProcessAttachmentsMixin):
273     name = "check-style"
274     help_text = "Run check-webkit-style on the specified attachments"
275     argument_names = "ATTACHMENT_ID [ATTACHMENT_IDS]"
276     main_steps = [
277         steps.DiscardLocalChanges,
278         steps.Update,
279         steps.ApplyPatch,
280         steps.CheckStyle,
281     ]
282
283
284 class BuildAttachment(AbstractPatchSequencingCommand, ProcessAttachmentsMixin):
285     name = "build-attachment"
286     help_text = "Apply and build patches from bugzilla"
287     argument_names = "ATTACHMENT_ID [ATTACHMENT_IDS]"
288     main_steps = [
289         steps.DiscardLocalChanges,
290         steps.Update,
291         steps.ApplyPatch,
292         steps.Build,
293     ]
294
295
296 class BuildAndTestAttachment(AbstractPatchSequencingCommand, ProcessAttachmentsMixin):
297     name = "build-and-test-attachment"
298     help_text = "Apply, build, and test patches from bugzilla"
299     argument_names = "ATTACHMENT_ID [ATTACHMENT_IDS]"
300     main_steps = [
301         steps.DiscardLocalChanges,
302         steps.Update,
303         steps.ApplyPatch,
304         steps.Build,
305         steps.RunTests,
306     ]
307
308
309 class AbstractPatchApplyingCommand(AbstractPatchSequencingCommand):
310     prepare_steps = [
311         steps.EnsureLocalCommitIfNeeded,
312         steps.CleanWorkingDirectory,
313         steps.Update,
314     ]
315     main_steps = [
316         steps.ApplyPatchWithLocalCommit,
317     ]
318     long_help = """Updates the working copy.
319 Downloads and applies the patches, creating local commits if necessary."""
320
321
322 class ApplyAttachment(AbstractPatchApplyingCommand, ProcessAttachmentsMixin):
323     name = "apply-attachment"
324     help_text = "Apply an attachment to the local working directory"
325     argument_names = "ATTACHMENT_ID [ATTACHMENT_IDS]"
326     show_in_main_help = True
327
328
329 class ApplyFromBug(AbstractPatchApplyingCommand, ProcessBugsMixin):
330     name = "apply-from-bug"
331     help_text = "Apply reviewed patches from provided bugs to the local working directory"
332     argument_names = "BUGID [BUGIDS]"
333     show_in_main_help = True
334
335
336 class ApplyWatchList(AbstractPatchSequencingCommand, ProcessAttachmentsMixin):
337     name = "apply-watchlist"
338     help_text = "Applies the watchlist to the specified attachments"
339     argument_names = "ATTACHMENT_ID [ATTACHMENT_IDS]"
340     main_steps = [
341         steps.DiscardLocalChanges,
342         steps.Update,
343         steps.ApplyPatch,
344         steps.ApplyWatchList,
345     ]
346     long_help = """"Applies the watchlist to the specified attachments.
347 Downloads the attachment, applies it locally, runs the watchlist against it, and updates the bug with the result."""
348
349
350 class AbstractPatchLandingCommand(AbstractPatchSequencingCommand):
351     main_steps = [
352         steps.DiscardLocalChanges,
353         steps.Update,
354         steps.ApplyPatch,
355         steps.ValidateChangeLogs,
356         steps.ValidateReviewer,
357         steps.Build,
358         steps.RunTests,
359         steps.Commit,
360         steps.ClosePatch,
361         steps.CloseBug,
362     ]
363     long_help = """Checks to make sure builders are green.
364 Updates the working copy.
365 Applies the patch.
366 Builds.
367 Runs the layout tests.
368 Commits the patch.
369 Clears the flags on the patch.
370 Closes the bug if no patches are marked for review."""
371
372
373 class LandAttachment(AbstractPatchLandingCommand, ProcessAttachmentsMixin):
374     name = "land-attachment"
375     help_text = "Land patches from bugzilla, optionally building and testing them first"
376     argument_names = "ATTACHMENT_ID [ATTACHMENT_IDS]"
377     show_in_main_help = True
378
379
380 class LandFromBug(AbstractPatchLandingCommand, ProcessBugsMixin):
381     name = "land-from-bug"
382     help_text = "Land all patches on the given bugs, optionally building and testing them first"
383     argument_names = "BUGID [BUGIDS]"
384     show_in_main_help = True
385
386
387 class LandFromURL(AbstractPatchLandingCommand, ProcessURLsMixin):
388     name = "land-from-url"
389     help_text = "Land all patches on the given URLs, optionally building and testing them first"
390     argument_names = "URL [URLS]"
391
392
393 class ValidateChangelog(AbstractSequencedCommand):
394     name = "validate-changelog"
395     help_text = "Validate that the ChangeLogs and reviewers look reasonable"
396     long_help = """Examines the current diff to see whether the ChangeLogs
397 and the reviewers listed in the ChangeLogs look reasonable.
398 """
399     steps = [
400         steps.ValidateChangeLogs,
401         steps.ValidateReviewer,
402     ]
403
404
405 class AbstractRolloutPrepCommand(AbstractSequencedCommand):
406     argument_names = "REVISION [REVISIONS] REASON"
407
408     def _commit_info(self, revision):
409         commit_info = self._tool.checkout().commit_info_for_revision(revision)
410         if commit_info and commit_info.bug_id():
411             # Note: Don't print a bug URL here because it will confuse the
412             #       SheriffBot because the SheriffBot just greps the output
413             #       of create-rollout for bug URLs.  It should do better
414             #       parsing instead.
415             _log.info("Preparing rollout for bug %s." % commit_info.bug_id())
416         else:
417             _log.info("Unable to parse bug number from diff.")
418         return commit_info
419
420     def _prepare_state(self, options, args, tool):
421         revision_list = []
422         description_list = []
423         bug_id_list = []
424         for revision in str(args[0]).split():
425             if revision.isdigit():
426                 revision_list.append(int(revision))
427             else:
428                 raise ScriptError(message="Invalid svn revision number: " + revision)
429         revision_list.sort()
430
431         earliest_revision = revision_list[0]
432         state = {
433             "revision": earliest_revision,
434             "revision_list": revision_list,
435             "reason": args[1],
436             "bug_id": None,
437             "bug_id_list": bug_id_list,
438             "description_list": description_list,
439         }
440         for revision in revision_list:
441             commit_info = self._commit_info(revision)
442             if commit_info:
443                 # We use the earliest revision for the bug info
444                 if revision == earliest_revision:
445                     state["bug_blocked"] = commit_info.bug_id()
446                     cc_list = sorted([party.bugzilla_email()
447                             for party in commit_info.responsible_parties()
448                             if party.bugzilla_email()])
449                     # FIXME: We should used the list as the canonical representation.
450                     state["bug_cc"] = ",".join(cc_list)
451                 description_list.append(commit_info.bug_description())
452                 bug_id_list.append(commit_info.bug_id())
453             else:
454                 description_list.append(None)
455                 bug_id_list.append(None)
456         return state
457
458
459 class PrepareRollout(AbstractRolloutPrepCommand):
460     name = "prepare-rollout"
461     help_text = "Revert the given revision(s) in the working copy and prepare ChangeLogs with revert reason"
462     long_help = """Updates the working copy.
463 Applies the inverse diff for the provided revision(s).
464 Creates an appropriate rollout ChangeLog, including a trac link and bug link.
465 """
466     steps = [
467         steps.DiscardLocalChanges,
468         steps.Update,
469         steps.RevertRevision,
470         steps.PrepareChangeLogForRevert,
471     ]
472
473
474 class CreateRollout(AbstractRolloutPrepCommand):
475     name = "create-rollout"
476     help_text = "Creates a bug to track the broken SVN revision(s) and uploads a rollout patch."
477     steps = [
478         steps.DiscardLocalChanges,
479         steps.Update,
480         steps.RevertRevision,
481         steps.CreateBug,
482         steps.PrepareChangeLogForRevert,
483         steps.PostDiffForRevert,
484     ]
485
486     def _prepare_state(self, options, args, tool):
487         state = AbstractRolloutPrepCommand._prepare_state(self, options, args, tool)
488         state["bug_title"] = "REGRESSION(r%s): %s" % (state["revision"], state["reason"])
489         state["bug_description"] = "%s broke the build:\n%s" % (urls.view_revision_url(state["revision"]), state["reason"])
490         # FIXME: If we had more context here, we could link to other open bugs
491         #        that mention the test that regressed.
492         if options.parent_command == "sheriff-bot":
493             state["bug_description"] += """
494
495 This is an automatic bug report generated by webkitbot. If this bug
496 report was created because of a flaky test, please file a bug for the flaky
497 test (if we don't already have one on file) and dup this bug against that bug
498 so that we can track how often these flaky tests fail.
499 """
500         return state
501
502
503 class Rollout(AbstractRolloutPrepCommand):
504     name = "rollout"
505     show_in_main_help = True
506     help_text = "Revert the given revision(s) in the working copy and optionally commit the revert and re-open the original bug"
507     long_help = """Updates the working copy.
508 Applies the inverse diff for the provided revision.
509 Creates an appropriate rollout ChangeLog, including a trac link and bug link.
510 Opens the generated ChangeLogs in $EDITOR.
511 Shows the prepared diff for confirmation.
512 Commits the revert and updates the bug (including re-opening the bug if necessary)."""
513     steps = [
514         steps.DiscardLocalChanges,
515         steps.Update,
516         steps.RevertRevision,
517         steps.PrepareChangeLogForRevert,
518         steps.EditChangeLog,
519         steps.ConfirmDiff,
520         steps.Build,
521         steps.Commit,
522         steps.ReopenBugAfterRollout,
523     ]