Rename WebKitTools to Tools
[WebKit-https.git] / Tools / Scripts / webkitpy / tool / commands / queries.py
1 # Copyright (c) 2009 Google Inc. All rights reserved.
2 # Copyright (c) 2009 Apple Inc. All rights reserved.
3 #
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
6 # met:
7
8 #     * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 #     * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
13 # distribution.
14 #     * Neither the name of Google Inc. nor the names of its
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
17
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30
31 from optparse import make_option
32
33 import webkitpy.tool.steps as steps
34
35 from webkitpy.common.checkout.commitinfo import CommitInfo
36 from webkitpy.common.config.committers import CommitterList
37 from webkitpy.common.net.buildbot import BuildBot
38 from webkitpy.common.net.regressionwindow import RegressionWindow
39 from webkitpy.common.system.user import User
40 from webkitpy.tool.grammar import pluralize
41 from webkitpy.tool.multicommandtool import AbstractDeclarativeCommand
42 from webkitpy.common.system.deprecated_logging import log
43 from webkitpy.layout_tests import port
44
45
46 class SuggestReviewers(AbstractDeclarativeCommand):
47     name = "suggest-reviewers"
48     help_text = "Suggest reviewers for a patch based on recent changes to the modified files."
49
50     def __init__(self):
51         options = [
52             steps.Options.git_commit,
53         ]
54         AbstractDeclarativeCommand.__init__(self, options=options)
55
56     def execute(self, options, args, tool):
57         reviewers = tool.checkout().suggested_reviewers(options.git_commit)
58         print "\n".join([reviewer.full_name for reviewer in reviewers])
59
60
61 class BugsToCommit(AbstractDeclarativeCommand):
62     name = "bugs-to-commit"
63     help_text = "List bugs in the commit-queue"
64
65     def execute(self, options, args, tool):
66         # FIXME: This command is poorly named.  It's fetching the commit-queue list here.  The name implies it's fetching pending-commit (all r+'d patches).
67         bug_ids = tool.bugs.queries.fetch_bug_ids_from_commit_queue()
68         for bug_id in bug_ids:
69             print "%s" % bug_id
70
71
72 class PatchesInCommitQueue(AbstractDeclarativeCommand):
73     name = "patches-in-commit-queue"
74     help_text = "List patches in the commit-queue"
75
76     def execute(self, options, args, tool):
77         patches = tool.bugs.queries.fetch_patches_from_commit_queue()
78         log("Patches in commit queue:")
79         for patch in patches:
80             print patch.url()
81
82
83 class PatchesToCommitQueue(AbstractDeclarativeCommand):
84     name = "patches-to-commit-queue"
85     help_text = "Patches which should be added to the commit queue"
86     def __init__(self):
87         options = [
88             make_option("--bugs", action="store_true", dest="bugs", help="Output bug links instead of patch links"),
89         ]
90         AbstractDeclarativeCommand.__init__(self, options=options)
91
92     @staticmethod
93     def _needs_commit_queue(patch):
94         if patch.commit_queue() == "+": # If it's already cq+, ignore the patch.
95             log("%s already has cq=%s" % (patch.id(), patch.commit_queue()))
96             return False
97
98         # We only need to worry about patches from contributers who are not yet committers.
99         committer_record = CommitterList().committer_by_email(patch.attacher_email())
100         if committer_record:
101             log("%s committer = %s" % (patch.id(), committer_record))
102         return not committer_record
103
104     def execute(self, options, args, tool):
105         patches = tool.bugs.queries.fetch_patches_from_pending_commit_list()
106         patches_needing_cq = filter(self._needs_commit_queue, patches)
107         if options.bugs:
108             bugs_needing_cq = map(lambda patch: patch.bug_id(), patches_needing_cq)
109             bugs_needing_cq = sorted(set(bugs_needing_cq))
110             for bug_id in bugs_needing_cq:
111                 print "%s" % tool.bugs.bug_url_for_bug_id(bug_id)
112         else:
113             for patch in patches_needing_cq:
114                 print "%s" % tool.bugs.attachment_url_for_id(patch.id(), action="edit")
115
116
117 class PatchesToReview(AbstractDeclarativeCommand):
118     name = "patches-to-review"
119     help_text = "List patches that are pending review"
120
121     def execute(self, options, args, tool):
122         patch_ids = tool.bugs.queries.fetch_attachment_ids_from_review_queue()
123         log("Patches pending review:")
124         for patch_id in patch_ids:
125             print patch_id
126
127
128 class LastGreenRevision(AbstractDeclarativeCommand):
129     name = "last-green-revision"
130     help_text = "Prints the last known good revision"
131
132     def execute(self, options, args, tool):
133         print self._tool.buildbot.last_green_revision()
134
135
136 class WhatBroke(AbstractDeclarativeCommand):
137     name = "what-broke"
138     help_text = "Print failing buildbots (%s) and what revisions broke them" % BuildBot.default_host
139
140     def _print_builder_line(self, builder_name, max_name_width, status_message):
141         print "%s : %s" % (builder_name.ljust(max_name_width), status_message)
142
143     def _print_blame_information_for_builder(self, builder_status, name_width, avoid_flakey_tests=True):
144         builder = self._tool.buildbot.builder_with_name(builder_status["name"])
145         red_build = builder.build(builder_status["build_number"])
146         regression_window = builder.find_regression_window(red_build)
147         if not regression_window.failing_build():
148             self._print_builder_line(builder.name(), name_width, "FAIL (error loading build information)")
149             return
150         if not regression_window.build_before_failure():
151             self._print_builder_line(builder.name(), name_width, "FAIL (blame-list: sometime before %s?)" % regression_window.failing_build().revision())
152             return
153
154         revisions = regression_window.revisions()
155         first_failure_message = ""
156         if (regression_window.failing_build() == builder.build(builder_status["build_number"])):
157             first_failure_message = " FIRST FAILURE, possibly a flaky test"
158         self._print_builder_line(builder.name(), name_width, "FAIL (blame-list: %s%s)" % (revisions, first_failure_message))
159         for revision in revisions:
160             commit_info = self._tool.checkout().commit_info_for_revision(revision)
161             if commit_info:
162                 print commit_info.blame_string(self._tool.bugs)
163             else:
164                 print "FAILED to fetch CommitInfo for r%s, likely missing ChangeLog" % revision
165
166     def execute(self, options, args, tool):
167         builder_statuses = tool.buildbot.builder_statuses()
168         longest_builder_name = max(map(len, map(lambda builder: builder["name"], builder_statuses)))
169         failing_builders = 0
170         for builder_status in builder_statuses:
171             # If the builder is green, print OK, exit.
172             if builder_status["is_green"]:
173                 continue
174             self._print_blame_information_for_builder(builder_status, name_width=longest_builder_name)
175             failing_builders += 1
176         if failing_builders:
177             print "%s of %s are failing" % (failing_builders, pluralize("builder", len(builder_statuses)))
178         else:
179             print "All builders are passing!"
180
181
182 class ResultsFor(AbstractDeclarativeCommand):
183     name = "results-for"
184     help_text = "Print a list of failures for the passed revision from bots on %s" % BuildBot.default_host
185     argument_names = "REVISION"
186
187     def _print_layout_test_results(self, results):
188         if not results:
189             print " No results."
190             return
191         for title, files in results.parsed_results().items():
192             print " %s" % title
193             for filename in files:
194                 print "  %s" % filename
195
196     def execute(self, options, args, tool):
197         builders = self._tool.buildbot.builders()
198         for builder in builders:
199             print "%s:" % builder.name()
200             build = builder.build_for_revision(args[0], allow_failed_lookups=True)
201             self._print_layout_test_results(build.layout_test_results())
202
203
204 class FailureReason(AbstractDeclarativeCommand):
205     name = "failure-reason"
206     help_text = "Lists revisions where individual test failures started at %s" % BuildBot.default_host
207
208     def _blame_line_for_revision(self, revision):
209         try:
210             commit_info = self._tool.checkout().commit_info_for_revision(revision)
211         except Exception, e:
212             return "FAILED to fetch CommitInfo for r%s, exception: %s" % (revision, e)
213         if not commit_info:
214             return "FAILED to fetch CommitInfo for r%s, likely missing ChangeLog" % revision
215         return commit_info.blame_string(self._tool.bugs)
216
217     def _print_blame_information_for_transition(self, regression_window, failing_tests):
218         red_build = regression_window.failing_build()
219         print "SUCCESS: Build %s (r%s) was the first to show failures: %s" % (red_build._number, red_build.revision(), failing_tests)
220         print "Suspect revisions:"
221         for revision in regression_window.revisions():
222             print self._blame_line_for_revision(revision)
223
224     def _explain_failures_for_builder(self, builder, start_revision):
225         print "Examining failures for \"%s\", starting at r%s" % (builder.name(), start_revision)
226         revision_to_test = start_revision
227         build = builder.build_for_revision(revision_to_test, allow_failed_lookups=True)
228         layout_test_results = build.layout_test_results()
229         if not layout_test_results:
230             # FIXME: This could be made more user friendly.
231             print "Failed to load layout test results; can't continue. (start revision = r%s)" % start_revision
232             return 1
233
234         results_to_explain = set(layout_test_results.failing_tests())
235         last_build_with_results = build
236         print "Starting at %s" % revision_to_test
237         while results_to_explain:
238             revision_to_test -= 1
239             new_build = builder.build_for_revision(revision_to_test, allow_failed_lookups=True)
240             if not new_build:
241                 print "No build for %s" % revision_to_test
242                 continue
243             build = new_build
244             latest_results = build.layout_test_results()
245             if not latest_results:
246                 print "No results build %s (r%s)" % (build._number, build.revision())
247                 continue
248             failures = set(latest_results.failing_tests())
249             if len(failures) >= 20:
250                 # FIXME: We may need to move this logic into the LayoutTestResults class.
251                 # The buildbot stops runs after 20 failures so we don't have full results to work with here.
252                 print "Too many failures in build %s (r%s), ignoring." % (build._number, build.revision())
253                 continue
254             fixed_results = results_to_explain - failures
255             if not fixed_results:
256                 print "No change in build %s (r%s), %s unexplained failures (%s in this build)" % (build._number, build.revision(), len(results_to_explain), len(failures))
257                 last_build_with_results = build
258                 continue
259             regression_window = RegressionWindow(build, last_build_with_results)
260             self._print_blame_information_for_transition(regression_window, fixed_results)
261             last_build_with_results = build
262             results_to_explain -= fixed_results
263         if results_to_explain:
264             print "Failed to explain failures: %s" % results_to_explain
265             return 1
266         print "Explained all results for %s" % builder.name()
267         return 0
268
269     def _builder_to_explain(self):
270         builder_statuses = self._tool.buildbot.builder_statuses()
271         red_statuses = [status for status in builder_statuses if not status["is_green"]]
272         print "%s failing" % (pluralize("builder", len(red_statuses)))
273         builder_choices = [status["name"] for status in red_statuses]
274         # We could offer an "All" choice here.
275         chosen_name = User.prompt_with_list("Which builder to diagnose:", builder_choices)
276         # FIXME: prompt_with_list should really take a set of objects and a set of names and then return the object.
277         for status in red_statuses:
278             if status["name"] == chosen_name:
279                 return (self._tool.buildbot.builder_with_name(chosen_name), status["built_revision"])
280
281     def execute(self, options, args, tool):
282         (builder, latest_revision) = self._builder_to_explain()
283         start_revision = self._tool.user.prompt("Revision to walk backwards from? [%s] " % latest_revision) or latest_revision
284         if not start_revision:
285             print "Revision required."
286             return 1
287         return self._explain_failures_for_builder(builder, start_revision=int(start_revision))
288
289
290 class FindFlakyTests(AbstractDeclarativeCommand):
291     name = "find-flaky-tests"
292     help_text = "Lists tests that often fail for a single build at %s" % BuildBot.default_host
293
294     def _find_failures(self, builder, revision):
295         build = builder.build_for_revision(revision, allow_failed_lookups=True)
296         if not build:
297             print "No build for %s" % revision
298             return (None, None)
299         results = build.layout_test_results()
300         if not results:
301             print "No results build %s (r%s)" % (build._number, build.revision())
302             return (None, None)
303         failures = set(results.failing_tests())
304         if len(failures) >= 20:
305             # FIXME: We may need to move this logic into the LayoutTestResults class.
306             # The buildbot stops runs after 20 failures so we don't have full results to work with here.
307             print "Too many failures in build %s (r%s), ignoring." % (build._number, build.revision())
308             return (None, None)
309         return (build, failures)
310
311     def _increment_statistics(self, flaky_tests, flaky_test_statistics):
312         for test in flaky_tests:
313             count = flaky_test_statistics.get(test, 0)
314             flaky_test_statistics[test] = count + 1
315
316     def _print_statistics(self, statistics):
317         print "=== Results ==="
318         print "Occurances Test name"
319         for value, key in sorted([(value, key) for key, value in statistics.items()]):
320             print "%10d %s" % (value, key)
321
322     def _walk_backwards_from(self, builder, start_revision, limit):
323         flaky_test_statistics = {}
324         all_previous_failures = set([])
325         one_time_previous_failures = set([])
326         previous_build = None
327         for i in range(limit):
328             revision = start_revision - i
329             print "Analyzing %s ... " % revision,
330             (build, failures) = self._find_failures(builder, revision)
331             if failures == None:
332                 # Notice that we don't loop on the empty set!
333                 continue
334             print "has %s failures" % len(failures)
335             flaky_tests = one_time_previous_failures - failures
336             if flaky_tests:
337                 print "Flaky tests: %s %s" % (sorted(flaky_tests),
338                                               previous_build.results_url())
339             self._increment_statistics(flaky_tests, flaky_test_statistics)
340             one_time_previous_failures = failures - all_previous_failures
341             all_previous_failures = failures
342             previous_build = build
343         self._print_statistics(flaky_test_statistics)
344
345     def _builder_to_analyze(self):
346         statuses = self._tool.buildbot.builder_statuses()
347         choices = [status["name"] for status in statuses]
348         chosen_name = User.prompt_with_list("Which builder to analyze:", choices)
349         for status in statuses:
350             if status["name"] == chosen_name:
351                 return (self._tool.buildbot.builder_with_name(chosen_name), status["built_revision"])
352
353     def execute(self, options, args, tool):
354         (builder, latest_revision) = self._builder_to_analyze()
355         limit = self._tool.user.prompt("How many revisions to look through? [10000] ") or 10000
356         return self._walk_backwards_from(builder, latest_revision, limit=int(limit))
357
358
359 class TreeStatus(AbstractDeclarativeCommand):
360     name = "tree-status"
361     help_text = "Print the status of the %s buildbots" % BuildBot.default_host
362     long_help = """Fetches build status from http://build.webkit.org/one_box_per_builder
363 and displayes the status of each builder."""
364
365     def execute(self, options, args, tool):
366         for builder in tool.buildbot.builder_statuses():
367             status_string = "ok" if builder["is_green"] else "FAIL"
368             print "%s : %s" % (status_string.ljust(4), builder["name"])
369
370
371 class SkippedPorts(AbstractDeclarativeCommand):
372     name = "skipped-ports"
373     help_text = "Print the list of ports skipping the given layout test(s)"
374     long_help = """Scans the the Skipped file of each port and figure
375 out what ports are skipping the test(s). Categories are taken in account too."""
376     argument_names = "TEST_NAME"
377
378     def execute(self, options, args, tool):
379         results = dict([(test_name, []) for test_name in args])
380         for port_name, port_object in tool.port_factory.get_all().iteritems():
381             for test_name in args:
382                 if port_object.skips_layout_test(test_name):
383                     results[test_name].append(port_name)
384
385         for test_name, ports in results.iteritems():
386             if ports:
387                 print "Ports skipping test %r: %s" % (test_name, ', '.join(ports))
388             else:
389                 print "Test %r is not skipped by any port." % test_name