2010-09-30 Adam Barth <abarth@webkit.org>
[WebKit-https.git] / WebKitTools / Scripts / webkitpy / tool / commands / queries.py
1 # Copyright (c) 2009 Google Inc. All rights reserved.
2 # Copyright (c) 2009 Apple Inc. All rights reserved.
3 #
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
6 # met:
7
8 #     * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 #     * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
13 # distribution.
14 #     * Neither the name of Google Inc. nor the names of its
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
17
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30
31 from optparse import make_option
32
33 from webkitpy.common.checkout.commitinfo import CommitInfo
34 from webkitpy.common.config.committers import CommitterList
35 from webkitpy.common.net.buildbot import BuildBot
36 from webkitpy.common.net.regressionwindow import RegressionWindow
37 from webkitpy.common.system.user import User
38 from webkitpy.tool.grammar import pluralize
39 from webkitpy.tool.multicommandtool import AbstractDeclarativeCommand
40 from webkitpy.common.system.deprecated_logging import log
41 from webkitpy.layout_tests import port
42
43
44 class BugsToCommit(AbstractDeclarativeCommand):
45     name = "bugs-to-commit"
46     help_text = "List bugs in the commit-queue"
47
48     def execute(self, options, args, tool):
49         # FIXME: This command is poorly named.  It's fetching the commit-queue list here.  The name implies it's fetching pending-commit (all r+'d patches).
50         bug_ids = tool.bugs.queries.fetch_bug_ids_from_commit_queue()
51         for bug_id in bug_ids:
52             print "%s" % bug_id
53
54
55 class PatchesInCommitQueue(AbstractDeclarativeCommand):
56     name = "patches-in-commit-queue"
57     help_text = "List patches in the commit-queue"
58
59     def execute(self, options, args, tool):
60         patches = tool.bugs.queries.fetch_patches_from_commit_queue()
61         log("Patches in commit queue:")
62         for patch in patches:
63             print patch.url()
64
65
66 class PatchesToCommitQueue(AbstractDeclarativeCommand):
67     name = "patches-to-commit-queue"
68     help_text = "Patches which should be added to the commit queue"
69     def __init__(self):
70         options = [
71             make_option("--bugs", action="store_true", dest="bugs", help="Output bug links instead of patch links"),
72         ]
73         AbstractDeclarativeCommand.__init__(self, options=options)
74
75     @staticmethod
76     def _needs_commit_queue(patch):
77         if patch.commit_queue() == "+": # If it's already cq+, ignore the patch.
78             log("%s already has cq=%s" % (patch.id(), patch.commit_queue()))
79             return False
80
81         # We only need to worry about patches from contributers who are not yet committers.
82         committer_record = CommitterList().committer_by_email(patch.attacher_email())
83         if committer_record:
84             log("%s committer = %s" % (patch.id(), committer_record))
85         return not committer_record
86
87     def execute(self, options, args, tool):
88         patches = tool.bugs.queries.fetch_patches_from_pending_commit_list()
89         patches_needing_cq = filter(self._needs_commit_queue, patches)
90         if options.bugs:
91             bugs_needing_cq = map(lambda patch: patch.bug_id(), patches_needing_cq)
92             bugs_needing_cq = sorted(set(bugs_needing_cq))
93             for bug_id in bugs_needing_cq:
94                 print "%s" % tool.bugs.bug_url_for_bug_id(bug_id)
95         else:
96             for patch in patches_needing_cq:
97                 print "%s" % tool.bugs.attachment_url_for_id(patch.id(), action="edit")
98
99
100 class PatchesToReview(AbstractDeclarativeCommand):
101     name = "patches-to-review"
102     help_text = "List patches that are pending review"
103
104     def execute(self, options, args, tool):
105         patch_ids = tool.bugs.queries.fetch_attachment_ids_from_review_queue()
106         log("Patches pending review:")
107         for patch_id in patch_ids:
108             print patch_id
109
110
111 class LastGreenRevision(AbstractDeclarativeCommand):
112     name = "last-green-revision"
113     help_text = "Prints the last known good revision"
114
115     def execute(self, options, args, tool):
116         print self._tool.buildbot.last_green_revision()
117
118
119 class WhatBroke(AbstractDeclarativeCommand):
120     name = "what-broke"
121     help_text = "Print failing buildbots (%s) and what revisions broke them" % BuildBot.default_host
122
123     def _print_builder_line(self, builder_name, max_name_width, status_message):
124         print "%s : %s" % (builder_name.ljust(max_name_width), status_message)
125
126     def _print_blame_information_for_builder(self, builder_status, name_width, avoid_flakey_tests=True):
127         builder = self._tool.buildbot.builder_with_name(builder_status["name"])
128         red_build = builder.build(builder_status["build_number"])
129         regression_window = builder.find_regression_window(red_build)
130         if not regression_window.failing_build():
131             self._print_builder_line(builder.name(), name_width, "FAIL (error loading build information)")
132             return
133         if not regression_window.build_before_failure():
134             self._print_builder_line(builder.name(), name_width, "FAIL (blame-list: sometime before %s?)" % regression_window.failing_build().revision())
135             return
136
137         revisions = regression_window.revisions()
138         first_failure_message = ""
139         if (regression_window.failing_build() == builder.build(builder_status["build_number"])):
140             first_failure_message = " FIRST FAILURE, possibly a flaky test"
141         self._print_builder_line(builder.name(), name_width, "FAIL (blame-list: %s%s)" % (revisions, first_failure_message))
142         for revision in revisions:
143             commit_info = self._tool.checkout().commit_info_for_revision(revision)
144             if commit_info:
145                 print commit_info.blame_string(self._tool.bugs)
146             else:
147                 print "FAILED to fetch CommitInfo for r%s, likely missing ChangeLog" % revision
148
149     def execute(self, options, args, tool):
150         builder_statuses = tool.buildbot.builder_statuses()
151         longest_builder_name = max(map(len, map(lambda builder: builder["name"], builder_statuses)))
152         failing_builders = 0
153         for builder_status in builder_statuses:
154             # If the builder is green, print OK, exit.
155             if builder_status["is_green"]:
156                 continue
157             self._print_blame_information_for_builder(builder_status, name_width=longest_builder_name)
158             failing_builders += 1
159         if failing_builders:
160             print "%s of %s are failing" % (failing_builders, pluralize("builder", len(builder_statuses)))
161         else:
162             print "All builders are passing!"
163
164
165 class ResultsFor(AbstractDeclarativeCommand):
166     name = "results-for"
167     help_text = "Print a list of failures for the passed revision from bots on %s" % BuildBot.default_host
168     argument_names = "REVISION"
169
170     def _print_layout_test_results(self, results):
171         if not results:
172             print " No results."
173             return
174         for title, files in results.parsed_results().items():
175             print " %s" % title
176             for filename in files:
177                 print "  %s" % filename
178
179     def execute(self, options, args, tool):
180         builders = self._tool.buildbot.builders()
181         for builder in builders:
182             print "%s:" % builder.name()
183             build = builder.build_for_revision(args[0], allow_failed_lookups=True)
184             self._print_layout_test_results(build.layout_test_results())
185
186
187 class FailureReason(AbstractDeclarativeCommand):
188     name = "failure-reason"
189     help_text = "Lists revisions where individual test failures started at %s" % BuildBot.default_host
190
191     def _blame_line_for_revision(self, revision):
192         try:
193             commit_info = self._tool.checkout().commit_info_for_revision(revision)
194         except Exception, e:
195             return "FAILED to fetch CommitInfo for r%s, exception: %s" % (revision, e)
196         if not commit_info:
197             return "FAILED to fetch CommitInfo for r%s, likely missing ChangeLog" % revision
198         return commit_info.blame_string(self._tool.bugs)
199
200     def _print_blame_information_for_transition(self, regression_window, failing_tests):
201         red_build = regression_window.failing_build()
202         print "SUCCESS: Build %s (r%s) was the first to show failures: %s" % (red_build._number, red_build.revision(), failing_tests)
203         print "Suspect revisions:"
204         for revision in regression_window.revisions():
205             print self._blame_line_for_revision(revision)
206
207     def _explain_failures_for_builder(self, builder, start_revision):
208         print "Examining failures for \"%s\", starting at r%s" % (builder.name(), start_revision)
209         revision_to_test = start_revision
210         build = builder.build_for_revision(revision_to_test, allow_failed_lookups=True)
211         layout_test_results = build.layout_test_results()
212         if not layout_test_results:
213             # FIXME: This could be made more user friendly.
214             print "Failed to load layout test results; can't continue. (start revision = r%s)" % start_revision
215             return 1
216
217         results_to_explain = set(layout_test_results.failing_tests())
218         last_build_with_results = build
219         print "Starting at %s" % revision_to_test
220         while results_to_explain:
221             revision_to_test -= 1
222             new_build = builder.build_for_revision(revision_to_test, allow_failed_lookups=True)
223             if not new_build:
224                 print "No build for %s" % revision_to_test
225                 continue
226             build = new_build
227             latest_results = build.layout_test_results()
228             if not latest_results:
229                 print "No results build %s (r%s)" % (build._number, build.revision())
230                 continue
231             failures = set(latest_results.failing_tests())
232             if len(failures) >= 20:
233                 # FIXME: We may need to move this logic into the LayoutTestResults class.
234                 # The buildbot stops runs after 20 failures so we don't have full results to work with here.
235                 print "Too many failures in build %s (r%s), ignoring." % (build._number, build.revision())
236                 continue
237             fixed_results = results_to_explain - failures
238             if not fixed_results:
239                 print "No change in build %s (r%s), %s unexplained failures (%s in this build)" % (build._number, build.revision(), len(results_to_explain), len(failures))
240                 last_build_with_results = build
241                 continue
242             regression_window = RegressionWindow(build, last_build_with_results)
243             self._print_blame_information_for_transition(regression_window, fixed_results)
244             last_build_with_results = build
245             results_to_explain -= fixed_results
246         if results_to_explain:
247             print "Failed to explain failures: %s" % results_to_explain
248             return 1
249         print "Explained all results for %s" % builder.name()
250         return 0
251
252     def _builder_to_explain(self):
253         builder_statuses = self._tool.buildbot.builder_statuses()
254         red_statuses = [status for status in builder_statuses if not status["is_green"]]
255         print "%s failing" % (pluralize("builder", len(red_statuses)))
256         builder_choices = [status["name"] for status in red_statuses]
257         # We could offer an "All" choice here.
258         chosen_name = User.prompt_with_list("Which builder to diagnose:", builder_choices)
259         # FIXME: prompt_with_list should really take a set of objects and a set of names and then return the object.
260         for status in red_statuses:
261             if status["name"] == chosen_name:
262                 return (self._tool.buildbot.builder_with_name(chosen_name), status["built_revision"])
263
264     def execute(self, options, args, tool):
265         (builder, latest_revision) = self._builder_to_explain()
266         start_revision = self._tool.user.prompt("Revision to walk backwards from? [%s] " % latest_revision) or latest_revision
267         if not start_revision:
268             print "Revision required."
269             return 1
270         return self._explain_failures_for_builder(builder, start_revision=int(start_revision))
271
272
273 class FindFlakyTests(AbstractDeclarativeCommand):
274     name = "find-flaky-tests"
275     help_text = "Lists tests that often fail for a single build at %s" % BuildBot.default_host
276
277     def _find_failures(self, builder, revision):
278         build = builder.build_for_revision(revision, allow_failed_lookups=True)
279         if not build:
280             print "No build for %s" % revision
281             return (None, None)
282         results = build.layout_test_results()
283         if not results:
284             print "No results build %s (r%s)" % (build._number, build.revision())
285             return (None, None)
286         failures = set(results.failing_tests())
287         if len(failures) >= 20:
288             # FIXME: We may need to move this logic into the LayoutTestResults class.
289             # The buildbot stops runs after 20 failures so we don't have full results to work with here.
290             print "Too many failures in build %s (r%s), ignoring." % (build._number, build.revision())
291             return (None, None)
292         return (build, failures)
293
294     def _increment_statistics(self, flaky_tests, flaky_test_statistics):
295         for test in flaky_tests:
296             count = flaky_test_statistics.get(test, 0)
297             flaky_test_statistics[test] = count + 1
298
299     def _print_statistics(self, statistics):
300         print "=== Results ==="
301         print "Occurances Test name"
302         for value, key in sorted([(value, key) for key, value in statistics.items()]):
303             print "%10d %s" % (value, key)
304
305     def _walk_backwards_from(self, builder, start_revision, limit):
306         flaky_test_statistics = {}
307         all_previous_failures = set([])
308         one_time_previous_failures = set([])
309         previous_build = None
310         for i in range(limit):
311             revision = start_revision - i
312             print "Analyzing %s ... " % revision,
313             (build, failures) = self._find_failures(builder, revision)
314             if failures == None:
315                 # Notice that we don't loop on the empty set!
316                 continue
317             print "has %s failures" % len(failures)
318             flaky_tests = one_time_previous_failures - failures
319             if flaky_tests:
320                 print "Flaky tests: %s %s" % (sorted(flaky_tests),
321                                               previous_build.results_url())
322             self._increment_statistics(flaky_tests, flaky_test_statistics)
323             one_time_previous_failures = failures - all_previous_failures
324             all_previous_failures = failures
325             previous_build = build
326         self._print_statistics(flaky_test_statistics)
327
328     def _builder_to_analyze(self):
329         statuses = self._tool.buildbot.builder_statuses()
330         choices = [status["name"] for status in statuses]
331         chosen_name = User.prompt_with_list("Which builder to analyze:", choices)
332         for status in statuses:
333             if status["name"] == chosen_name:
334                 return (self._tool.buildbot.builder_with_name(chosen_name), status["built_revision"])
335
336     def execute(self, options, args, tool):
337         (builder, latest_revision) = self._builder_to_analyze()
338         limit = self._tool.user.prompt("How many revisions to look through? [10000] ") or 10000
339         return self._walk_backwards_from(builder, latest_revision, limit=int(limit))
340
341
342 class TreeStatus(AbstractDeclarativeCommand):
343     name = "tree-status"
344     help_text = "Print the status of the %s buildbots" % BuildBot.default_host
345     long_help = """Fetches build status from http://build.webkit.org/one_box_per_builder
346 and displayes the status of each builder."""
347
348     def execute(self, options, args, tool):
349         for builder in tool.buildbot.builder_statuses():
350             status_string = "ok" if builder["is_green"] else "FAIL"
351             print "%s : %s" % (status_string.ljust(4), builder["name"])
352
353
354 class SkippedPorts(AbstractDeclarativeCommand):
355     name = "skipped-ports"
356     help_text = "Print the list of ports skipping the given layout test(s)"
357     long_help = """Scans the the Skipped file of each port and figure
358 out what ports are skipping the test(s). Categories are taken in account too."""
359     argument_names = "TEST_NAME"
360
361     def execute(self, options, args, tool):
362         class Options:
363             # Required for chromium port.
364             use_drt = True
365
366         results = dict([(test_name, []) for test_name in args])
367         for port_name, port_object in tool.port_factory.get_all(options=Options).iteritems():
368             for test_name in args:
369                 if port_object.skips_layout_test(test_name):
370                     results[test_name].append(port_name)
371
372         for test_name, ports in results.iteritems():
373             if ports:
374                 print "Ports skipping test %r: %s" % (test_name, ', '.join(ports))
375             else:
376                 print "Test %r is not skipped by any port." % test_name