e5c58bf8b8d5ba4659735ae9681c9aef7bb96c26
[WebKit-https.git] / Tools / Scripts / webkitpy / tool / commands / rebaseline.py
1 # Copyright (c) 2010 Google Inc. All rights reserved.
2 #
3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions are
5 # met:
6 #
7 #     * Redistributions of source code must retain the above copyright
8 # notice, this list of conditions and the following disclaimer.
9 #     * Redistributions in binary form must reproduce the above
10 # copyright notice, this list of conditions and the following disclaimer
11 # in the documentation and/or other materials provided with the
12 # distribution.
13 #     * Neither the name of Google Inc. nor the names of its
14 # contributors may be used to endorse or promote products derived from
15 # this software without specific prior written permission.
16 #
17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29 import os.path
30 import re
31 import shutil
32 import urllib
33
34 import webkitpy.common.config.urls as config_urls
35 from webkitpy.common.net.buildbot import BuildBot
36 from webkitpy.common.net.buildbot.chromiumbuildbot import ChromiumBuildBot
37 from webkitpy.common.net.layouttestresults import LayoutTestResults
38 from webkitpy.common.system.user import User
39 from webkitpy.layout_tests.models import test_failures
40 from webkitpy.layout_tests.port import factory
41 from webkitpy.tool.grammar import pluralize
42 from webkitpy.tool.multicommandtool import AbstractDeclarativeCommand
43
44
45 # FIXME: This logic should be moved to builders.py.
46 class BuilderToPort(object):
47     _builder_name_to_port_name = {
48         # These builders are on build.webkit.org.
49         r"SnowLeopard": "mac-snowleopard",
50         r"Leopard": "mac-leopard",
51         r"Windows": "win",
52         r"GTK": "gtk",
53         r"Qt": "qt",
54         r"Chromium Mac": "chromium-mac",
55         r"Chromium Linux": "chromium-linux",
56         r"Chromium Win": "chromium-win",
57
58         # These builders are on build.chromium.org.
59         r"Webkit Win": "chromium-win-xp",
60         r"Webkit Vista": "chromium-win-vista",
61         r"Webkit Win7": "chromium-win-win7",
62         r"Webkit Win (dbg)(1)": "chromium-win-win7",  # FIXME: Is this correct?
63         r"Webkit Win (dbg)(2)": "chromium-win-win7",  # FIXME: Is this correct?
64         r"Webkit Linux": "chromium-linux-x86_64",
65         r"Webkit Linux 32": "chromium-linux-x86",
66         r"Webkit Linux (dbg)(1)": "chromium-linux-x86_64",
67         r"Webkit Linux (dbg)(2)": "chromium-linux-x86_64",
68         r"Webkit Mac10\.5": "chromium-mac-leopard",
69         r"Webkit Mac10\.5 (dbg)(1)": "chromium-mac-leopard",
70         r"Webkit Mac10\.5 (dbg)(2)": "chromium-mac-leopard",
71         r"Webkit Mac10\.6": "chromium-mac-snowleopard",
72         r"Webkit Mac10\.6 (dbg)": "chromium-mac-snowleopard",
73     }
74
75     def _port_name_for_builder_name(self, builder_name):
76         for regexp, port_name in self._builder_name_to_port_name.items():
77             if re.match(regexp, builder_name):
78                 return port_name
79
80     def port_for_builder(self, builder_name):
81         port_name = self._port_name_for_builder_name(builder_name)
82         assert(port_name)  # Need to update _builder_name_to_port_name
83         port = factory.get(port_name)
84         assert(port)  # Need to update _builder_name_to_port_name
85         return port
86
87
88 class RebaselineTest(AbstractDeclarativeCommand):
89     name = "rebaseline-test"
90     help_text = "Rebaseline a single test from a buildbot.  (Currently works only with build.chromium.org buildbots.)"
91     argument_names = "BUILDER_NAME TEST_NAME SUFFIX"
92
93     def _results_url(self, builder_name):
94         # FIXME: Generalize this command to work with non-build.chromium.org builders.
95         # FIXME: We should really get the buildbot from the tool!
96         builder = ChromiumBuildBot().builder_with_name(builder_name)
97         return builder.accumulated_results_url()
98
99     def _baseline_directory(self, builder_name):
100         port = BuilderToPort().port_for_builder(builder_name)
101         return port.baseline_path()
102
103     def _save_baseline(self, data, target_baseline):
104         self._tool.filesystem.write_binary_file(target_baseline, data)
105         if not self._tool.scm().exists(target_baseline):
106             self._tool.scm().add(target_baseline)
107
108     def _test_root(self, test_name):
109         return os.path.splitext(test_name)[0]
110
111     def _file_name_for_actual_result(self, test_name, suffix):
112         return "%s-actual.%s" % (self._test_root(test_name), suffix)
113
114     def _file_name_for_expected_result(self, test_name, suffix):
115         return "%s-expected.%s" % (self._test_root(test_name), suffix)
116
117     def _rebaseline_test(self, builder_name, test_name, suffix):
118         results_url = self._results_url(builder_name)
119         baseline_directory = self._baseline_directory(builder_name)
120
121         source_baseline = "%s/%s" % (results_url, self._file_name_for_actual_result(test_name, suffix))
122         target_baseline = os.path.join(baseline_directory, self._file_name_for_expected_result(test_name, suffix))
123
124         print "Retrieving %s ..." % source_baseline
125         self._save_baseline(self._tool.web.get_binary(source_baseline), target_baseline)
126
127     def execute(self, options, args, tool):
128         self._rebaseline_test(args[0], args[1], args[2])
129
130
131 class Rebaseline(AbstractDeclarativeCommand):
132     name = "rebaseline"
133     help_text = "Replaces local expected.txt files with new results from build bots"
134
135     # FIXME: This should share more code with FailureReason._builder_to_explain
136     def _builder_to_pull_from(self):
137         builder_statuses = self._tool.buildbot.builder_statuses()
138         red_statuses = [status for status in builder_statuses if not status["is_green"]]
139         print "%s failing" % (pluralize("builder", len(red_statuses)))
140         builder_choices = [status["name"] for status in red_statuses]
141         chosen_name = self._tool.user.prompt_with_list("Which builder to pull results from:", builder_choices)
142         # FIXME: prompt_with_list should really take a set of objects and a set of names and then return the object.
143         for status in red_statuses:
144             if status["name"] == chosen_name:
145                 return (self._tool.buildbot.builder_with_name(chosen_name), status["build_number"])
146
147     def _replace_expectation_with_remote_result(self, local_file, remote_file):
148         (downloaded_file, headers) = urllib.urlretrieve(remote_file)
149         shutil.move(downloaded_file, local_file)
150
151     def _tests_to_update(self, build):
152         failing_tests = build.layout_test_results().tests_matching_failure_types([test_failures.FailureTextMismatch])
153         return self._tool.user.prompt_with_list("Which test(s) to rebaseline:", failing_tests, can_choose_multiple=True)
154
155     def _results_url_for_test(self, build, test):
156         test_base = os.path.splitext(test)[0]
157         actual_path = test_base + "-actual.txt"
158         return build.results_url() + "/" + actual_path
159
160     def execute(self, options, args, tool):
161         builder, build_number = self._builder_to_pull_from()
162         build = builder.build(build_number)
163         port = BuilderToPort().port_for_builder(builder.name())
164
165         for test in self._tests_to_update(build):
166             results_url = self._results_url_for_test(build, test)
167             # Port operates with absolute paths.
168             expected_file = port.expected_filename(test, '.txt')
169             print test
170             self._replace_expectation_with_remote_result(expected_file, results_url)
171
172         # FIXME: We should handle new results too.