9ea39a3b2c4f06efc7f702c43590552543ea9b4b
[WebKit-https.git] / Tools / Scripts / webkitpy / tool / commands / rebaseline.py
1 # Copyright (c) 2010 Google Inc. All rights reserved.
2 #
3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions are
5 # met:
6 #
7 #     * Redistributions of source code must retain the above copyright
8 # notice, this list of conditions and the following disclaimer.
9 #     * Redistributions in binary form must reproduce the above
10 # copyright notice, this list of conditions and the following disclaimer
11 # in the documentation and/or other materials provided with the
12 # distribution.
13 #     * Neither the name of Google Inc. nor the names of its
14 # contributors may be used to endorse or promote products derived from
15 # this software without specific prior written permission.
16 #
17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29 import os.path
30 import re
31 import shutil
32 import urllib
33
34 import webkitpy.common.config.urls as config_urls
35 from webkitpy.common.checkout.baselineoptimizer import BaselineOptimizer
36 from webkitpy.common.net.buildbot import BuildBot
37 from webkitpy.common.net.layouttestresults import LayoutTestResults
38 from webkitpy.common.system.executive import ScriptError
39 from webkitpy.common.system.user import User
40 from webkitpy.layout_tests.controllers.test_result_writer import TestResultWriter
41 from webkitpy.layout_tests.models import test_failures
42 from webkitpy.layout_tests.models.test_expectations import TestExpectations
43 from webkitpy.layout_tests.port import builders
44 from webkitpy.layout_tests.port import test_files
45 from webkitpy.tool.grammar import pluralize
46 from webkitpy.tool.multicommandtool import AbstractDeclarativeCommand
47
48
49 _baseline_suffix_list = ['png', 'txt']
50
51
52 # FIXME: Should TestResultWriter know how to compute this string?
53 def _baseline_name(fs, test_name, suffix):
54     return fs.splitext(test_name)[0] + TestResultWriter.FILENAME_SUFFIX_EXPECTED + "." + suffix
55
56
57 class RebaselineTest(AbstractDeclarativeCommand):
58     name = "rebaseline-test"
59     help_text = "Rebaseline a single test from a buildbot.  (Currently works only with build.chromium.org buildbots.)"
60     argument_names = "BUILDER_NAME TEST_NAME"
61
62     def _results_url(self, builder_name):
63         # FIXME: Generalize this command to work with non-build.chromium.org builders.
64         builder = self._tool.chromium_buildbot().builder_with_name(builder_name)
65         return builder.accumulated_results_url()
66
67     def _baseline_directory(self, builder_name):
68         port = self._tool.port_factory.get_from_builder_name(builder_name)
69         return port.baseline_path()
70
71     def _save_baseline(self, data, target_baseline):
72         if not data:
73             return
74         filesystem = self._tool.filesystem
75         filesystem.maybe_make_directory(filesystem.dirname(target_baseline))
76         filesystem.write_binary_file(target_baseline, data)
77         if not self._tool.scm().exists(target_baseline):
78             self._tool.scm().add(target_baseline)
79
80     def _test_root(self, test_name):
81         return os.path.splitext(test_name)[0]
82
83     def _file_name_for_actual_result(self, test_name, suffix):
84         return "%s-actual.%s" % (self._test_root(test_name), suffix)
85
86     def _file_name_for_expected_result(self, test_name, suffix):
87         return "%s-expected.%s" % (self._test_root(test_name), suffix)
88
89     def _rebaseline_test(self, builder_name, test_name, suffix):
90         results_url = self._results_url(builder_name)
91         baseline_directory = self._baseline_directory(builder_name)
92
93         source_baseline = "%s/%s" % (results_url, self._file_name_for_actual_result(test_name, suffix))
94         target_baseline = os.path.join(baseline_directory, self._file_name_for_expected_result(test_name, suffix))
95
96         print "Retrieving %s." % source_baseline
97         self._save_baseline(self._tool.web.get_binary(source_baseline, convert_404_to_None=True), target_baseline)
98
99     def execute(self, options, args, tool):
100         for suffix in _baseline_suffix_list:
101             self._rebaseline_test(args[0], args[1], suffix)
102
103
104 class OptimizeBaselines(AbstractDeclarativeCommand):
105     name = "optimize-baselines"
106     help_text = "Reshuffles the baselines for the given tests to use as litte space on disk as possible."
107     argument_names = "TEST_NAMES"
108
109     def _optimize_baseline(self, test_name):
110         for suffix in _baseline_suffix_list:
111             baseline_name = _baseline_name(self._tool.filesystem, test_name, suffix)
112             if not self._baseline_optimizer.optimize(baseline_name):
113                 print "Hueristics failed to optimize %s" % baseline_name
114
115     def _to_test_name(self, file_name): 
116         return self._tool.filesystem.relpath(file_name, self._port.layout_tests_dir())
117
118     def execute(self, options, args, tool):
119         self._baseline_optimizer = BaselineOptimizer(tool)
120         self._port = tool.port_factory.get("chromium-win-win7")  # FIXME: This should be selectable.
121
122         for test_name in map(self._to_test_name, test_files.find(self._port, args)):
123             print "Optimizing %s." % test_name
124             self._optimize_baseline(test_name)
125
126
127 class AnalyzeBaselines(AbstractDeclarativeCommand):
128     name = "analyze-baselines"
129     help_text = "Analyzes the baselines for the given tests and prints results that are identical."
130     argument_names = "TEST_NAMES"
131
132     def _print(self, baseline_name, directories_by_result):
133         for result, directories in directories_by_result.items():
134             if len(directories) <= 1:
135                 continue
136             results_names = [self._tool.filesystem.join(directory, baseline_name) for directory in directories]
137             print ' '.join(results_names)
138
139     def _analyze_baseline(self, test_name):
140         for suffix in _baseline_suffix_list:
141             baseline_name = _baseline_name(self._tool.filesystem, test_name, suffix)
142             directories_by_result = self._baseline_optimizer.directories_by_result(baseline_name)
143             self._print(baseline_name, directories_by_result)
144
145     def _to_test_name(self, file_name): 
146         return self._tool.filesystem.relpath(file_name, self._port.layout_tests_dir())
147
148     def execute(self, options, args, tool):
149         self._baseline_optimizer = BaselineOptimizer(tool)
150         self._port = tool.port_factory.get("chromium-win-win7")  # FIXME: This should be selectable.
151
152         for test_name in map(self._to_test_name, test_files.find(self._port, args)):
153             self._analyze_baseline(test_name)
154
155
156 class RebaselineExpectations(AbstractDeclarativeCommand):
157     name = "rebaseline-expectations"
158     help_text = "Rebaselines the tests indicated in test_expectations.txt."
159
160     def _run_webkit_patch(self, args):
161         try:
162             self._tool.executive.run_command([self._tool.path()] + args, cwd=self._tool.scm().checkout_root)
163         except ScriptError, e:
164             pass
165
166     def _is_supported_port(self, port_name):
167         # FIXME: Support non-Chromium ports.
168         return port_name.startswith('chromium-')
169
170     def _expectations(self, port):
171         return TestExpectations(port, None, port.test_expectations(), port.test_configuration())
172
173     def _update_expectations_file(self, port_name):
174         if not self._is_supported_port(port_name):
175             return
176         port = self._tool.port_factory.get(port_name)
177         expectations = self._expectations(port)
178         path = port.path_to_test_expectations_file()
179         self._tool.filesystem.write_text_file(path, expectations.remove_rebaselined_tests(expectations.get_rebaselining_failures()))
180
181     def _tests_to_rebaseline(self, port):
182         return self._expectations(port).get_rebaselining_failures()
183
184     def _rebaseline_port(self, port_name):
185         if not self._is_supported_port(port_name):
186             return
187         builder_name = builders.builder_name_for_port_name(port_name)
188         if not builder_name:
189             return
190         print "Retrieving results for %s from %s." % (port_name, builder_name)
191         for test_name in self._tests_to_rebaseline(self._tool.port_factory.get(port_name)):
192             self._touched_test_names.add(test_name)
193             print "    %s" % test_name
194             self._run_webkit_patch(['rebaseline-test', builder_name, test_name])
195
196     def execute(self, options, args, tool):
197         self._touched_test_names = set([])
198         for port_name in tool.port_factory.all_port_names():
199             self._rebaseline_port(port_name)
200         for port_name in tool.port_factory.all_port_names():
201             self._update_expectations_file(port_name)
202         for test_name in self._touched_test_names:
203             print "Optimizing baselines for %s." % test_name
204             self._run_webkit_patch(['optimize-baselines', test_name])
205
206
207 class Rebaseline(AbstractDeclarativeCommand):
208     name = "rebaseline"
209     help_text = "Replaces local expected.txt files with new results from build bots"
210
211     # FIXME: This should share more code with FailureReason._builder_to_explain
212     def _builder_to_pull_from(self):
213         builder_statuses = self._tool.buildbot.builder_statuses()
214         red_statuses = [status for status in builder_statuses if not status["is_green"]]
215         print "%s failing" % (pluralize("builder", len(red_statuses)))
216         builder_choices = [status["name"] for status in red_statuses]
217         chosen_name = self._tool.user.prompt_with_list("Which builder to pull results from:", builder_choices)
218         # FIXME: prompt_with_list should really take a set of objects and a set of names and then return the object.
219         for status in red_statuses:
220             if status["name"] == chosen_name:
221                 return (self._tool.buildbot.builder_with_name(chosen_name), status["build_number"])
222
223     def _replace_expectation_with_remote_result(self, local_file, remote_file):
224         (downloaded_file, headers) = urllib.urlretrieve(remote_file)
225         shutil.move(downloaded_file, local_file)
226
227     def _tests_to_update(self, build):
228         failing_tests = build.layout_test_results().tests_matching_failure_types([test_failures.FailureTextMismatch])
229         return self._tool.user.prompt_with_list("Which test(s) to rebaseline:", failing_tests, can_choose_multiple=True)
230
231     def _results_url_for_test(self, build, test):
232         test_base = os.path.splitext(test)[0]
233         actual_path = test_base + "-actual.txt"
234         return build.results_url() + "/" + actual_path
235
236     def execute(self, options, args, tool):
237         builder, build_number = self._builder_to_pull_from()
238         build = builder.build(build_number)
239         port = tool.port_factory.get_from_builder_name(builder.name())
240
241         for test in self._tests_to_update(build):
242             results_url = self._results_url_for_test(build, test)
243             # Port operates with absolute paths.
244             expected_file = port.expected_filename(test, '.txt')
245             print test
246             self._replace_expectation_with_remote_result(expected_file, results_url)
247
248         # FIXME: We should handle new results too.