There should be a way to disable optimizer in webkit-patch rebaseline-expectations
[WebKit-https.git] / Tools / Scripts / webkitpy / tool / commands / rebaseline.py
1 # Copyright (c) 2010 Google Inc. All rights reserved.
2 #
3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions are
5 # met:
6 #
7 #     * Redistributions of source code must retain the above copyright
8 # notice, this list of conditions and the following disclaimer.
9 #     * Redistributions in binary form must reproduce the above
10 # copyright notice, this list of conditions and the following disclaimer
11 # in the documentation and/or other materials provided with the
12 # distribution.
13 #     * Neither the name of Google Inc. nor the names of its
14 # contributors may be used to endorse or promote products derived from
15 # this software without specific prior written permission.
16 #
17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29 import optparse
30 import os.path
31 import re
32 import shutil
33 import urllib
34
35 import webkitpy.common.config.urls as config_urls
36 from webkitpy.common.checkout.baselineoptimizer import BaselineOptimizer
37 from webkitpy.common.net.buildbot import BuildBot
38 from webkitpy.common.net.layouttestresults import LayoutTestResults
39 from webkitpy.common.system.executive import ScriptError
40 from webkitpy.common.system.user import User
41 from webkitpy.layout_tests.controllers.test_result_writer import TestResultWriter
42 from webkitpy.layout_tests.models import test_failures
43 from webkitpy.layout_tests.models.test_expectations import TestExpectations
44 from webkitpy.layout_tests.port import builders
45 from webkitpy.tool.grammar import pluralize
46 from webkitpy.tool.multicommandtool import AbstractDeclarativeCommand
47
48
49 _baseline_suffix_list = ['png', 'txt']
50
51
52 # FIXME: Should TestResultWriter know how to compute this string?
53 def _baseline_name(fs, test_name, suffix):
54     return fs.splitext(test_name)[0] + TestResultWriter.FILENAME_SUFFIX_EXPECTED + "." + suffix
55
56
57 class RebaselineTest(AbstractDeclarativeCommand):
58     name = "rebaseline-test"
59     help_text = "Rebaseline a single test from a buildbot.  (Currently works only with build.chromium.org buildbots.)"
60     argument_names = "BUILDER_NAME TEST_NAME"
61
62     def _results_url(self, builder_name):
63         # FIXME: Generalize this command to work with non-build.chromium.org builders.
64         builder = self._tool.chromium_buildbot().builder_with_name(builder_name)
65         return builder.accumulated_results_url()
66
67     def _baseline_directory(self, builder_name):
68         port = self._tool.port_factory.get_from_builder_name(builder_name)
69         return port.baseline_path()
70
71     def _save_baseline(self, data, target_baseline):
72         if not data:
73             return
74         filesystem = self._tool.filesystem
75         filesystem.maybe_make_directory(filesystem.dirname(target_baseline))
76         filesystem.write_binary_file(target_baseline, data)
77         if not self._tool.scm().exists(target_baseline):
78             self._tool.scm().add(target_baseline)
79
80     def _test_root(self, test_name):
81         return os.path.splitext(test_name)[0]
82
83     def _file_name_for_actual_result(self, test_name, suffix):
84         return "%s-actual.%s" % (self._test_root(test_name), suffix)
85
86     def _file_name_for_expected_result(self, test_name, suffix):
87         return "%s-expected.%s" % (self._test_root(test_name), suffix)
88
89     def _rebaseline_test(self, builder_name, test_name, suffix):
90         results_url = self._results_url(builder_name)
91         baseline_directory = self._baseline_directory(builder_name)
92
93         source_baseline = "%s/%s" % (results_url, self._file_name_for_actual_result(test_name, suffix))
94         target_baseline = os.path.join(baseline_directory, self._file_name_for_expected_result(test_name, suffix))
95
96         print "Retrieving %s." % source_baseline
97         self._save_baseline(self._tool.web.get_binary(source_baseline, convert_404_to_None=True), target_baseline)
98
99     def execute(self, options, args, tool):
100         for suffix in _baseline_suffix_list:
101             self._rebaseline_test(args[0], args[1], suffix)
102
103
104 class OptimizeBaselines(AbstractDeclarativeCommand):
105     name = "optimize-baselines"
106     help_text = "Reshuffles the baselines for the given tests to use as litte space on disk as possible."
107     argument_names = "TEST_NAMES"
108
109     def _optimize_baseline(self, test_name):
110         for suffix in _baseline_suffix_list:
111             baseline_name = _baseline_name(self._tool.filesystem, test_name, suffix)
112             if not self._baseline_optimizer.optimize(baseline_name):
113                 print "Hueristics failed to optimize %s" % baseline_name
114
115     def execute(self, options, args, tool):
116         self._baseline_optimizer = BaselineOptimizer(tool)
117         self._port = tool.port_factory.get("chromium-win-win7")  # FIXME: This should be selectable.
118
119         for test_name in self._port.tests(args):
120             print "Optimizing %s." % test_name
121             self._optimize_baseline(test_name)
122
123
124 class AnalyzeBaselines(AbstractDeclarativeCommand):
125     name = "analyze-baselines"
126     help_text = "Analyzes the baselines for the given tests and prints results that are identical."
127     argument_names = "TEST_NAMES"
128
129     def _print(self, baseline_name, directories_by_result):
130         for result, directories in directories_by_result.items():
131             if len(directories) <= 1:
132                 continue
133             results_names = [self._tool.filesystem.join(directory, baseline_name) for directory in directories]
134             print ' '.join(results_names)
135
136     def _analyze_baseline(self, test_name):
137         for suffix in _baseline_suffix_list:
138             baseline_name = _baseline_name(self._tool.filesystem, test_name, suffix)
139             directories_by_result = self._baseline_optimizer.directories_by_result(baseline_name)
140             self._print(baseline_name, directories_by_result)
141
142     def execute(self, options, args, tool):
143         self._baseline_optimizer = BaselineOptimizer(tool)
144         self._port = tool.port_factory.get("chromium-win-win7")  # FIXME: This should be selectable.
145
146         for test_name in self._port.tests(args):
147             self._analyze_baseline(test_name)
148
149
150 class RebaselineExpectations(AbstractDeclarativeCommand):
151     name = "rebaseline-expectations"
152     help_text = "Rebaselines the tests indicated in test_expectations.txt."
153
154     def __init__(self):
155         options = [
156             optparse.make_option('--no-optimize', dest='optimize', action='store_false', default=True,
157                 help=('Do not optimize/de-dup the expectations after rebaselining '
158                       '(default is to de-dup automatically). '
159                       'You can use "webkit-patch optimize-baselines" to optimize separately.')),
160         ]
161         AbstractDeclarativeCommand.__init__(self, options=options)
162
163     def _run_webkit_patch(self, args):
164         try:
165             self._tool.executive.run_command([self._tool.path()] + args, cwd=self._tool.scm().checkout_root)
166         except ScriptError, e:
167             pass
168
169     def _is_supported_port(self, port_name):
170         # FIXME: Support non-Chromium ports.
171         return port_name.startswith('chromium-')
172
173     def _expectations(self, port):
174         return TestExpectations(port, None, port.test_expectations(), port.test_configuration())
175
176     def _update_expectations_file(self, port_name):
177         if not self._is_supported_port(port_name):
178             return
179         port = self._tool.port_factory.get(port_name)
180         expectations = self._expectations(port)
181         path = port.path_to_test_expectations_file()
182         self._tool.filesystem.write_text_file(path, expectations.remove_rebaselined_tests(expectations.get_rebaselining_failures()))
183
184     def _tests_to_rebaseline(self, port):
185         return self._expectations(port).get_rebaselining_failures()
186
187     def _rebaseline_port(self, port_name):
188         if not self._is_supported_port(port_name):
189             return
190         builder_name = builders.builder_name_for_port_name(port_name)
191         if not builder_name:
192             return
193         print "Retrieving results for %s from %s." % (port_name, builder_name)
194         for test_name in self._tests_to_rebaseline(self._tool.port_factory.get(port_name)):
195             self._touched_test_names.add(test_name)
196             print "    %s" % test_name
197             self._run_webkit_patch(['rebaseline-test', builder_name, test_name])
198
199     def execute(self, options, args, tool):
200         self._touched_test_names = set([])
201         for port_name in tool.port_factory.all_port_names():
202             self._rebaseline_port(port_name)
203         for port_name in tool.port_factory.all_port_names():
204             self._update_expectations_file(port_name)
205         if not options.optimize:
206             return
207         for test_name in self._touched_test_names:
208             print "Optimizing baselines for %s." % test_name
209             self._run_webkit_patch(['optimize-baselines', test_name])
210
211
212 class Rebaseline(AbstractDeclarativeCommand):
213     name = "rebaseline"
214     help_text = "Replaces local expected.txt files with new results from build bots"
215
216     # FIXME: This should share more code with FailureReason._builder_to_explain
217     def _builder_to_pull_from(self):
218         builder_statuses = self._tool.buildbot.builder_statuses()
219         red_statuses = [status for status in builder_statuses if not status["is_green"]]
220         print "%s failing" % (pluralize("builder", len(red_statuses)))
221         builder_choices = [status["name"] for status in red_statuses]
222         chosen_name = self._tool.user.prompt_with_list("Which builder to pull results from:", builder_choices)
223         # FIXME: prompt_with_list should really take a set of objects and a set of names and then return the object.
224         for status in red_statuses:
225             if status["name"] == chosen_name:
226                 return (self._tool.buildbot.builder_with_name(chosen_name), status["build_number"])
227
228     def _replace_expectation_with_remote_result(self, local_file, remote_file):
229         (downloaded_file, headers) = urllib.urlretrieve(remote_file)
230         shutil.move(downloaded_file, local_file)
231
232     def _tests_to_update(self, build):
233         failing_tests = build.layout_test_results().tests_matching_failure_types([test_failures.FailureTextMismatch])
234         return self._tool.user.prompt_with_list("Which test(s) to rebaseline:", failing_tests, can_choose_multiple=True)
235
236     def _results_url_for_test(self, build, test):
237         test_base = os.path.splitext(test)[0]
238         actual_path = test_base + "-actual.txt"
239         return build.results_url() + "/" + actual_path
240
241     def execute(self, options, args, tool):
242         builder, build_number = self._builder_to_pull_from()
243         build = builder.build(build_number)
244         port = tool.port_factory.get_from_builder_name(builder.name())
245
246         for test in self._tests_to_update(build):
247             results_url = self._results_url_for_test(build, test)
248             # Port operates with absolute paths.
249             expected_file = port.expected_filename(test, '.txt')
250             print test
251             self._replace_expectation_with_remote_result(expected_file, results_url)
252
253         # FIXME: We should handle new results too.