effe6ce4aba3aed0d121c9cc3aabbc76c10ae7c9
[WebKit-https.git] / Tools / Scripts / webkitpy / tool / commands / rebaseline.py
1 # Copyright (c) 2010 Google Inc. All rights reserved.
2 #
3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions are
5 # met:
6 #
7 #     * Redistributions of source code must retain the above copyright
8 # notice, this list of conditions and the following disclaimer.
9 #     * Redistributions in binary form must reproduce the above
10 # copyright notice, this list of conditions and the following disclaimer
11 # in the documentation and/or other materials provided with the
12 # distribution.
13 #     * Neither the name of Google Inc. nor the names of its
14 # contributors may be used to endorse or promote products derived from
15 # this software without specific prior written permission.
16 #
17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29 import json
30 import logging
31 import optparse
32 import os.path
33 import re
34 import shutil
35 import sys
36 import urllib
37
38 import webkitpy.common.config.urls as config_urls
39 from webkitpy.common.checkout.baselineoptimizer import BaselineOptimizer
40 from webkitpy.common.net.buildbot import BuildBot
41 from webkitpy.common.net.layouttestresults import LayoutTestResults
42 from webkitpy.common.system.executive import ScriptError
43 from webkitpy.common.system.user import User
44 from webkitpy.layout_tests.controllers.test_result_writer import TestResultWriter
45 from webkitpy.layout_tests.models import test_failures
46 from webkitpy.layout_tests.models.test_configuration import TestConfiguration
47 from webkitpy.layout_tests.models.test_expectations import TestExpectations, BASELINE_SUFFIX_LIST
48 from webkitpy.layout_tests.port import builders
49 from webkitpy.layout_tests.port import factory
50 from webkitpy.tool.grammar import pluralize
51 from webkitpy.tool.multicommandtool import AbstractDeclarativeCommand
52
53
54 _log = logging.getLogger(__name__)
55
56
57 # FIXME: Should TestResultWriter know how to compute this string?
58 def _baseline_name(fs, test_name, suffix):
59     return fs.splitext(test_name)[0] + TestResultWriter.FILENAME_SUFFIX_EXPECTED + "." + suffix
60
61
62 class AbstractRebaseliningCommand(AbstractDeclarativeCommand):
63     move_overwritten_baselines_option = optparse.make_option("--move-overwritten-baselines", action="store_true", default=False,
64         help="Move overwritten baselines elsewhere in the baseline path. This is for bringing up new ports.")
65
66     no_optimize_option = optparse.make_option('--no-optimize', dest='optimize', action='store_false', default=True,
67         help=('Do not optimize/de-dup the expectations after rebaselining (default is to de-dup automatically). '
68               'You can use "webkit-patch optimize-baselines" to optimize separately.'))
69
70     platform_options = factory.platform_options()
71
72     results_directory_option = optparse.make_option("--results-directory", help="Local results directory to use")
73
74     suffixes_option = optparse.make_option("--suffixes", default=','.join(BASELINE_SUFFIX_LIST), action="store",
75         help="Comma-separated-list of file types to rebaseline")
76
77     def __init__(self, options=None):
78         super(AbstractRebaseliningCommand, self).__init__(options=options)
79         self._baseline_suffix_list = BASELINE_SUFFIX_LIST
80
81
82 class RebaselineTest(AbstractRebaseliningCommand):
83     name = "rebaseline-test-internal"
84     help_text = "Rebaseline a single test from a buildbot. Only intended for use by other webkit-patch commands."
85
86     def __init__(self):
87         super(RebaselineTest, self).__init__(options=[
88             self.no_optimize_option,
89             self.results_directory_option,
90             self.suffixes_option,
91             optparse.make_option("--builder", help="Builder to pull new baselines from"),
92             optparse.make_option("--move-overwritten-baselines-to", action="append", default=[],
93                 help="Platform to move existing baselines to before rebaselining. This is for bringing up new ports."),
94             optparse.make_option("--test", help="Test to rebaseline"),
95             ])
96         self._scm_changes = {'add': []}
97
98     def _results_url(self, builder_name):
99         return self._tool.buildbot_for_builder_name(builder_name).builder_with_name(builder_name).latest_layout_test_results_url()
100
101     def _baseline_directory(self, builder_name):
102         port = self._tool.port_factory.get_from_builder_name(builder_name)
103         override_dir = builders.rebaseline_override_dir(builder_name)
104         if override_dir:
105             return self._tool.filesystem.join(port.layout_tests_dir(), 'platform', override_dir)
106         return port.baseline_version_dir()
107
108     def _copy_existing_baseline(self, move_overwritten_baselines_to, test_name, suffix):
109         old_baselines = []
110         new_baselines = []
111
112         # Need to gather all the baseline paths before modifying the filesystem since
113         # the modifications can affect the results of port.expected_filename.
114         for platform in move_overwritten_baselines_to:
115             port = self._tool.port_factory.get(platform)
116             old_baseline = port.expected_filename(test_name, "." + suffix)
117             if not self._tool.filesystem.exists(old_baseline):
118                 _log.debug("No existing baseline for %s." % test_name)
119                 continue
120
121             new_baseline = self._tool.filesystem.join(port.baseline_path(), self._file_name_for_expected_result(test_name, suffix))
122             if self._tool.filesystem.exists(new_baseline):
123                 _log.debug("Existing baseline at %s, not copying over it." % new_baseline)
124                 continue
125
126             old_baselines.append(old_baseline)
127             new_baselines.append(new_baseline)
128
129         for i in range(len(old_baselines)):
130             old_baseline = old_baselines[i]
131             new_baseline = new_baselines[i]
132
133             _log.debug("Copying baseline from %s to %s." % (old_baseline, new_baseline))
134             self._tool.filesystem.maybe_make_directory(self._tool.filesystem.dirname(new_baseline))
135             self._tool.filesystem.copyfile(old_baseline, new_baseline)
136             if not self._tool.scm().exists(new_baseline):
137                 self._add_to_scm(new_baseline)
138
139     def _save_baseline(self, data, target_baseline):
140         if not data:
141             return
142         filesystem = self._tool.filesystem
143         filesystem.maybe_make_directory(filesystem.dirname(target_baseline))
144         filesystem.write_binary_file(target_baseline, data)
145         if not self._tool.scm().exists(target_baseline):
146             self._add_to_scm(target_baseline)
147
148     def _add_to_scm(self, path):
149         self._scm_changes['add'].append(path)
150
151     def _update_expectations_file(self, builder_name, test_name):
152         port = self._tool.port_factory.get_from_builder_name(builder_name)
153         expectations = TestExpectations(port, include_overrides=False)
154
155         for test_configuration in port.all_test_configurations():
156             if test_configuration.version == port.test_configuration().version:
157                 expectationsString = expectations.remove_configuration_from_test(test_name, test_configuration)
158
159         self._tool.filesystem.write_text_file(port.path_to_test_expectations_file(), expectationsString)
160
161     def _test_root(self, test_name):
162         return os.path.splitext(test_name)[0]
163
164     def _file_name_for_actual_result(self, test_name, suffix):
165         return "%s-actual.%s" % (self._test_root(test_name), suffix)
166
167     def _file_name_for_expected_result(self, test_name, suffix):
168         return "%s-expected.%s" % (self._test_root(test_name), suffix)
169
170     def _rebaseline_test(self, builder_name, test_name, move_overwritten_baselines_to, suffix, results_url):
171         baseline_directory = self._baseline_directory(builder_name)
172
173         source_baseline = "%s/%s" % (results_url, self._file_name_for_actual_result(test_name, suffix))
174         target_baseline = self._tool.filesystem.join(baseline_directory, self._file_name_for_expected_result(test_name, suffix))
175
176         if move_overwritten_baselines_to:
177             self._copy_existing_baseline(move_overwritten_baselines_to, test_name, suffix)
178
179         _log.debug("Retrieving %s." % source_baseline)
180         self._save_baseline(self._tool.web.get_binary(source_baseline, convert_404_to_None=True), target_baseline)
181
182     def _rebaseline_test_and_update_expectations(self, builder_name, test_name, platforms_to_move_existing_baselines_to, results_url):
183         for suffix in self._baseline_suffix_list:
184             self._rebaseline_test(builder_name, test_name, platforms_to_move_existing_baselines_to, suffix, results_url)
185         self._update_expectations_file(builder_name, test_name)
186
187     def execute(self, options, args, tool):
188         self._baseline_suffix_list = options.suffixes.split(',')
189         results_url = options.results_directory or self._results_url(options.builder)
190         self._rebaseline_test_and_update_expectations(options.builder, options.test, options.move_overwritten_baselines_to, results_url)
191         print json.dumps(self._scm_changes)
192
193
194 class OptimizeBaselines(AbstractRebaseliningCommand):
195     name = "optimize-baselines"
196     help_text = "Reshuffles the baselines for the given tests to use as litte space on disk as possible."
197     argument_names = "TEST_NAMES"
198
199     def __init__(self):
200         return super(OptimizeBaselines, self).__init__(options=[self.suffixes_option])
201
202     def _optimize_baseline(self, test_name):
203         for suffix in self._baseline_suffix_list:
204             baseline_name = _baseline_name(self._tool.filesystem, test_name, suffix)
205             if not self._baseline_optimizer.optimize(baseline_name):
206                 print "Heuristics failed to optimize %s" % baseline_name
207
208     def execute(self, options, args, tool):
209         self._baseline_suffix_list = options.suffixes.split(',')
210         self._baseline_optimizer = BaselineOptimizer(tool)
211         self._port = tool.port_factory.get()
212         for test_name in self._port.tests(args):
213             _log.info("Optimizing %s" % test_name)
214             self._optimize_baseline(test_name)
215
216
217 class AnalyzeBaselines(AbstractRebaseliningCommand):
218     name = "analyze-baselines"
219     help_text = "Analyzes the baselines for the given tests and prints results that are identical."
220     argument_names = "TEST_NAMES"
221
222     def __init__(self):
223         super(AnalyzeBaselines, self).__init__(options=[
224             self.suffixes_option,
225             optparse.make_option('--missing', action='store_true', default=False, help='show missing baselines as well'),
226             ])
227         self._optimizer_class = BaselineOptimizer  # overridable for testing
228
229     def _write(self, msg):
230         print msg
231
232     def _analyze_baseline(self, options, test_name):
233         for suffix in self._baseline_suffix_list:
234             baseline_name = _baseline_name(self._tool.filesystem, test_name, suffix)
235             results_by_directory = self._baseline_optimizer.read_results_by_directory(baseline_name)
236             if results_by_directory:
237                 self._write("%s:" % baseline_name)
238                 self._baseline_optimizer.write_by_directory(results_by_directory, self._write, "  ")
239             elif options.missing:
240                 self._write("%s: (no baselines found)" % baseline_name)
241
242     def execute(self, options, args, tool):
243         self._baseline_suffix_list = options.suffixes.split(',')
244         self._baseline_optimizer = self._optimizer_class(tool)
245         self._port = tool.port_factory.get()
246         for test_name in self._port.tests(args):
247             self._analyze_baseline(options, test_name)
248
249
250 class AbstractParallelRebaselineCommand(AbstractRebaseliningCommand):
251
252     def _run_webkit_patch(self, args, verbose):
253         try:
254             verbose_args = ['--verbose'] if verbose else []
255             stderr = self._tool.executive.run_command([self._tool.path()] + verbose_args + args, cwd=self._tool.scm().checkout_root, return_stderr=True)
256             for line in stderr.splitlines():
257                 print >> sys.stderr, line
258         except ScriptError, e:
259             _log.error(e)
260
261     def _builders_to_fetch_from(self, builders):
262         # This routine returns the subset of builders that will cover all of the baseline search paths
263         # used in the input list. In particular, if the input list contains both Release and Debug
264         # versions of a configuration, we *only* return the Release version (since we don't save
265         # debug versions of baselines).
266         release_builders = set()
267         debug_builders = set()
268         builders_to_fallback_paths = {}
269         for builder in builders:
270             port = self._tool.port_factory.get_from_builder_name(builder)
271             if port.test_configuration().build_type == 'Release':
272                 release_builders.add(builder)
273             else:
274                 debug_builders.add(builder)
275         for builder in list(release_builders) + list(debug_builders):
276             port = self._tool.port_factory.get_from_builder_name(builder)
277             fallback_path = port.baseline_search_path()
278             if fallback_path not in builders_to_fallback_paths.values():
279                 builders_to_fallback_paths[builder] = fallback_path
280         return builders_to_fallback_paths.keys()
281
282     def _rebaseline_commands(self, test_list, options):
283
284         path_to_webkit_patch = self._tool.path()
285         cwd = self._tool.scm().checkout_root
286         commands = []
287         for test in test_list:
288             for builder in self._builders_to_fetch_from(test_list[test]):
289                 suffixes = ','.join(test_list[test][builder])
290                 cmd_line = [path_to_webkit_patch, 'rebaseline-test-internal', '--suffixes', suffixes, '--builder', builder, '--test', test]
291                 if options.move_overwritten_baselines:
292                     move_overwritten_baselines_to = builders.move_overwritten_baselines_to(builder)
293                     for platform in move_overwritten_baselines_to:
294                         cmd_line.extend(['--move-overwritten-baselines-to', platform])
295                 if options.results_directory:
296                     cmd_line.extend(['--results_directory', options.results_directory])
297                 if options.verbose:
298                     cmd_line.append('--verbose')
299                 commands.append(tuple([cmd_line, cwd]))
300         return commands
301
302     def _files_to_add(self, command_results):
303         files_to_add = set()
304         for output in [result[1].split('\n') for result in command_results]:
305             file_added = False
306             for line in output:
307                 try:
308                     if line:
309                         files_to_add.update(json.loads(line)['add'])
310                         file_added = True
311                 except ValueError, e:
312                     _log.debug('"%s" is not a JSON object, ignoring' % line)
313
314             if not file_added:
315                 _log.debug('Could not add file based off output "%s"' % output)
316
317
318         return list(files_to_add)
319
320     def _optimize_baselines(self, test_list, verbose=False):
321         # We don't run this in parallel because modifying the SCM in parallel is unreliable.
322         for test in test_list:
323             all_suffixes = set()
324             for builder in self._builders_to_fetch_from(test_list[test]):
325                 all_suffixes.update(test_list[test][builder])
326             # FIXME: We should propagate the platform options as well.
327             self._run_webkit_patch(['optimize-baselines', '--suffixes', ','.join(all_suffixes), test], verbose)
328
329     def _rebaseline(self, options, test_list):
330         for test, builders in sorted(test_list.items()):
331             _log.info("Rebaselining %s" % test)
332             for builder, suffixes in sorted(builders.items()):
333                 _log.debug("  %s: %s" % (builder, ",".join(suffixes)))
334
335         commands = self._rebaseline_commands(test_list, options)
336         command_results = self._tool.executive.run_in_parallel(commands)
337
338         log_output = '\n'.join(result[2] for result in command_results).replace('\n\n', '\n')
339         for line in log_output.split('\n'):
340             if line:
341                 print >> sys.stderr, line  # FIXME: Figure out how to log properly.
342
343         files_to_add = self._files_to_add(command_results)
344         if files_to_add:
345             self._tool.scm().add_list(list(files_to_add))
346
347         if options.optimize:
348             self._optimize_baselines(test_list, options.verbose)
349
350
351 class RebaselineJson(AbstractParallelRebaselineCommand):
352     name = "rebaseline-json"
353     help_text = "Rebaseline based off JSON passed to stdin. Intended to only be called from other scripts."
354
355     def __init__(self,):
356         return super(RebaselineJson, self).__init__(options=[
357             self.move_overwritten_baselines_option,
358             self.no_optimize_option,
359             self.results_directory_option,
360             ])
361
362     def execute(self, options, args, tool):
363         self._rebaseline(options, json.loads(sys.stdin.read()))
364
365
366 class RebaselineExpectations(AbstractParallelRebaselineCommand):
367     name = "rebaseline-expectations"
368     help_text = "Rebaselines the tests indicated in TestExpectations."
369
370     def __init__(self):
371         # FIXME: We should also support platform_options here so that we only look at some TestExpectations files instead of all of them.
372         return super(RebaselineExpectations, self).__init__(options=[
373             self.move_overwritten_baselines_option,
374             self.no_optimize_option,
375             ])
376
377     def _update_expectations_files(self, port_name):
378         port = self._tool.port_factory.get(port_name)
379
380         expectations = TestExpectations(port)
381         for path in port.expectations_dict():
382             if self._tool.filesystem.exists(path):
383                 self._tool.filesystem.write_text_file(path, expectations.remove_rebaselined_tests(expectations.get_rebaselining_failures(), path))
384
385     def _tests_to_rebaseline(self, port):
386         tests_to_rebaseline = {}
387         expectations = TestExpectations(port, include_overrides=True)
388         for test in expectations.get_rebaselining_failures():
389             tests_to_rebaseline[test] = TestExpectations.suffixes_for_expectations(expectations.get_expectations(test))
390         return tests_to_rebaseline
391
392     def _add_tests_to_rebaseline_for_port(self, port_name):
393         builder_name = builders.builder_name_for_port_name(port_name)
394         if not builder_name:
395             return
396         tests = self._tests_to_rebaseline(self._tool.port_factory.get(port_name)).items()
397
398         if tests:
399             _log.debug("Retrieving results for %s from %s." % (port_name, builder_name))
400
401         for test_name, suffixes in tests:
402             _log.info("    %s (%s)" % (test_name, ','.join(suffixes)))
403             if test_name not in self._test_list:
404                 self._test_list[test_name] = {}
405             self._test_list[test_name][builder_name] = suffixes
406
407     def execute(self, options, args, tool):
408         options.results_directory = None
409         self._test_list = {}
410         for port_name in tool.port_factory.all_port_names():
411             self._add_tests_to_rebaseline_for_port(port_name)
412         if not self._test_list:
413             _log.warning("Did not find any tests marked Rebaseline.")
414             return
415
416         self._rebaseline(options, self._test_list)
417
418         for port_name in tool.port_factory.all_port_names():
419             self._update_expectations_files(port_name)
420
421
422 class Rebaseline(AbstractParallelRebaselineCommand):
423     name = "rebaseline"
424     help_text = "Rebaseline tests with results from the build bots. Shows the list of failing tests on the builders if no test names are provided."
425     argument_names = "[TEST_NAMES]"
426
427     def __init__(self):
428         super(Rebaseline, self).__init__(options=[
429             self.move_overwritten_baselines_option,
430             self.no_optimize_option,
431             # FIXME: should we support the platform options in addition to (or instead of) --builders?
432             self.suffixes_option,
433             optparse.make_option("--builders", default=None, action="append", help="Comma-separated-list of builders to pull new baselines from (can also be provided multiple times)"),
434             ])
435
436     def _builders_to_pull_from(self):
437         chromium_buildbot_builder_names = []
438         webkit_buildbot_builder_names = []
439         for name in builders.all_builder_names():
440             if self._tool.port_factory.get_from_builder_name(name).is_chromium():
441                 chromium_buildbot_builder_names.append(name)
442             else:
443                 webkit_buildbot_builder_names.append(name)
444
445         titles = ["build.webkit.org bots", "build.chromium.org bots"]
446         lists = [webkit_buildbot_builder_names, chromium_buildbot_builder_names]
447
448         chosen_names = self._tool.user.prompt_with_multiple_lists("Which builder to pull results from:", titles, lists, can_choose_multiple=True)
449         return [self._builder_with_name(name) for name in chosen_names]
450
451     def _builder_with_name(self, name):
452         return self._tool.buildbot_for_builder_name(name).builder_with_name(name)
453
454     def _tests_to_update(self, builder):
455         failing_tests = builder.latest_layout_test_results().tests_matching_failure_types([test_failures.FailureTextMismatch])
456         return self._tool.user.prompt_with_list("Which test(s) to rebaseline for %s:" % builder.name(), failing_tests, can_choose_multiple=True)
457
458     def execute(self, options, args, tool):
459         options.results_directory = None
460         if options.builders:
461             builders = []
462             for builder_names in options.builders:
463                 builders += [self._builder_with_name(name) for name in builder_names.split(",")]
464         else:
465             builders = self._builders_to_pull_from()
466
467         test_list = {}
468         suffixes_to_update = options.suffixes.split(",")
469
470         for builder in builders:
471             tests = args or self._tests_to_update(builder)
472             for test in tests:
473                 if test not in test_list:
474                     test_list[test] = {}
475                 test_list[test][builder.name()] = suffixes_to_update
476
477         if options.verbose:
478             _log.debug("rebaseline-json: " + str(test_list))
479
480         self._rebaseline(options, test_list)