Allow a port to run tests with a custom device setup
[WebKit-https.git] / Tools / Scripts / webkitpy / layout_tests / controllers / manager.py
1 # Copyright (C) 2010 Google Inc. All rights reserved.
2 # Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Szeged
3 #
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
6 # met:
7 #
8 #     * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 #     * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
13 # distribution.
14 #     * Neither the name of Google Inc. nor the names of its
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
17 #
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30 """
31 The Manager runs a series of tests (TestType interface) against a set
32 of test files.  If a test file fails a TestType, it returns a list of TestFailure
33 objects to the Manager. The Manager then aggregates the TestFailures to
34 create a final report.
35 """
36
37 import json
38 import logging
39 import random
40 import sys
41 import time
42 from collections import defaultdict
43
44 from webkitpy.common.checkout.scm.detection import SCMDetector
45 from webkitpy.common.net.file_uploader import FileUploader
46 from webkitpy.layout_tests.controllers.layout_test_finder import LayoutTestFinder
47 from webkitpy.layout_tests.controllers.layout_test_runner import LayoutTestRunner
48 from webkitpy.layout_tests.controllers.test_result_writer import TestResultWriter
49 from webkitpy.layout_tests.layout_package import json_layout_results_generator
50 from webkitpy.layout_tests.layout_package import json_results_generator
51 from webkitpy.layout_tests.models import test_expectations
52 from webkitpy.layout_tests.models import test_failures
53 from webkitpy.layout_tests.models import test_results
54 from webkitpy.layout_tests.models import test_run_results
55 from webkitpy.layout_tests.models.test_input import TestInput
56 from webkitpy.layout_tests.models.test_run_results import INTERRUPTED_EXIT_STATUS
57 from webkitpy.tool.grammar import pluralize
58
59 _log = logging.getLogger(__name__)
60
61 TestExpectations = test_expectations.TestExpectations
62
63
64 class Manager(object):
65     """A class for managing running a series of tests on a series of layout
66     test files."""
67
68     def __init__(self, port, options, printer):
69         """Initialize test runner data structures.
70
71         Args:
72           port: an object implementing port-specific
73           options: a dictionary of command line options
74           printer: a Printer object to record updates to.
75         """
76         self._port = port
77         self._filesystem = port.host.filesystem
78         self._options = options
79         self._printer = printer
80         self._expectations = None
81         self.HTTP_SUBDIR = 'http' + port.TEST_PATH_SEPARATOR
82         self.WEBSOCKET_SUBDIR = 'websocket' + port.TEST_PATH_SEPARATOR
83         self.web_platform_test_subdir = self._port.web_platform_test_server_doc_root()
84         self.LAYOUT_TESTS_DIRECTORY = 'LayoutTests'
85         self._results_directory = self._port.results_directory()
86         self._finder = LayoutTestFinder(self._port, self._options)
87         self._runner = LayoutTestRunner(self._options, self._port, self._printer, self._results_directory, self._test_is_slow)
88
89     def _collect_tests(self, args):
90         return self._finder.find_tests(self._options, args)
91
92     def _is_http_test(self, test):
93         return self.HTTP_SUBDIR in test or self._is_websocket_test(test) or self._is_web_platform_test(test)
94
95     def _is_websocket_test(self, test):
96         return self.WEBSOCKET_SUBDIR in test
97
98     def _is_web_platform_test(self, test):
99         return self.web_platform_test_subdir in test
100
101     def _custom_device_for_test(self, test):
102         for device_class in self._port.CUSTOM_DEVICE_CLASSES:
103             directory_suffix = device_class + self._port.TEST_PATH_SEPARATOR
104             if directory_suffix in test:
105                 return device_class
106         return None
107
108     def _http_tests(self, test_names):
109         return set(test for test in test_names if self._is_http_test(test))
110
111     def _prepare_lists(self, paths, test_names):
112         tests_to_skip = self._finder.skip_tests(paths, test_names, self._expectations, self._http_tests(test_names))
113         tests_to_run = [test for test in test_names if test not in tests_to_skip]
114
115         # Create a sorted list of test files so the subset chunk,
116         # if used, contains alphabetically consecutive tests.
117         if self._options.order == 'natural':
118             tests_to_run.sort(key=self._port.test_key)
119         elif self._options.order == 'random':
120             random.shuffle(tests_to_run)
121
122         tests_to_run, tests_in_other_chunks = self._finder.split_into_chunks(tests_to_run)
123         self._expectations.add_skipped_tests(tests_in_other_chunks)
124         tests_to_skip.update(tests_in_other_chunks)
125
126         return tests_to_run, tests_to_skip
127
128     def _test_input_for_file(self, test_file):
129         return TestInput(test_file,
130             self._options.slow_time_out_ms if self._test_is_slow(test_file) else self._options.time_out_ms,
131             self._is_http_test(test_file))
132
133     def _test_is_slow(self, test_file):
134         return self._expectations.model().has_modifier(test_file, test_expectations.SLOW)
135
136     def needs_servers(self, test_names):
137         return any(self._is_http_test(test_name) for test_name in test_names) and self._options.http
138
139     def _get_test_inputs(self, tests_to_run, repeat_each, iterations):
140         test_inputs = []
141         for _ in xrange(iterations):
142             for test in tests_to_run:
143                 for _ in xrange(repeat_each):
144                     test_inputs.append(self._test_input_for_file(test))
145         return test_inputs
146
147     def _update_worker_count(self, test_names):
148         test_inputs = self._get_test_inputs(test_names, self._options.repeat_each, self._options.iterations)
149         worker_count = self._runner.get_worker_count(test_inputs, int(self._options.child_processes))
150         self._options.child_processes = worker_count
151
152     def _set_up_run(self, test_names, device_class=None):
153         self._printer.write_update("Checking build ...")
154         if not self._port.check_build(self.needs_servers(test_names)):
155             _log.error("Build check failed")
156             return False
157
158         self._options.device_class = device_class
159
160         # This must be started before we check the system dependencies,
161         # since the helper may do things to make the setup correct.
162         self._printer.write_update("Starting helper ...")
163         if not self._port.start_helper(self._options.pixel_tests):
164             return False
165
166         self._update_worker_count(test_names)
167         self._port.reset_preferences()
168
169         # Check that the system dependencies (themes, fonts, ...) are correct.
170         if not self._options.nocheck_sys_deps:
171             self._printer.write_update("Checking system dependencies ...")
172             if not self._port.check_sys_deps(self.needs_servers(test_names)):
173                 self._port.stop_helper()
174                 return False
175
176         if self._options.clobber_old_results:
177             self._clobber_old_results()
178
179         # Create the output directory if it doesn't already exist.
180         self._port.host.filesystem.maybe_make_directory(self._results_directory)
181
182         self._port.setup_test_run(self._options.device_class)
183         return True
184
185     def run(self, args):
186         """Run the tests and return a RunDetails object with the results."""
187         self._printer.write_update("Collecting tests ...")
188         try:
189             paths, test_names = self._collect_tests(args)
190         except IOError:
191             # This is raised if --test-list doesn't exist
192             return test_run_results.RunDetails(exit_code=-1)
193
194         self._printer.write_update("Parsing expectations ...")
195         self._expectations = test_expectations.TestExpectations(self._port, test_names, force_expectations_pass=self._options.force)
196         self._expectations.parse_all_expectations()
197
198         tests_to_run, tests_to_skip = self._prepare_lists(paths, test_names)
199         self._printer.print_found(len(test_names), len(tests_to_run), self._options.repeat_each, self._options.iterations)
200         start_time = time.time()
201
202         # Check to make sure we're not skipping every test.
203         if not tests_to_run:
204             _log.critical('No tests to run.')
205             return test_run_results.RunDetails(exit_code=-1)
206
207         default_device_tests = []
208
209         # Look for tests with custom device requirements.
210         custom_device_tests = defaultdict(list)
211         for test_file in tests_to_run:
212             custom_device = self._custom_device_for_test(test_file)
213             if custom_device:
214                 custom_device_tests[custom_device].append(test_file)
215             else:
216                 default_device_tests.append(test_file)
217
218         if custom_device_tests:
219             for device_class in custom_device_tests:
220                 _log.debug('{} tests use device {}'.format(len(custom_device_tests[device_class]), device_class))
221
222         initial_results = None
223         retry_results = None
224         enabled_pixel_tests_in_retry = False
225
226         if default_device_tests:
227             _log.info('')
228             _log.info("Running %s", pluralize(len(tests_to_run), "test"))
229             _log.info('')
230             if not self._set_up_run(tests_to_run):
231                 return test_run_results.RunDetails(exit_code=-1)
232
233             initial_results, retry_results, enabled_pixel_tests_in_retry = self._run_test_subset(default_device_tests, tests_to_skip)
234
235         for device_class in custom_device_tests:
236             device_tests = custom_device_tests[device_class]
237             if device_tests:
238                 _log.info('')
239                 _log.info('Running %s for %s', pluralize(len(device_tests), "test"), device_class)
240                 _log.info('')
241                 if not self._set_up_run(device_tests, device_class):
242                     return test_run_results.RunDetails(exit_code=-1)
243
244                 device_initial_results, device_retry_results, device_enabled_pixel_tests_in_retry = self._run_test_subset(device_tests, tests_to_skip)
245
246                 initial_results = initial_results.merge(device_initial_results) if initial_results else device_initial_results
247                 retry_results = retry_results.merge(device_retry_results) if retry_results else device_retry_results
248                 enabled_pixel_tests_in_retry |= device_enabled_pixel_tests_in_retry
249
250         end_time = time.time()
251         return self._end_test_run(start_time, end_time, initial_results, retry_results, enabled_pixel_tests_in_retry)
252
253     def _run_test_subset(self, tests_to_run, tests_to_skip):
254         try:
255             enabled_pixel_tests_in_retry = False
256             initial_results = self._run_tests(tests_to_run, tests_to_skip, self._options.repeat_each, self._options.iterations, int(self._options.child_processes), retrying=False)
257
258             tests_to_retry = self._tests_to_retry(initial_results, include_crashes=self._port.should_retry_crashes())
259             # Don't retry failures when interrupted by user or failures limit exception.
260             retry_failures = self._options.retry_failures and not (initial_results.interrupted or initial_results.keyboard_interrupted)
261             if retry_failures and tests_to_retry:
262                 enabled_pixel_tests_in_retry = self._force_pixel_tests_if_needed()
263
264                 _log.info('')
265                 _log.info("Retrying %s ..." % pluralize(len(tests_to_retry), "unexpected failure"))
266                 _log.info('')
267                 retry_results = self._run_tests(tests_to_retry, tests_to_skip=set(), repeat_each=1, iterations=1, num_workers=1, retrying=True)
268
269                 if enabled_pixel_tests_in_retry:
270                     self._options.pixel_tests = False
271             else:
272                 retry_results = None
273         finally:
274             self._clean_up_run()
275
276         return (initial_results, retry_results, enabled_pixel_tests_in_retry)
277
278     def _end_test_run(self, start_time, end_time, initial_results, retry_results, enabled_pixel_tests_in_retry):
279         # Some crash logs can take a long time to be written out so look
280         # for new logs after the test run finishes.
281
282         _log.debug("looking for new crash logs")
283         self._look_for_new_crash_logs(initial_results, start_time)
284         if retry_results:
285             self._look_for_new_crash_logs(retry_results, start_time)
286
287         _log.debug("summarizing results")
288         summarized_results = test_run_results.summarize_results(self._port, self._expectations, initial_results, retry_results, enabled_pixel_tests_in_retry)
289         results_including_passes = None
290         if self._options.results_server_host:
291             results_including_passes = test_run_results.summarize_results(self._port, self._expectations, initial_results, retry_results, enabled_pixel_tests_in_retry, include_passes=True, include_time_and_modifiers=True)
292         self._printer.print_results(end_time - start_time, initial_results, summarized_results)
293
294         exit_code = -1
295         if not self._options.dry_run:
296             self._port.print_leaks_summary()
297             self._upload_json_files(summarized_results, initial_results, results_including_passes, start_time, end_time)
298
299             results_path = self._filesystem.join(self._results_directory, "results.html")
300             self._copy_results_html_file(results_path)
301             if initial_results.keyboard_interrupted:
302                 exit_code = INTERRUPTED_EXIT_STATUS
303             else:
304                 if self._options.show_results and (initial_results.unexpected_results_by_name or
305                     (self._options.full_results_html and initial_results.total_failures)):
306                     self._port.show_results_html_file(results_path)
307                 exit_code = self._port.exit_code_from_summarized_results(summarized_results)
308         return test_run_results.RunDetails(exit_code, summarized_results, initial_results, retry_results, enabled_pixel_tests_in_retry)
309
310     def _run_tests(self, tests_to_run, tests_to_skip, repeat_each, iterations, num_workers, retrying):
311         needs_http = any((self._is_http_test(test) and not self._is_web_platform_test(test)) for test in tests_to_run)
312         needs_web_platform_test_server = any(self._is_web_platform_test(test) for test in tests_to_run)
313         needs_websockets = any(self._is_websocket_test(test) for test in tests_to_run)
314
315         test_inputs = self._get_test_inputs(tests_to_run, repeat_each, iterations)
316
317         return self._runner.run_tests(self._expectations, test_inputs, tests_to_skip, num_workers, needs_http, needs_websockets, needs_web_platform_test_server, retrying)
318
319     def _clean_up_run(self):
320         _log.debug("Flushing stdout")
321         sys.stdout.flush()
322         _log.debug("Flushing stderr")
323         sys.stderr.flush()
324         _log.debug("Stopping helper")
325         self._port.stop_helper()
326         _log.debug("Cleaning up port")
327         self._port.clean_up_test_run()
328
329     def _force_pixel_tests_if_needed(self):
330         if self._options.pixel_tests:
331             return False
332
333         _log.debug("Restarting helper")
334         self._port.stop_helper()
335         self._options.pixel_tests = True
336         return self._port.start_helper()
337
338     def _look_for_new_crash_logs(self, run_results, start_time):
339         """Since crash logs can take a long time to be written out if the system is
340            under stress do a second pass at the end of the test run.
341
342            run_results: the results of the test run
343            start_time: time the tests started at.  We're looking for crash
344                logs after that time.
345         """
346         crashed_processes = []
347         for test, result in run_results.unexpected_results_by_name.iteritems():
348             if (result.type != test_expectations.CRASH):
349                 continue
350             for failure in result.failures:
351                 if not isinstance(failure, test_failures.FailureCrash):
352                     continue
353                 crashed_processes.append([test, failure.process_name, failure.pid])
354
355         sample_files = self._port.look_for_new_samples(crashed_processes, start_time)
356         if sample_files:
357             for test, sample_file in sample_files.iteritems():
358                 writer = TestResultWriter(self._port._filesystem, self._port, self._port.results_directory(), test)
359                 writer.copy_sample_file(sample_file)
360
361         crash_logs = self._port.look_for_new_crash_logs(crashed_processes, start_time)
362         if crash_logs:
363             for test, crash_log in crash_logs.iteritems():
364                 writer = TestResultWriter(self._port._filesystem, self._port, self._port.results_directory(), test)
365                 writer.write_crash_log(crash_log)
366
367                 # Check if this crashing 'test' is already in list of crashed_processes, if not add it to the run_results
368                 if not any(process[0] == test for process in crashed_processes):
369                     result = test_results.TestResult(test)
370                     result.type = test_expectations.CRASH
371                     result.is_other_crash = True
372                     run_results.add(result, expected=False, test_is_slow=False)
373                     _log.debug("Adding results for other crash: " + str(test))
374
375     def _clobber_old_results(self):
376         # Just clobber the actual test results directories since the other
377         # files in the results directory are explicitly used for cross-run
378         # tracking.
379         self._printer.write_update("Clobbering old results in %s" %
380                                    self._results_directory)
381         layout_tests_dir = self._port.layout_tests_dir()
382         possible_dirs = self._port.test_dirs()
383         for dirname in possible_dirs:
384             if self._filesystem.isdir(self._filesystem.join(layout_tests_dir, dirname)):
385                 self._filesystem.rmtree(self._filesystem.join(self._results_directory, dirname))
386
387     def _tests_to_retry(self, run_results, include_crashes):
388         return [result.test_name for result in run_results.unexpected_results_by_name.values() if
389                    ((result.type != test_expectations.PASS) and
390                     (result.type != test_expectations.MISSING) and
391                     (result.type != test_expectations.CRASH or include_crashes))]
392
393     def _upload_json_files(self, summarized_results, initial_results, results_including_passes=None, start_time=None, end_time=None):
394         """Writes the results of the test run as JSON files into the results
395         dir and upload the files to the appengine server.
396
397         Args:
398           summarized_results: dict of results
399           initial_results: full summary object
400         """
401         _log.debug("Writing JSON files in %s." % self._results_directory)
402
403         # FIXME: Upload stats.json to the server and delete times_ms.
404         times_trie = json_results_generator.test_timings_trie(self._port, initial_results.results_by_name.values())
405         times_json_path = self._filesystem.join(self._results_directory, "times_ms.json")
406         json_results_generator.write_json(self._filesystem, times_trie, times_json_path)
407
408         stats_trie = self._stats_trie(initial_results)
409         stats_path = self._filesystem.join(self._results_directory, "stats.json")
410         self._filesystem.write_text_file(stats_path, json.dumps(stats_trie))
411
412         full_results_path = self._filesystem.join(self._results_directory, "full_results.json")
413         # We write full_results.json out as jsonp because we need to load it from a file url and Chromium doesn't allow that.
414         json_results_generator.write_json(self._filesystem, summarized_results, full_results_path, callback="ADD_RESULTS")
415
416         results_json_path = self._filesystem.join(self._results_directory, "results_including_passes.json")
417         if results_including_passes:
418             json_results_generator.write_json(self._filesystem, results_including_passes, results_json_path)
419
420         generator = json_layout_results_generator.JSONLayoutResultsGenerator(
421             self._port, self._options.builder_name, self._options.build_name,
422             self._options.build_number, self._results_directory,
423             self._expectations, initial_results,
424             self._options.test_results_server,
425             "layout-tests",
426             self._options.master_name)
427
428         if generator.generate_json_output():
429             _log.debug("Finished writing JSON file for the test results server.")
430         else:
431             _log.debug("Failed to generate JSON file for the test results server.")
432             return
433
434         json_files = ["incremental_results.json", "full_results.json", "times_ms.json"]
435
436         generator.upload_json_files(json_files)
437         if results_including_passes:
438             self.upload_results(results_json_path, start_time, end_time)
439
440         incremental_results_path = self._filesystem.join(self._results_directory, "incremental_results.json")
441
442         # Remove these files from the results directory so they don't take up too much space on the buildbot.
443         # The tools use the version we uploaded to the results server anyway.
444         self._filesystem.remove(times_json_path)
445         self._filesystem.remove(incremental_results_path)
446         if results_including_passes:
447             self._filesystem.remove(results_json_path)
448
449     def upload_results(self, results_json_path, start_time, end_time):
450         hostname = self._options.results_server_host
451         if not hostname:
452             return
453         master_name = self._options.master_name
454         builder_name = self._options.builder_name
455         build_number = self._options.build_number
456         build_slave = self._options.build_slave
457         if not master_name or not builder_name or not build_number or not build_slave:
458             _log.error("--results-server-host was set, but --master-name, --builder-name, --build-number, or --build-slave was not. Not uploading JSON files.")
459             return
460
461         revisions = {}
462         # FIXME: This code is duplicated in PerfTestRunner._generate_results_dict
463         for (name, path) in self._port.repository_paths():
464             scm = SCMDetector(self._port.host.filesystem, self._port.host.executive).detect_scm_system(path) or self._port.host.scm()
465             revision = scm.svn_revision(path)
466             revisions[name] = {'revision': revision, 'timestamp': scm.timestamp_of_revision(path, revision)}
467
468         _log.info("Uploading JSON files for master: %s builder: %s build: %s slave: %s to %s", master_name, builder_name, build_number, build_slave, hostname)
469
470         attrs = [
471             ('master', 'build.webkit.org' if master_name == 'webkit.org' else master_name),  # FIXME: Pass in build.webkit.org.
472             ('builder_name', builder_name),
473             ('build_number', build_number),
474             ('build_slave', build_slave),
475             ('revisions', json.dumps(revisions)),
476             ('start_time', str(start_time)),
477             ('end_time', str(end_time)),
478         ]
479
480         uploader = FileUploader("http://%s/api/report" % hostname, 360)
481         try:
482             response = uploader.upload_as_multipart_form_data(self._filesystem, [('results.json', results_json_path)], attrs)
483             if not response:
484                 _log.error("JSON upload failed; no response returned")
485                 return
486
487             if response.code != 200:
488                 _log.error("JSON upload failed, %d: '%s'" % (response.code, response.read()))
489                 return
490
491             response_text = response.read()
492             try:
493                 response_json = json.loads(response_text)
494             except ValueError, error:
495                 _log.error("JSON upload failed; failed to parse the response: %s", response_text)
496                 return
497
498             if response_json['status'] != 'OK':
499                 _log.error("JSON upload failed, %s: %s", response_json['status'], response_text)
500                 return
501
502             _log.info("JSON uploaded.")
503         except Exception, error:
504             _log.error("Upload failed: %s" % error)
505             return
506
507     def _copy_results_html_file(self, destination_path):
508         base_dir = self._port.path_from_webkit_base('LayoutTests', 'fast', 'harness')
509         results_file = self._filesystem.join(base_dir, 'results.html')
510         # Note that the results.html template file won't exist when we're using a MockFileSystem during unit tests,
511         # so make sure it exists before we try to copy it.
512         if self._filesystem.exists(results_file):
513             self._filesystem.copyfile(results_file, destination_path)
514
515     def _stats_trie(self, initial_results):
516         def _worker_number(worker_name):
517             return int(worker_name.split('/')[1]) if worker_name else -1
518
519         stats = {}
520         for result in initial_results.results_by_name.values():
521             if result.type != test_expectations.SKIP:
522                 stats[result.test_name] = {'results': (_worker_number(result.worker_name), result.test_number, result.pid, int(result.test_run_time * 1000), int(result.total_run_time * 1000))}
523         stats_trie = {}
524         for name, value in stats.iteritems():
525             json_results_generator.add_path_to_trie(name, value, stats_trie)
526         return stats_trie