2010-11-17 Hayato Ito <hayato@chromium.org>
[WebKit-https.git] / WebKitTools / Scripts / webkitpy / layout_tests / run_webkit_tests.py
1 #!/usr/bin/env python
2 # Copyright (C) 2010 Google Inc. All rights reserved.
3 # Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Szeged
4 #
5 # Redistribution and use in source and binary forms, with or without
6 # modification, are permitted provided that the following conditions are
7 # met:
8 #
9 #     * Redistributions of source code must retain the above copyright
10 # notice, this list of conditions and the following disclaimer.
11 #     * Redistributions in binary form must reproduce the above
12 # copyright notice, this list of conditions and the following disclaimer
13 # in the documentation and/or other materials provided with the
14 # distribution.
15 #     * Neither the name of Google Inc. nor the names of its
16 # contributors may be used to endorse or promote products derived from
17 # this software without specific prior written permission.
18 #
19 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30
31 """Run layout tests.
32
33 This is a port of the existing webkit test script run-webkit-tests.
34
35 The TestRunner class runs a series of tests (TestType interface) against a set
36 of test files.  If a test file fails a TestType, it returns a list TestFailure
37 objects to the TestRunner.  The TestRunner then aggregates the TestFailures to
38 create a final report.
39
40 This script reads several files, if they exist in the test_lists subdirectory
41 next to this script itself.  Each should contain a list of paths to individual
42 tests or entire subdirectories of tests, relative to the outermost test
43 directory.  Entire lines starting with '//' (comments) will be ignored.
44
45 For details of the files' contents and purposes, see test_lists/README.
46 """
47
48 from __future__ import with_statement
49
50 import codecs
51 import errno
52 import glob
53 import logging
54 import math
55 import optparse
56 import os
57 import platform
58 import Queue
59 import random
60 import re
61 import shutil
62 import signal
63 import sys
64 import time
65 import traceback
66
67 from layout_package import dump_render_tree_thread
68 from layout_package import json_layout_results_generator
69 from layout_package import printing
70 from layout_package import test_expectations
71 from layout_package import test_failures
72 from layout_package import test_results
73 from layout_package import test_results_uploader
74 from test_types import image_diff
75 from test_types import text_diff
76 from test_types import test_type_base
77
78 from webkitpy.common.system import user
79 from webkitpy.thirdparty import simplejson
80
81 import port
82
83 _log = logging.getLogger("webkitpy.layout_tests.run_webkit_tests")
84
85 # Builder base URL where we have the archived test results.
86 BUILDER_BASE_URL = "http://build.chromium.org/buildbot/layout_test_results/"
87
88 LAYOUT_TESTS_DIRECTORY = "LayoutTests" + os.sep
89
90 TestExpectationsFile = test_expectations.TestExpectationsFile
91
92
93 class TestInput:
94     """Groups information about a test for easy passing of data."""
95
96     def __init__(self, filename, timeout):
97         """Holds the input parameters for a test.
98         Args:
99           filename: Full path to the test.
100           timeout: Timeout in msecs the driver should use while running the test
101           """
102         # FIXME: filename should really be test_name as a relative path.
103         self.filename = filename
104         self.timeout = timeout
105
106         # FIXME: Maybe the URI shouldn't be part of the TestInput at all?
107         self.uri = None
108
109
110 class ResultSummary(object):
111     """A class for partitioning the test results we get into buckets.
112
113     This class is basically a glorified struct and it's private to this file
114     so we don't bother with any information hiding."""
115
116     def __init__(self, expectations, test_files):
117         self.total = len(test_files)
118         self.remaining = self.total
119         self.expectations = expectations
120         self.expected = 0
121         self.unexpected = 0
122         self.tests_by_expectation = {}
123         self.tests_by_timeline = {}
124         self.results = {}
125         self.unexpected_results = {}
126         self.failures = {}
127         self.tests_by_expectation[test_expectations.SKIP] = set()
128         for expectation in TestExpectationsFile.EXPECTATIONS.values():
129             self.tests_by_expectation[expectation] = set()
130         for timeline in TestExpectationsFile.TIMELINES.values():
131             self.tests_by_timeline[timeline] = (
132                 expectations.get_tests_with_timeline(timeline))
133
134     def add(self, result, expected):
135         """Add a TestResult into the appropriate bin.
136
137         Args:
138           result: TestResult from dump_render_tree_thread.
139           expected: whether the result was what we expected it to be.
140         """
141
142         self.tests_by_expectation[result.type].add(result.filename)
143         self.results[result.filename] = result
144         self.remaining -= 1
145         if len(result.failures):
146             self.failures[result.filename] = result.failures
147         if expected:
148             self.expected += 1
149         else:
150             self.unexpected_results[result.filename] = result.type
151             self.unexpected += 1
152
153
154 def summarize_unexpected_results(port_obj, expectations, result_summary,
155                                  retry_summary):
156     """Summarize any unexpected results as a dict.
157
158     FIXME: split this data structure into a separate class?
159
160     Args:
161         port_obj: interface to port-specific hooks
162         expectations: test_expectations.TestExpectations object
163         result_summary: summary object from initial test runs
164         retry_summary: summary object from final test run of retried tests
165     Returns:
166         A dictionary containing a summary of the unexpected results from the
167         run, with the following fields:
168         'version': a version indicator (1 in this version)
169         'fixable': # of fixable tests (NOW - PASS)
170         'skipped': # of skipped tests (NOW & SKIPPED)
171         'num_regressions': # of non-flaky failures
172         'num_flaky': # of flaky failures
173         'num_passes': # of unexpected passes
174         'tests': a dict of tests -> {'expected': '...', 'actual': '...'}
175     """
176     results = {}
177     results['version'] = 1
178
179     tbe = result_summary.tests_by_expectation
180     tbt = result_summary.tests_by_timeline
181     results['fixable'] = len(tbt[test_expectations.NOW] -
182                                 tbe[test_expectations.PASS])
183     results['skipped'] = len(tbt[test_expectations.NOW] &
184                                 tbe[test_expectations.SKIP])
185
186     num_passes = 0
187     num_flaky = 0
188     num_regressions = 0
189     keywords = {}
190     for k, v in TestExpectationsFile.EXPECTATIONS.iteritems():
191         keywords[v] = k.upper()
192
193     tests = {}
194     for filename, result in result_summary.unexpected_results.iteritems():
195         # Note that if a test crashed in the original run, we ignore
196         # whether or not it crashed when we retried it (if we retried it),
197         # and always consider the result not flaky.
198         test = port_obj.relative_test_filename(filename)
199         expected = expectations.get_expectations_string(filename)
200         actual = [keywords[result]]
201
202         if result == test_expectations.PASS:
203             num_passes += 1
204         elif result == test_expectations.CRASH:
205             num_regressions += 1
206         else:
207             if filename not in retry_summary.unexpected_results:
208                 actual.extend(expectations.get_expectations_string(
209                     filename).split(" "))
210                 num_flaky += 1
211             else:
212                 retry_result = retry_summary.unexpected_results[filename]
213                 if result != retry_result:
214                     actual.append(keywords[retry_result])
215                     num_flaky += 1
216                 else:
217                     num_regressions += 1
218
219         tests[test] = {}
220         tests[test]['expected'] = expected
221         tests[test]['actual'] = " ".join(actual)
222
223     results['tests'] = tests
224     results['num_passes'] = num_passes
225     results['num_flaky'] = num_flaky
226     results['num_regressions'] = num_regressions
227
228     return results
229
230
231 class TestRunner:
232     """A class for managing running a series of tests on a series of layout
233     test files."""
234
235     HTTP_SUBDIR = os.sep.join(['', 'http', ''])
236     WEBSOCKET_SUBDIR = os.sep.join(['', 'websocket', ''])
237
238     # The per-test timeout in milliseconds, if no --time-out-ms option was
239     # given to run_webkit_tests. This should correspond to the default timeout
240     # in DumpRenderTree.
241     DEFAULT_TEST_TIMEOUT_MS = 6 * 1000
242
243     def __init__(self, port, options, printer):
244         """Initialize test runner data structures.
245
246         Args:
247           port: an object implementing port-specific
248           options: a dictionary of command line options
249           printer: a Printer object to record updates to.
250         """
251         self._port = port
252         self._options = options
253         self._printer = printer
254
255         # disable wss server. need to install pyOpenSSL on buildbots.
256         # self._websocket_secure_server = websocket_server.PyWebSocket(
257         #        options.results_directory, use_tls=True, port=9323)
258
259         # a list of TestType objects
260         self._test_types = [text_diff.TestTextDiff]
261         if options.pixel_tests:
262             self._test_types.append(image_diff.ImageDiff)
263
264         # a set of test files, and the same tests as a list
265         self._test_files = set()
266         self._test_files_list = None
267         self._result_queue = Queue.Queue()
268         self._retrying = False
269
270     def collect_tests(self, args, last_unexpected_results):
271         """Find all the files to test.
272
273         Args:
274           args: list of test arguments from the command line
275           last_unexpected_results: list of unexpected results to retest, if any
276
277         """
278         paths = [self._strip_test_dir_prefix(arg) for arg in args if arg and arg != '']
279         paths += last_unexpected_results
280         if self._options.test_list:
281             paths += read_test_files(self._options.test_list)
282         self._test_files = self._port.tests(paths)
283
284     def _strip_test_dir_prefix(self, path):
285         if path.startswith(LAYOUT_TESTS_DIRECTORY):
286             return path[len(LAYOUT_TESTS_DIRECTORY):]
287         return path
288
289     def lint(self):
290         # Creating the expecations for each platform/configuration pair does
291         # all the test list parsing and ensures it's correct syntax (e.g. no
292         # dupes).
293         for platform_name in self._port.test_platform_names():
294             self.parse_expectations(platform_name, is_debug_mode=True)
295             self.parse_expectations(platform_name, is_debug_mode=False)
296         self._printer.write("")
297         _log.info("If there are no fail messages, errors or exceptions, "
298                   "then the lint succeeded.")
299         return 0
300
301     def parse_expectations(self, test_platform_name, is_debug_mode):
302         """Parse the expectations from the test_list files and return a data
303         structure holding them. Throws an error if the test_list files have
304         invalid syntax."""
305         if self._options.lint_test_files:
306             test_files = None
307         else:
308             test_files = self._test_files
309
310         try:
311             expectations_str = self._port.test_expectations()
312             overrides_str = self._port.test_expectations_overrides()
313             self._expectations = test_expectations.TestExpectations(
314                 self._port, test_files, expectations_str, test_platform_name,
315                 is_debug_mode, self._options.lint_test_files,
316                 overrides=overrides_str)
317             return self._expectations
318         except SyntaxError, err:
319             if self._options.lint_test_files:
320                 print str(err)
321             else:
322                 raise err
323
324     def prepare_lists_and_print_output(self):
325         """Create appropriate subsets of test lists and returns a
326         ResultSummary object. Also prints expected test counts.
327         """
328
329         # Remove skipped - both fixable and ignored - files from the
330         # top-level list of files to test.
331         num_all_test_files = len(self._test_files)
332         self._printer.print_expected("Found:  %d tests" %
333                                      (len(self._test_files)))
334         if not num_all_test_files:
335             _log.critical('No tests to run.')
336             return None
337
338         skipped = set()
339         if num_all_test_files > 1 and not self._options.force:
340             skipped = self._expectations.get_tests_with_result_type(
341                            test_expectations.SKIP)
342             self._test_files -= skipped
343
344         # Create a sorted list of test files so the subset chunk,
345         # if used, contains alphabetically consecutive tests.
346         self._test_files_list = list(self._test_files)
347         if self._options.randomize_order:
348             random.shuffle(self._test_files_list)
349         else:
350             self._test_files_list.sort()
351
352         # If the user specifies they just want to run a subset of the tests,
353         # just grab a subset of the non-skipped tests.
354         if self._options.run_chunk or self._options.run_part:
355             chunk_value = self._options.run_chunk or self._options.run_part
356             test_files = self._test_files_list
357             try:
358                 (chunk_num, chunk_len) = chunk_value.split(":")
359                 chunk_num = int(chunk_num)
360                 assert(chunk_num >= 0)
361                 test_size = int(chunk_len)
362                 assert(test_size > 0)
363             except:
364                 _log.critical("invalid chunk '%s'" % chunk_value)
365                 return None
366
367             # Get the number of tests
368             num_tests = len(test_files)
369
370             # Get the start offset of the slice.
371             if self._options.run_chunk:
372                 chunk_len = test_size
373                 # In this case chunk_num can be really large. We need
374                 # to make the slave fit in the current number of tests.
375                 slice_start = (chunk_num * chunk_len) % num_tests
376             else:
377                 # Validate the data.
378                 assert(test_size <= num_tests)
379                 assert(chunk_num <= test_size)
380
381                 # To count the chunk_len, and make sure we don't skip
382                 # some tests, we round to the next value that fits exactly
383                 # all the parts.
384                 rounded_tests = num_tests
385                 if rounded_tests % test_size != 0:
386                     rounded_tests = (num_tests + test_size -
387                                      (num_tests % test_size))
388
389                 chunk_len = rounded_tests / test_size
390                 slice_start = chunk_len * (chunk_num - 1)
391                 # It does not mind if we go over test_size.
392
393             # Get the end offset of the slice.
394             slice_end = min(num_tests, slice_start + chunk_len)
395
396             files = test_files[slice_start:slice_end]
397
398             tests_run_msg = 'Running: %d tests (chunk slice [%d:%d] of %d)' % (
399                 (slice_end - slice_start), slice_start, slice_end, num_tests)
400             self._printer.print_expected(tests_run_msg)
401
402             # If we reached the end and we don't have enough tests, we run some
403             # from the beginning.
404             if slice_end - slice_start < chunk_len:
405                 extra = chunk_len - (slice_end - slice_start)
406                 extra_msg = ('   last chunk is partial, appending [0:%d]' %
407                             extra)
408                 self._printer.print_expected(extra_msg)
409                 tests_run_msg += "\n" + extra_msg
410                 files.extend(test_files[0:extra])
411             tests_run_filename = os.path.join(self._options.results_directory,
412                                               "tests_run.txt")
413             with codecs.open(tests_run_filename, "w", "utf-8") as file:
414                 file.write(tests_run_msg + "\n")
415
416             len_skip_chunk = int(len(files) * len(skipped) /
417                                  float(len(self._test_files)))
418             skip_chunk_list = list(skipped)[0:len_skip_chunk]
419             skip_chunk = set(skip_chunk_list)
420
421             # Update expectations so that the stats are calculated correctly.
422             # We need to pass a list that includes the right # of skipped files
423             # to ParseExpectations so that ResultSummary() will get the correct
424             # stats. So, we add in the subset of skipped files, and then
425             # subtract them back out.
426             self._test_files_list = files + skip_chunk_list
427             self._test_files = set(self._test_files_list)
428
429             self._expectations = self.parse_expectations(
430                 self._port.test_platform_name(),
431                 self._options.configuration == 'Debug')
432
433             self._test_files = set(files)
434             self._test_files_list = files
435         else:
436             skip_chunk = skipped
437
438         result_summary = ResultSummary(self._expectations,
439             self._test_files | skip_chunk)
440         self._print_expected_results_of_type(result_summary,
441             test_expectations.PASS, "passes")
442         self._print_expected_results_of_type(result_summary,
443             test_expectations.FAIL, "failures")
444         self._print_expected_results_of_type(result_summary,
445             test_expectations.FLAKY, "flaky")
446         self._print_expected_results_of_type(result_summary,
447             test_expectations.SKIP, "skipped")
448
449         if self._options.force:
450             self._printer.print_expected('Running all tests, including '
451                                          'skips (--force)')
452         else:
453             # Note that we don't actually run the skipped tests (they were
454             # subtracted out of self._test_files, above), but we stub out the
455             # results here so the statistics can remain accurate.
456             for test in skip_chunk:
457                 result = test_results.TestResult(test,
458                     failures=[], test_run_time=0, total_time_for_all_diffs=0,
459                     time_for_diffs=0)
460                 result.type = test_expectations.SKIP
461                 result_summary.add(result, expected=True)
462         self._printer.print_expected('')
463
464         return result_summary
465
466     def _get_dir_for_test_file(self, test_file):
467         """Returns the highest-level directory by which to shard the given
468         test file."""
469         index = test_file.rfind(os.sep + LAYOUT_TESTS_DIRECTORY)
470
471         test_file = test_file[index + len(LAYOUT_TESTS_DIRECTORY):]
472         test_file_parts = test_file.split(os.sep, 1)
473         directory = test_file_parts[0]
474         test_file = test_file_parts[1]
475
476         # The http tests are very stable on mac/linux.
477         # TODO(ojan): Make the http server on Windows be apache so we can
478         # turn shard the http tests there as well. Switching to apache is
479         # what made them stable on linux/mac.
480         return_value = directory
481         while ((directory != 'http' or sys.platform in ('darwin', 'linux2'))
482                 and test_file.find(os.sep) >= 0):
483             test_file_parts = test_file.split(os.sep, 1)
484             directory = test_file_parts[0]
485             return_value = os.path.join(return_value, directory)
486             test_file = test_file_parts[1]
487
488         return return_value
489
490     def _get_test_input_for_file(self, test_file):
491         """Returns the appropriate TestInput object for the file. Mostly this
492         is used for looking up the timeout value (in ms) to use for the given
493         test."""
494         if self._expectations.has_modifier(test_file, test_expectations.SLOW):
495             return TestInput(test_file, self._options.slow_time_out_ms)
496         return TestInput(test_file, self._options.time_out_ms)
497
498     def _test_requires_lock(self, test_file):
499         """Return True if the test needs to be locked when
500         running multiple copies of NRWTs."""
501         split_path = test_file.split(os.sep)
502         return 'http' in split_path or 'websocket' in split_path
503
504     def _get_test_file_queue(self, test_files):
505         """Create the thread safe queue of lists of (test filenames, test URIs)
506         tuples. Each TestShellThread pulls a list from this queue and runs
507         those tests in order before grabbing the next available list.
508
509         Shard the lists by directory. This helps ensure that tests that depend
510         on each other (aka bad tests!) continue to run together as most
511         cross-tests dependencies tend to occur within the same directory.
512
513         Return:
514           The Queue of lists of TestInput objects.
515         """
516
517         test_lists = []
518         tests_to_http_lock = []
519         if (self._options.experimental_fully_parallel or
520             self._is_single_threaded()):
521             for test_file in test_files:
522                 test_input = self._get_test_input_for_file(test_file)
523                 if self._test_requires_lock(test_file):
524                     tests_to_http_lock.append(test_input)
525                 else:
526                     test_lists.append((".", [test_input]))
527         else:
528             tests_by_dir = {}
529             for test_file in test_files:
530                 directory = self._get_dir_for_test_file(test_file)
531                 test_input = self._get_test_input_for_file(test_file)
532                 if self._test_requires_lock(test_file):
533                     tests_to_http_lock.append(test_input)
534                 else:
535                     tests_by_dir.setdefault(directory, [])
536                     tests_by_dir[directory].append(test_input)
537             # Sort by the number of tests in the dir so that the ones with the
538             # most tests get run first in order to maximize parallelization.
539             # Number of tests is a good enough, but not perfect, approximation
540             # of how long that set of tests will take to run. We can't just use
541             # a PriorityQueue until we move to Python 2.6.
542             for directory in tests_by_dir:
543                 test_list = tests_by_dir[directory]
544                 # Keep the tests in alphabetical order.
545                 # FIXME: Remove once tests are fixed so they can be run in any
546                 # order.
547                 test_list.reverse()
548                 test_list_tuple = (directory, test_list)
549                 test_lists.append(test_list_tuple)
550             test_lists.sort(lambda a, b: cmp(len(b[1]), len(a[1])))
551
552         # Put the http tests first. There are only a couple hundred of them,
553         # but each http test takes a very long time to run, so sorting by the
554         # number of tests doesn't accurately capture how long they take to run.
555         if tests_to_http_lock:
556             tests_to_http_lock.reverse()
557             test_lists.insert(0, ("tests_to_http_lock", tests_to_http_lock))
558
559         filename_queue = Queue.Queue()
560         for item in test_lists:
561             filename_queue.put(item)
562         return filename_queue
563
564     def _get_test_args(self, index):
565         """Returns the tuple of arguments for tests and for DumpRenderTree."""
566         test_args = test_type_base.TestArguments()
567         test_args.png_path = None
568         if self._options.pixel_tests:
569             png_path = os.path.join(self._options.results_directory,
570                                     "png_result%s.png" % index)
571             test_args.png_path = png_path
572         test_args.new_baseline = self._options.new_baseline
573         test_args.reset_results = self._options.reset_results
574
575         return test_args
576
577     def _contains_tests(self, subdir):
578         for test_file in self._test_files:
579             if test_file.find(subdir) >= 0:
580                 return True
581         return False
582
583     def _instantiate_dump_render_tree_threads(self, test_files,
584                                               result_summary):
585         """Instantitates and starts the TestShellThread(s).
586
587         Return:
588           The list of threads.
589         """
590         filename_queue = self._get_test_file_queue(test_files)
591
592         # Instantiate TestShellThreads and start them.
593         threads = []
594         for i in xrange(int(self._options.child_processes)):
595             # Create separate TestTypes instances for each thread.
596             test_types = []
597             for test_type in self._test_types:
598                 test_types.append(test_type(self._port,
599                                     self._options.results_directory))
600
601             test_args = self._get_test_args(i)
602             thread = dump_render_tree_thread.TestShellThread(self._port,
603                 self._options, filename_queue, self._result_queue,
604                 test_types, test_args)
605             if self._is_single_threaded():
606                 thread.run_in_main_thread(self, result_summary)
607             else:
608                 thread.start()
609             threads.append(thread)
610
611         return threads
612
613     def _is_single_threaded(self):
614         """Returns whether we should run all the tests in the main thread."""
615         return int(self._options.child_processes) == 1
616
617     def _run_tests(self, file_list, result_summary):
618         """Runs the tests in the file_list.
619
620         Return: A tuple (keyboard_interrupted, thread_timings, test_timings,
621             individual_test_timings)
622             keyboard_interrupted is whether someone typed Ctrl^C
623             thread_timings is a list of dicts with the total runtime
624               of each thread with 'name', 'num_tests', 'total_time' properties
625             test_timings is a list of timings for each sharded subdirectory
626               of the form [time, directory_name, num_tests]
627             individual_test_timings is a list of run times for each test
628               in the form {filename:filename, test_run_time:test_run_time}
629             result_summary: summary object to populate with the results
630         """
631         # FIXME: We should use webkitpy.tool.grammar.pluralize here.
632         plural = ""
633         if not self._is_single_threaded():
634             plural = "s"
635         self._printer.print_update('Starting %s%s ...' %
636                                    (self._port.driver_name(), plural))
637         threads = self._instantiate_dump_render_tree_threads(file_list,
638                                                              result_summary)
639         self._printer.print_update("Starting testing ...")
640
641         keyboard_interrupted = self._wait_for_threads_to_finish(threads,
642                                                                 result_summary)
643         (thread_timings, test_timings, individual_test_timings) = \
644             self._collect_timing_info(threads)
645
646         return (keyboard_interrupted, thread_timings, test_timings,
647                 individual_test_timings)
648
649     def _wait_for_threads_to_finish(self, threads, result_summary):
650         keyboard_interrupted = False
651         try:
652             # Loop through all the threads waiting for them to finish.
653             some_thread_is_alive = True
654             while some_thread_is_alive:
655                 some_thread_is_alive = False
656                 t = time.time()
657                 for thread in threads:
658                     exception_info = thread.exception_info()
659                     if exception_info is not None:
660                         # Re-raise the thread's exception here to make it
661                         # clear that testing was aborted. Otherwise,
662                         # the tests that did not run would be assumed
663                         # to have passed.
664                         raise exception_info[0], exception_info[1], exception_info[2]
665
666                     if thread.isAlive():
667                         some_thread_is_alive = True
668                         next_timeout = thread.next_timeout()
669                         if (next_timeout and t > next_timeout):
670                             _log_wedged_thread(thread)
671                             thread.clear_next_timeout()
672
673                 self.update_summary(result_summary)
674
675                 if some_thread_is_alive:
676                     time.sleep(0.01)
677
678         except KeyboardInterrupt:
679             keyboard_interrupted = True
680             for thread in threads:
681                 thread.cancel()
682
683         return keyboard_interrupted
684
685     def _collect_timing_info(self, threads):
686         test_timings = {}
687         individual_test_timings = []
688         thread_timings = []
689
690         for thread in threads:
691             thread_timings.append({'name': thread.getName(),
692                                    'num_tests': thread.get_num_tests(),
693                                    'total_time': thread.get_total_time()})
694             test_timings.update(thread.get_test_group_timing_stats())
695             individual_test_timings.extend(thread.get_test_results())
696
697         return (thread_timings, test_timings, individual_test_timings)
698
699     def needs_http(self):
700         """Returns whether the test runner needs an HTTP server."""
701         return self._contains_tests(self.HTTP_SUBDIR)
702
703     def needs_websocket(self):
704         """Returns whether the test runner needs a WEBSOCKET server."""
705         return self._contains_tests(self.WEBSOCKET_SUBDIR)
706
707     def set_up_run(self):
708         """Configures the system to be ready to run tests.
709
710         Returns a ResultSummary object if we should continue to run tests,
711         or None if we should abort.
712
713         """
714         # This must be started before we check the system dependencies,
715         # since the helper may do things to make the setup correct.
716         self._printer.print_update("Starting helper ...")
717         self._port.start_helper()
718
719         # Check that the system dependencies (themes, fonts, ...) are correct.
720         if not self._options.nocheck_sys_deps:
721             self._printer.print_update("Checking system dependencies ...")
722             if not self._port.check_sys_deps(self.needs_http()):
723                 self._port.stop_helper()
724                 return None
725
726         if self._options.clobber_old_results:
727             self._clobber_old_results()
728
729         # Create the output directory if it doesn't already exist.
730         self._port.maybe_make_directory(self._options.results_directory)
731
732         self._port.setup_test_run()
733
734         self._printer.print_update("Preparing tests ...")
735         result_summary = self.prepare_lists_and_print_output()
736         if not result_summary:
737             return None
738
739         return result_summary
740
741     def run(self, result_summary):
742         """Run all our tests on all our test files.
743
744         For each test file, we run each test type. If there are any failures,
745         we collect them for reporting.
746
747         Args:
748           result_summary: a summary object tracking the test results.
749
750         Return:
751           The number of unexpected results (0 == success)
752         """
753         # gather_test_files() must have been called first to initialize us.
754         # If we didn't find any files to test, we've errored out already in
755         # prepare_lists_and_print_output().
756         assert(len(self._test_files))
757
758         start_time = time.time()
759
760         keyboard_interrupted, thread_timings, test_timings, \
761             individual_test_timings = (
762             self._run_tests(self._test_files_list, result_summary))
763
764         # We exclude the crashes from the list of results to retry, because
765         # we want to treat even a potentially flaky crash as an error.
766         failures = self._get_failures(result_summary, include_crashes=False)
767         retry_summary = result_summary
768         while (len(failures) and self._options.retry_failures and
769             not self._retrying and not keyboard_interrupted):
770             _log.info('')
771             _log.info("Retrying %d unexpected failure(s) ..." % len(failures))
772             _log.info('')
773             self._retrying = True
774             retry_summary = ResultSummary(self._expectations, failures.keys())
775             # Note that we intentionally ignore the return value here.
776             self._run_tests(failures.keys(), retry_summary)
777             failures = self._get_failures(retry_summary, include_crashes=True)
778
779         end_time = time.time()
780
781         self._print_timing_statistics(end_time - start_time,
782                                       thread_timings, test_timings,
783                                       individual_test_timings,
784                                       result_summary)
785
786         self._print_result_summary(result_summary)
787
788         sys.stdout.flush()
789         sys.stderr.flush()
790
791         self._printer.print_one_line_summary(result_summary.total,
792                                              result_summary.expected,
793                                              result_summary.unexpected)
794
795         unexpected_results = summarize_unexpected_results(self._port,
796             self._expectations, result_summary, retry_summary)
797         self._printer.print_unexpected_results(unexpected_results)
798
799         if self._options.record_results:
800             # Write the same data to log files and upload generated JSON files
801             # to appengine server.
802             self._upload_json_files(unexpected_results, result_summary,
803                                     individual_test_timings)
804
805         # Write the summary to disk (results.html) and display it if requested.
806         wrote_results = self._write_results_html_file(result_summary)
807         if self._options.show_results and wrote_results:
808             self._show_results_html_file()
809
810         # Now that we've completed all the processing we can, we re-raise
811         # a KeyboardInterrupt if necessary so the caller can handle it.
812         if keyboard_interrupted:
813             raise KeyboardInterrupt
814
815         # Ignore flaky failures and unexpected passes so we don't turn the
816         # bot red for those.
817         return unexpected_results['num_regressions']
818
819     def clean_up_run(self):
820         """Restores the system after we're done running tests."""
821
822         _log.debug("flushing stdout")
823         sys.stdout.flush()
824         _log.debug("flushing stderr")
825         sys.stderr.flush()
826         _log.debug("stopping helper")
827         self._port.stop_helper()
828
829     def update_summary(self, result_summary):
830         """Update the summary and print results with any completed tests."""
831         while True:
832             try:
833                 result = test_results.TestResult.loads(self._result_queue.get_nowait())
834             except Queue.Empty:
835                 return
836
837             expected = self._expectations.matches_an_expected_result(
838                 result.filename, result.type, self._options.pixel_tests)
839             result_summary.add(result, expected)
840             exp_str = self._expectations.get_expectations_string(
841                 result.filename)
842             got_str = self._expectations.expectation_to_string(result.type)
843             self._printer.print_test_result(result, expected, exp_str, got_str)
844             self._printer.print_progress(result_summary, self._retrying,
845                                          self._test_files_list)
846
847     def _clobber_old_results(self):
848         # Just clobber the actual test results directories since the other
849         # files in the results directory are explicitly used for cross-run
850         # tracking.
851         self._printer.print_update("Clobbering old results in %s" %
852                                    self._options.results_directory)
853         layout_tests_dir = self._port.layout_tests_dir()
854         possible_dirs = self._port.test_dirs()
855         for dirname in possible_dirs:
856             if os.path.isdir(os.path.join(layout_tests_dir, dirname)):
857                 shutil.rmtree(os.path.join(self._options.results_directory,
858                                            dirname),
859                               ignore_errors=True)
860
861     def _get_failures(self, result_summary, include_crashes):
862         """Filters a dict of results and returns only the failures.
863
864         Args:
865           result_summary: the results of the test run
866           include_crashes: whether crashes are included in the output.
867             We use False when finding the list of failures to retry
868             to see if the results were flaky. Although the crashes may also be
869             flaky, we treat them as if they aren't so that they're not ignored.
870         Returns:
871           a dict of files -> results
872         """
873         failed_results = {}
874         for test, result in result_summary.unexpected_results.iteritems():
875             if (result == test_expectations.PASS or
876                 result == test_expectations.CRASH and not include_crashes):
877                 continue
878             failed_results[test] = result
879
880         return failed_results
881
882     def _upload_json_files(self, unexpected_results, result_summary,
883                         individual_test_timings):
884         """Writes the results of the test run as JSON files into the results
885         dir and upload the files to the appengine server.
886
887         There are three different files written into the results dir:
888           unexpected_results.json: A short list of any unexpected results.
889             This is used by the buildbots to display results.
890           expectations.json: This is used by the flakiness dashboard.
891           results.json: A full list of the results - used by the flakiness
892             dashboard and the aggregate results dashboard.
893
894         Args:
895           unexpected_results: dict of unexpected results
896           result_summary: full summary object
897           individual_test_timings: list of test times (used by the flakiness
898             dashboard).
899         """
900         results_directory = self._options.results_directory
901         _log.debug("Writing JSON files in %s." % results_directory)
902         unexpected_json_path = os.path.join(results_directory, "unexpected_results.json")
903         with codecs.open(unexpected_json_path, "w", "utf-8") as file:
904             simplejson.dump(unexpected_results, file, sort_keys=True, indent=2)
905
906         # Write a json file of the test_expectations.txt file for the layout
907         # tests dashboard.
908         expectations_path = os.path.join(results_directory, "expectations.json")
909         expectations_json = \
910             self._expectations.get_expectations_json_for_all_platforms()
911         with codecs.open(expectations_path, "w", "utf-8") as file:
912             file.write(u"ADD_EXPECTATIONS(%s);" % expectations_json)
913
914         generator = json_layout_results_generator.JSONLayoutResultsGenerator(
915             self._port, self._options.builder_name, self._options.build_name,
916             self._options.build_number, self._options.results_directory,
917             BUILDER_BASE_URL, individual_test_timings,
918             self._expectations, result_summary, self._test_files_list,
919             not self._options.upload_full_results,
920             self._options.test_results_server,
921             "layout-tests",
922             self._options.master_name)
923
924         _log.debug("Finished writing JSON files.")
925
926         json_files = ["expectations.json"]
927         if self._options.upload_full_results:
928             json_files.append("results.json")
929         else:
930             json_files.append("incremental_results.json")
931
932         generator.upload_json_files(json_files)
933
934     def _print_config(self):
935         """Prints the configuration for the test run."""
936         p = self._printer
937         p.print_config("Using port '%s'" % self._port.name())
938         p.print_config("Placing test results in %s" %
939                        self._options.results_directory)
940         if self._options.new_baseline:
941             p.print_config("Placing new baselines in %s" %
942                            self._port.baseline_path())
943         p.print_config("Using %s build" % self._options.configuration)
944         if self._options.pixel_tests:
945             p.print_config("Pixel tests enabled")
946         else:
947             p.print_config("Pixel tests disabled")
948
949         p.print_config("Regular timeout: %s, slow test timeout: %s" %
950                        (self._options.time_out_ms,
951                         self._options.slow_time_out_ms))
952
953         if self._is_single_threaded():
954             p.print_config("Running one %s" % self._port.driver_name())
955         else:
956             p.print_config("Running %s %ss in parallel" %
957                            (self._options.child_processes,
958                             self._port.driver_name()))
959         p.print_config("")
960
961     def _print_expected_results_of_type(self, result_summary,
962                                         result_type, result_type_str):
963         """Print the number of the tests in a given result class.
964
965         Args:
966           result_summary - the object containing all the results to report on
967           result_type - the particular result type to report in the summary.
968           result_type_str - a string description of the result_type.
969         """
970         tests = self._expectations.get_tests_with_result_type(result_type)
971         now = result_summary.tests_by_timeline[test_expectations.NOW]
972         wontfix = result_summary.tests_by_timeline[test_expectations.WONTFIX]
973
974         # We use a fancy format string in order to print the data out in a
975         # nicely-aligned table.
976         fmtstr = ("Expect: %%5d %%-8s (%%%dd now, %%%dd wontfix)"
977                   % (self._num_digits(now), self._num_digits(wontfix)))
978         self._printer.print_expected(fmtstr %
979             (len(tests), result_type_str, len(tests & now), len(tests & wontfix)))
980
981     def _num_digits(self, num):
982         """Returns the number of digits needed to represent the length of a
983         sequence."""
984         ndigits = 1
985         if len(num):
986             ndigits = int(math.log10(len(num))) + 1
987         return ndigits
988
989     def _print_timing_statistics(self, total_time, thread_timings,
990                                directory_test_timings, individual_test_timings,
991                                result_summary):
992         """Record timing-specific information for the test run.
993
994         Args:
995           total_time: total elapsed time (in seconds) for the test run
996           thread_timings: wall clock time each thread ran for
997           directory_test_timings: timing by directory
998           individual_test_timings: timing by file
999           result_summary: summary object for the test run
1000         """
1001         self._printer.print_timing("Test timing:")
1002         self._printer.print_timing("  %6.2f total testing time" % total_time)
1003         self._printer.print_timing("")
1004         self._printer.print_timing("Thread timing:")
1005         cuml_time = 0
1006         for t in thread_timings:
1007             self._printer.print_timing("    %10s: %5d tests, %6.2f secs" %
1008                   (t['name'], t['num_tests'], t['total_time']))
1009             cuml_time += t['total_time']
1010         self._printer.print_timing("   %6.2f cumulative, %6.2f optimal" %
1011               (cuml_time, cuml_time / int(self._options.child_processes)))
1012         self._printer.print_timing("")
1013
1014         self._print_aggregate_test_statistics(individual_test_timings)
1015         self._print_individual_test_times(individual_test_timings,
1016                                           result_summary)
1017         self._print_directory_timings(directory_test_timings)
1018
1019     def _print_aggregate_test_statistics(self, individual_test_timings):
1020         """Prints aggregate statistics (e.g. median, mean, etc.) for all tests.
1021         Args:
1022           individual_test_timings: List of dump_render_tree_thread.TestStats
1023               for all tests.
1024         """
1025         test_types = []  # Unit tests don't actually produce any timings.
1026         if individual_test_timings:
1027             test_types = individual_test_timings[0].time_for_diffs.keys()
1028         times_for_dump_render_tree = []
1029         times_for_diff_processing = []
1030         times_per_test_type = {}
1031         for test_type in test_types:
1032             times_per_test_type[test_type] = []
1033
1034         for test_stats in individual_test_timings:
1035             times_for_dump_render_tree.append(test_stats.test_run_time)
1036             times_for_diff_processing.append(
1037                 test_stats.total_time_for_all_diffs)
1038             time_for_diffs = test_stats.time_for_diffs
1039             for test_type in test_types:
1040                 times_per_test_type[test_type].append(
1041                     time_for_diffs[test_type])
1042
1043         self._print_statistics_for_test_timings(
1044             "PER TEST TIME IN TESTSHELL (seconds):",
1045             times_for_dump_render_tree)
1046         self._print_statistics_for_test_timings(
1047             "PER TEST DIFF PROCESSING TIMES (seconds):",
1048             times_for_diff_processing)
1049         for test_type in test_types:
1050             self._print_statistics_for_test_timings(
1051                 "PER TEST TIMES BY TEST TYPE: %s" % test_type,
1052                 times_per_test_type[test_type])
1053
1054     def _print_individual_test_times(self, individual_test_timings,
1055                                   result_summary):
1056         """Prints the run times for slow, timeout and crash tests.
1057         Args:
1058           individual_test_timings: List of dump_render_tree_thread.TestStats
1059               for all tests.
1060           result_summary: summary object for test run
1061         """
1062         # Reverse-sort by the time spent in DumpRenderTree.
1063         individual_test_timings.sort(lambda a, b:
1064             cmp(b.test_run_time, a.test_run_time))
1065
1066         num_printed = 0
1067         slow_tests = []
1068         timeout_or_crash_tests = []
1069         unexpected_slow_tests = []
1070         for test_tuple in individual_test_timings:
1071             filename = test_tuple.filename
1072             is_timeout_crash_or_slow = False
1073             if self._expectations.has_modifier(filename,
1074                                                test_expectations.SLOW):
1075                 is_timeout_crash_or_slow = True
1076                 slow_tests.append(test_tuple)
1077
1078             if filename in result_summary.failures:
1079                 result = result_summary.results[filename].type
1080                 if (result == test_expectations.TIMEOUT or
1081                     result == test_expectations.CRASH):
1082                     is_timeout_crash_or_slow = True
1083                     timeout_or_crash_tests.append(test_tuple)
1084
1085             if (not is_timeout_crash_or_slow and
1086                 num_printed < printing.NUM_SLOW_TESTS_TO_LOG):
1087                 num_printed = num_printed + 1
1088                 unexpected_slow_tests.append(test_tuple)
1089
1090         self._printer.print_timing("")
1091         self._print_test_list_timing("%s slowest tests that are not "
1092             "marked as SLOW and did not timeout/crash:" %
1093             printing.NUM_SLOW_TESTS_TO_LOG, unexpected_slow_tests)
1094         self._printer.print_timing("")
1095         self._print_test_list_timing("Tests marked as SLOW:", slow_tests)
1096         self._printer.print_timing("")
1097         self._print_test_list_timing("Tests that timed out or crashed:",
1098                                      timeout_or_crash_tests)
1099         self._printer.print_timing("")
1100
1101     def _print_test_list_timing(self, title, test_list):
1102         """Print timing info for each test.
1103
1104         Args:
1105           title: section heading
1106           test_list: tests that fall in this section
1107         """
1108         if self._printer.disabled('slowest'):
1109             return
1110
1111         self._printer.print_timing(title)
1112         for test_tuple in test_list:
1113             filename = test_tuple.filename[len(
1114                 self._port.layout_tests_dir()) + 1:]
1115             filename = filename.replace('\\', '/')
1116             test_run_time = round(test_tuple.test_run_time, 1)
1117             self._printer.print_timing("  %s took %s seconds" %
1118                                        (filename, test_run_time))
1119
1120     def _print_directory_timings(self, directory_test_timings):
1121         """Print timing info by directory for any directories that
1122         take > 10 seconds to run.
1123
1124         Args:
1125           directory_test_timing: time info for each directory
1126         """
1127         timings = []
1128         for directory in directory_test_timings:
1129             num_tests, time_for_directory = directory_test_timings[directory]
1130             timings.append((round(time_for_directory, 1), directory,
1131                             num_tests))
1132         timings.sort()
1133
1134         self._printer.print_timing("Time to process slowest subdirectories:")
1135         min_seconds_to_print = 10
1136         for timing in timings:
1137             if timing[0] > min_seconds_to_print:
1138                 self._printer.print_timing(
1139                     "  %s took %s seconds to run %s tests." % (timing[1],
1140                     timing[0], timing[2]))
1141         self._printer.print_timing("")
1142
1143     def _print_statistics_for_test_timings(self, title, timings):
1144         """Prints the median, mean and standard deviation of the values in
1145         timings.
1146
1147         Args:
1148           title: Title for these timings.
1149           timings: A list of floats representing times.
1150         """
1151         self._printer.print_timing(title)
1152         timings.sort()
1153
1154         num_tests = len(timings)
1155         if not num_tests:
1156             return
1157         percentile90 = timings[int(.9 * num_tests)]
1158         percentile99 = timings[int(.99 * num_tests)]
1159
1160         if num_tests % 2 == 1:
1161             median = timings[((num_tests - 1) / 2) - 1]
1162         else:
1163             lower = timings[num_tests / 2 - 1]
1164             upper = timings[num_tests / 2]
1165             median = (float(lower + upper)) / 2
1166
1167         mean = sum(timings) / num_tests
1168
1169         for time in timings:
1170             sum_of_deviations = math.pow(time - mean, 2)
1171
1172         std_deviation = math.sqrt(sum_of_deviations / num_tests)
1173         self._printer.print_timing("  Median:          %6.3f" % median)
1174         self._printer.print_timing("  Mean:            %6.3f" % mean)
1175         self._printer.print_timing("  90th percentile: %6.3f" % percentile90)
1176         self._printer.print_timing("  99th percentile: %6.3f" % percentile99)
1177         self._printer.print_timing("  Standard dev:    %6.3f" % std_deviation)
1178         self._printer.print_timing("")
1179
1180     def _print_result_summary(self, result_summary):
1181         """Print a short summary about how many tests passed.
1182
1183         Args:
1184           result_summary: information to log
1185         """
1186         failed = len(result_summary.failures)
1187         skipped = len(
1188             result_summary.tests_by_expectation[test_expectations.SKIP])
1189         total = result_summary.total
1190         passed = total - failed - skipped
1191         pct_passed = 0.0
1192         if total > 0:
1193             pct_passed = float(passed) * 100 / total
1194
1195         self._printer.print_actual("")
1196         self._printer.print_actual("=> Results: %d/%d tests passed (%.1f%%)" %
1197                      (passed, total, pct_passed))
1198         self._printer.print_actual("")
1199         self._print_result_summary_entry(result_summary,
1200             test_expectations.NOW, "Tests to be fixed")
1201
1202         self._printer.print_actual("")
1203         self._print_result_summary_entry(result_summary,
1204             test_expectations.WONTFIX,
1205             "Tests that will only be fixed if they crash (WONTFIX)")
1206         self._printer.print_actual("")
1207
1208     def _print_result_summary_entry(self, result_summary, timeline,
1209                                     heading):
1210         """Print a summary block of results for a particular timeline of test.
1211
1212         Args:
1213           result_summary: summary to print results for
1214           timeline: the timeline to print results for (NOT, WONTFIX, etc.)
1215           heading: a textual description of the timeline
1216         """
1217         total = len(result_summary.tests_by_timeline[timeline])
1218         not_passing = (total -
1219            len(result_summary.tests_by_expectation[test_expectations.PASS] &
1220                result_summary.tests_by_timeline[timeline]))
1221         self._printer.print_actual("=> %s (%d):" % (heading, not_passing))
1222
1223         for result in TestExpectationsFile.EXPECTATION_ORDER:
1224             if result == test_expectations.PASS:
1225                 continue
1226             results = (result_summary.tests_by_expectation[result] &
1227                        result_summary.tests_by_timeline[timeline])
1228             desc = TestExpectationsFile.EXPECTATION_DESCRIPTIONS[result]
1229             if not_passing and len(results):
1230                 pct = len(results) * 100.0 / not_passing
1231                 self._printer.print_actual("  %5d %-24s (%4.1f%%)" %
1232                     (len(results), desc[len(results) != 1], pct))
1233
1234     def _results_html(self, test_files, failures, title="Test Failures", override_time=None):
1235         """
1236         test_files = a list of file paths
1237         failures = dictionary mapping test paths to failure objects
1238         title = title printed at top of test
1239         override_time = current time (used by unit tests)
1240         """
1241         page = """<html>
1242   <head>
1243     <title>Layout Test Results (%(time)s)</title>
1244   </head>
1245   <body>
1246     <h2>%(title)s (%(time)s)</h2>
1247         """ % {'title': title, 'time': override_time or time.asctime()}
1248
1249         for test_file in sorted(test_files):
1250             test_name = self._port.relative_test_filename(test_file)
1251             test_url = self._port.filename_to_uri(test_file)
1252             page += u"<p><a href='%s'>%s</a><br />\n" % (test_url, test_name)
1253             test_failures = failures.get(test_file, [])
1254             for failure in test_failures:
1255                 page += (u"&nbsp;&nbsp;%s<br/>" %
1256                          failure.result_html_output(test_name))
1257             page += "</p>\n"
1258         page += "</body></html>\n"
1259         return page
1260
1261     def _write_results_html_file(self, result_summary):
1262         """Write results.html which is a summary of tests that failed.
1263
1264         Args:
1265           result_summary: a summary of the results :)
1266
1267         Returns:
1268           True if any results were written (since expected failures may be
1269           omitted)
1270         """
1271         # test failures
1272         if self._options.full_results_html:
1273             results_title = "Test Failures"
1274             test_files = result_summary.failures.keys()
1275         else:
1276             results_title = "Unexpected Test Failures"
1277             unexpected_failures = self._get_failures(result_summary,
1278                 include_crashes=True)
1279             test_files = unexpected_failures.keys()
1280         if not len(test_files):
1281             return False
1282
1283         out_filename = os.path.join(self._options.results_directory,
1284                                     "results.html")
1285         with codecs.open(out_filename, "w", "utf-8") as results_file:
1286             html = self._results_html(test_files, result_summary.failures, results_title)
1287             results_file.write(html)
1288
1289         return True
1290
1291     def _show_results_html_file(self):
1292         """Shows the results.html page."""
1293         results_filename = os.path.join(self._options.results_directory,
1294                                         "results.html")
1295         self._port.show_results_html_file(results_filename)
1296
1297
1298 def read_test_files(files):
1299     tests = []
1300     for file in files:
1301         try:
1302             with codecs.open(file, 'r', 'utf-8') as file_contents:
1303                 # FIXME: This could be cleaner using a list comprehension.
1304                 for line in file_contents:
1305                     line = test_expectations.strip_comments(line)
1306                     if line:
1307                         tests.append(line)
1308         except IOError, e:
1309             if e.errno == errno.ENOENT:
1310                 _log.critical('')
1311                 _log.critical('--test-list file "%s" not found' % file)
1312             raise
1313     return tests
1314
1315
1316 def run(port, options, args, regular_output=sys.stderr,
1317         buildbot_output=sys.stdout):
1318     """Run the tests.
1319
1320     Args:
1321       port: Port object for port-specific behavior
1322       options: a dictionary of command line options
1323       args: a list of sub directories or files to test
1324       regular_output: a stream-like object that we can send logging/debug
1325           output to
1326       buildbot_output: a stream-like object that we can write all output that
1327           is intended to be parsed by the buildbot to
1328     Returns:
1329       the number of unexpected results that occurred, or -1 if there is an
1330           error.
1331
1332     """
1333     _set_up_derived_options(port, options)
1334
1335     printer = printing.Printer(port, options, regular_output, buildbot_output,
1336         int(options.child_processes), options.experimental_fully_parallel)
1337     if options.help_printing:
1338         printer.help_printing()
1339         printer.cleanup()
1340         return 0
1341
1342     last_unexpected_results = _gather_unexpected_results(options)
1343     if options.print_last_failures:
1344         printer.write("\n".join(last_unexpected_results) + "\n")
1345         printer.cleanup()
1346         return 0
1347
1348     # We wrap any parts of the run that are slow or likely to raise exceptions
1349     # in a try/finally to ensure that we clean up the logging configuration.
1350     num_unexpected_results = -1
1351     try:
1352         test_runner = TestRunner(port, options, printer)
1353         test_runner._print_config()
1354
1355         printer.print_update("Collecting tests ...")
1356         try:
1357             test_runner.collect_tests(args, last_unexpected_results)
1358         except IOError, e:
1359             if e.errno == errno.ENOENT:
1360                 return -1
1361             raise
1362
1363         printer.print_update("Parsing expectations ...")
1364         if options.lint_test_files:
1365             return test_runner.lint()
1366         test_runner.parse_expectations(port.test_platform_name(),
1367                                        options.configuration == 'Debug')
1368
1369         printer.print_update("Checking build ...")
1370         if not port.check_build(test_runner.needs_http()):
1371             _log.error("Build check failed")
1372             return -1
1373
1374         result_summary = test_runner.set_up_run()
1375         if result_summary:
1376             num_unexpected_results = test_runner.run(result_summary)
1377             test_runner.clean_up_run()
1378             _log.debug("Testing completed, Exit status: %d" %
1379                        num_unexpected_results)
1380     finally:
1381         printer.cleanup()
1382
1383     return num_unexpected_results
1384
1385
1386 def _set_up_derived_options(port_obj, options):
1387     """Sets the options values that depend on other options values."""
1388
1389     if not options.child_processes:
1390         # FIXME: Investigate perf/flakiness impact of using cpu_count + 1.
1391         options.child_processes = os.environ.get("WEBKIT_TEST_CHILD_PROCESSES",
1392                                                  str(port_obj.default_child_processes()))
1393
1394     if not options.configuration:
1395         options.configuration = port_obj.default_configuration()
1396
1397     if options.pixel_tests is None:
1398         options.pixel_tests = True
1399
1400     if not options.use_apache:
1401         options.use_apache = sys.platform in ('darwin', 'linux2')
1402
1403     if not os.path.isabs(options.results_directory):
1404         # This normalizes the path to the build dir.
1405         # FIXME: how this happens is not at all obvious; this is a dumb
1406         # interface and should be cleaned up.
1407         options.results_directory = port_obj.results_directory()
1408
1409     if not options.time_out_ms:
1410         if options.configuration == "Debug":
1411             options.time_out_ms = str(2 * TestRunner.DEFAULT_TEST_TIMEOUT_MS)
1412         else:
1413             options.time_out_ms = str(TestRunner.DEFAULT_TEST_TIMEOUT_MS)
1414
1415     options.slow_time_out_ms = str(5 * int(options.time_out_ms))
1416
1417
1418 def _gather_unexpected_results(options):
1419     """Returns the unexpected results from the previous run, if any."""
1420     last_unexpected_results = []
1421     if options.print_last_failures or options.retest_last_failures:
1422         unexpected_results_filename = os.path.join(
1423         options.results_directory, "unexpected_results.json")
1424         with codecs.open(unexpected_results_filename, "r", "utf-8") as file:
1425             results = simplejson.load(file)
1426         last_unexpected_results = results['tests'].keys()
1427     return last_unexpected_results
1428
1429
1430 def _compat_shim_callback(option, opt_str, value, parser):
1431     print "Ignoring unsupported option: %s" % opt_str
1432
1433
1434 def _compat_shim_option(option_name, **kwargs):
1435     return optparse.make_option(option_name, action="callback",
1436         callback=_compat_shim_callback,
1437         help="Ignored, for old-run-webkit-tests compat only.", **kwargs)
1438
1439
1440 def parse_args(args=None):
1441     """Provides a default set of command line args.
1442
1443     Returns a tuple of options, args from optparse"""
1444
1445     # FIXME: All of these options should be stored closer to the code which
1446     # FIXME: actually uses them. configuration_options should move
1447     # FIXME: to WebKitPort and be shared across all scripts.
1448     configuration_options = [
1449         optparse.make_option("-t", "--target", dest="configuration",
1450                              help="(DEPRECATED)"),
1451         # FIXME: --help should display which configuration is default.
1452         optparse.make_option('--debug', action='store_const', const='Debug',
1453                              dest="configuration",
1454                              help='Set the configuration to Debug'),
1455         optparse.make_option('--release', action='store_const',
1456                              const='Release', dest="configuration",
1457                              help='Set the configuration to Release'),
1458         # old-run-webkit-tests also accepts -c, --configuration CONFIGURATION.
1459     ]
1460
1461     print_options = printing.print_options()
1462
1463     # FIXME: These options should move onto the ChromiumPort.
1464     chromium_options = [
1465         optparse.make_option("--chromium", action="store_true", default=False,
1466             help="use the Chromium port"),
1467         optparse.make_option("--startup-dialog", action="store_true",
1468             default=False, help="create a dialog on DumpRenderTree startup"),
1469         optparse.make_option("--gp-fault-error-box", action="store_true",
1470             default=False, help="enable Windows GP fault error box"),
1471         optparse.make_option("--multiple-loads",
1472             type="int", help="turn on multiple loads of each test"),
1473         optparse.make_option("--js-flags",
1474             type="string", help="JavaScript flags to pass to tests"),
1475         optparse.make_option("--nocheck-sys-deps", action="store_true",
1476             default=False,
1477             help="Don't check the system dependencies (themes)"),
1478         optparse.make_option("--use-drt", action="store_true",
1479             default=None,
1480             help="Use DumpRenderTree instead of test_shell"),
1481         optparse.make_option("--accelerated-compositing",
1482             action="store_true",
1483             help="Use hardware-accelated compositing for rendering"),
1484         optparse.make_option("--no-accelerated-compositing",
1485             action="store_false",
1486             dest="accelerated_compositing",
1487             help="Don't use hardware-accelerated compositing for rendering"),
1488         optparse.make_option("--accelerated-2d-canvas",
1489             action="store_true",
1490             help="Use hardware-accelerated 2D Canvas calls"),
1491         optparse.make_option("--no-accelerated-2d-canvas",
1492             action="store_false",
1493             dest="accelerated_2d_canvas",
1494             help="Don't use hardware-accelerated 2D Canvas calls"),
1495     ]
1496
1497     # Missing Mac-specific old-run-webkit-tests options:
1498     # FIXME: Need: -g, --guard for guard malloc support on Mac.
1499     # FIXME: Need: -l --leaks    Enable leaks checking.
1500     # FIXME: Need: --sample-on-timeout Run sample on timeout
1501
1502     old_run_webkit_tests_compat = [
1503         # NRWT doesn't generate results by default anyway.
1504         _compat_shim_option("--no-new-test-results"),
1505         # NRWT doesn't sample on timeout yet anyway.
1506         _compat_shim_option("--no-sample-on-timeout"),
1507         # FIXME: NRWT needs to support remote links eventually.
1508         _compat_shim_option("--use-remote-links-to-tests"),
1509         # FIXME: NRWT doesn't need this option as much since failures are
1510         # designed to be cheap.  We eventually plan to add this support.
1511         _compat_shim_option("--exit-after-n-failures", nargs=1, type="int"),
1512     ]
1513
1514     results_options = [
1515         # NEED for bots: --use-remote-links-to-tests Link to test files
1516         # within the SVN repository in the results.
1517         optparse.make_option("-p", "--pixel-tests", action="store_true",
1518             dest="pixel_tests", help="Enable pixel-to-pixel PNG comparisons"),
1519         optparse.make_option("--no-pixel-tests", action="store_false",
1520             dest="pixel_tests", help="Disable pixel-to-pixel PNG comparisons"),
1521         optparse.make_option("--tolerance",
1522             help="Ignore image differences less than this percentage (some "
1523                 "ports may ignore this option)", type="float"),
1524         optparse.make_option("--results-directory",
1525             default="layout-test-results",
1526             help="Output results directory source dir, relative to Debug or "
1527                  "Release"),
1528         optparse.make_option("--new-baseline", action="store_true",
1529             default=False, help="Save all generated results as new baselines "
1530                  "into the platform directory, overwriting whatever's "
1531                  "already there."),
1532         optparse.make_option("--reset-results", action="store_true",
1533             default=False, help="Reset any existing baselines to the "
1534                  "generated results"),
1535         optparse.make_option("--no-show-results", action="store_false",
1536             default=True, dest="show_results",
1537             help="Don't launch a browser with results after the tests "
1538                  "are done"),
1539         # FIXME: We should have a helper function to do this sort of
1540         # deprectated mapping and automatically log, etc.
1541         optparse.make_option("--noshow-results", action="store_false",
1542             dest="show_results",
1543             help="Deprecated, same as --no-show-results."),
1544         optparse.make_option("--no-launch-safari", action="store_false",
1545             dest="show_results",
1546             help="old-run-webkit-tests compat, same as --noshow-results."),
1547         # old-run-webkit-tests:
1548         # --[no-]launch-safari    Launch (or do not launch) Safari to display
1549         #                         test results (default: launch)
1550         optparse.make_option("--full-results-html", action="store_true",
1551             default=False,
1552             help="Show all failures in results.html, rather than only "
1553                  "regressions"),
1554         optparse.make_option("--clobber-old-results", action="store_true",
1555             default=False, help="Clobbers test results from previous runs."),
1556         optparse.make_option("--platform",
1557             help="Override the platform for expected results"),
1558         optparse.make_option("--no-record-results", action="store_false",
1559             default=True, dest="record_results",
1560             help="Don't record the results."),
1561         # old-run-webkit-tests also has HTTP toggle options:
1562         # --[no-]http                     Run (or do not run) http tests
1563         #                                 (default: run)
1564     ]
1565
1566     test_options = [
1567         optparse.make_option("--build", dest="build",
1568             action="store_true", default=True,
1569             help="Check to ensure the DumpRenderTree build is up-to-date "
1570                  "(default)."),
1571         optparse.make_option("--no-build", dest="build",
1572             action="store_false", help="Don't check to see if the "
1573                                        "DumpRenderTree build is up-to-date."),
1574         # old-run-webkit-tests has --valgrind instead of wrapper.
1575         optparse.make_option("--wrapper",
1576             help="wrapper command to insert before invocations of "
1577                  "DumpRenderTree; option is split on whitespace before "
1578                  "running. (Example: --wrapper='valgrind --smc-check=all')"),
1579         # old-run-webkit-tests:
1580         # -i|--ignore-tests               Comma-separated list of directories
1581         #                                 or tests to ignore
1582         optparse.make_option("--test-list", action="append",
1583             help="read list of tests to run from file", metavar="FILE"),
1584         # old-run-webkit-tests uses --skipped==[default|ignore|only]
1585         # instead of --force:
1586         optparse.make_option("--force", action="store_true", default=False,
1587             help="Run all tests, even those marked SKIP in the test list"),
1588         optparse.make_option("--use-apache", action="store_true",
1589             default=False, help="Whether to use apache instead of lighttpd."),
1590         optparse.make_option("--time-out-ms",
1591             help="Set the timeout for each test"),
1592         # old-run-webkit-tests calls --randomize-order --random:
1593         optparse.make_option("--randomize-order", action="store_true",
1594             default=False, help=("Run tests in random order (useful "
1595                                 "for tracking down corruption)")),
1596         optparse.make_option("--run-chunk",
1597             help=("Run a specified chunk (n:l), the nth of len l, "
1598                  "of the layout tests")),
1599         optparse.make_option("--run-part", help=("Run a specified part (n:m), "
1600                   "the nth of m parts, of the layout tests")),
1601         # old-run-webkit-tests calls --batch-size: --nthly n
1602         #   Restart DumpRenderTree every n tests (default: 1000)
1603         optparse.make_option("--batch-size",
1604             help=("Run a the tests in batches (n), after every n tests, "
1605                   "DumpRenderTree is relaunched."), type="int", default=0),
1606         # old-run-webkit-tests calls --run-singly: -1|--singly
1607         # Isolate each test case run (implies --nthly 1 --verbose)
1608         optparse.make_option("--run-singly", action="store_true",
1609             default=False, help="run a separate DumpRenderTree for each test"),
1610         optparse.make_option("--child-processes",
1611             help="Number of DumpRenderTrees to run in parallel."),
1612         # FIXME: Display default number of child processes that will run.
1613         optparse.make_option("--experimental-fully-parallel",
1614             action="store_true", default=False,
1615             help="run all tests in parallel"),
1616         # FIXME: Need --exit-after-n-failures N
1617         #      Exit after the first N failures instead of running all tests
1618         # FIXME: Need --exit-after-n-crashes N
1619         #      Exit after the first N crashes instead of running all tests
1620         # FIXME: consider: --iterations n
1621         #      Number of times to run the set of tests (e.g. ABCABCABC)
1622         optparse.make_option("--print-last-failures", action="store_true",
1623             default=False, help="Print the tests in the last run that "
1624             "had unexpected failures (or passes)."),
1625         optparse.make_option("--retest-last-failures", action="store_true",
1626             default=False, help="re-test the tests in the last run that "
1627             "had unexpected failures (or passes)."),
1628         optparse.make_option("--retry-failures", action="store_true",
1629             default=True,
1630             help="Re-try any tests that produce unexpected results (default)"),
1631         optparse.make_option("--no-retry-failures", action="store_false",
1632             dest="retry_failures",
1633             help="Don't re-try any tests that produce unexpected results."),
1634     ]
1635
1636     misc_options = [
1637         optparse.make_option("--lint-test-files", action="store_true",
1638         default=False, help=("Makes sure the test files parse for all "
1639                             "configurations. Does not run any tests.")),
1640     ]
1641
1642     # FIXME: Move these into json_results_generator.py
1643     results_json_options = [
1644         optparse.make_option("--master-name", help="The name of the buildbot master."),
1645         optparse.make_option("--builder-name", default="DUMMY_BUILDER_NAME",
1646             help=("The name of the builder shown on the waterfall running "
1647                   "this script e.g. WebKit.")),
1648         optparse.make_option("--build-name", default="DUMMY_BUILD_NAME",
1649             help=("The name of the builder used in its path, e.g. "
1650                   "webkit-rel.")),
1651         optparse.make_option("--build-number", default="DUMMY_BUILD_NUMBER",
1652             help=("The build number of the builder running this script.")),
1653         optparse.make_option("--test-results-server", default="",
1654             help=("If specified, upload results json files to this appengine "
1655                   "server.")),
1656         optparse.make_option("--upload-full-results",
1657             action="store_true",
1658             default=False,
1659             help="If true, upload full json results to server."),
1660     ]
1661
1662     option_list = (configuration_options + print_options +
1663                    chromium_options + results_options + test_options +
1664                    misc_options + results_json_options +
1665                    old_run_webkit_tests_compat)
1666     option_parser = optparse.OptionParser(option_list=option_list)
1667
1668     options, args = option_parser.parse_args(args)
1669
1670     return options, args
1671
1672
1673 def _log_wedged_thread(thread):
1674     """Log information about the given thread state."""
1675     id = thread.id()
1676     stack = dump_render_tree_thread.find_thread_stack(id)
1677     assert(stack is not None)
1678     _log.error("")
1679     _log.error("thread %s (%d) is wedged" % (thread.getName(), id))
1680     dump_render_tree_thread.log_stack(stack)
1681     _log.error("")
1682
1683
1684 def main():
1685     options, args = parse_args()
1686     port_obj = port.get(options.platform, options)
1687     return run(port_obj, options, args)
1688
1689 if '__main__' == __name__:
1690     try:
1691         sys.exit(main())
1692     except KeyboardInterrupt:
1693         # this mirrors what the shell normally does
1694         sys.exit(signal.SIGINT + 128)