2010-12-02 Dirk Pranke <dpranke@chromium.org>
[WebKit.git] / WebKitTools / Scripts / webkitpy / layout_tests / run_webkit_tests.py
1 #!/usr/bin/env python
2 # Copyright (C) 2010 Google Inc. All rights reserved.
3 # Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Szeged
4 #
5 # Redistribution and use in source and binary forms, with or without
6 # modification, are permitted provided that the following conditions are
7 # met:
8 #
9 #     * Redistributions of source code must retain the above copyright
10 # notice, this list of conditions and the following disclaimer.
11 #     * Redistributions in binary form must reproduce the above
12 # copyright notice, this list of conditions and the following disclaimer
13 # in the documentation and/or other materials provided with the
14 # distribution.
15 #     * Neither the name of Google Inc. nor the names of its
16 # contributors may be used to endorse or promote products derived from
17 # this software without specific prior written permission.
18 #
19 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30
31 """Run layout tests.
32
33 This is a port of the existing webkit test script run-webkit-tests.
34
35 The TestRunner class runs a series of tests (TestType interface) against a set
36 of test files.  If a test file fails a TestType, it returns a list TestFailure
37 objects to the TestRunner.  The TestRunner then aggregates the TestFailures to
38 create a final report.
39
40 This script reads several files, if they exist in the test_lists subdirectory
41 next to this script itself.  Each should contain a list of paths to individual
42 tests or entire subdirectories of tests, relative to the outermost test
43 directory.  Entire lines starting with '//' (comments) will be ignored.
44
45 For details of the files' contents and purposes, see test_lists/README.
46 """
47
48 from __future__ import with_statement
49
50 import codecs
51 import errno
52 import glob
53 import logging
54 import math
55 import optparse
56 import os
57 import platform
58 import Queue
59 import random
60 import re
61 import shutil
62 import signal
63 import sys
64 import time
65 import traceback
66
67 from layout_package import dump_render_tree_thread
68 from layout_package import json_layout_results_generator
69 from layout_package import message_broker
70 from layout_package import printing
71 from layout_package import test_expectations
72 from layout_package import test_failures
73 from layout_package import test_results
74 from layout_package import test_results_uploader
75
76 from webkitpy.common.system import user
77 from webkitpy.thirdparty import simplejson
78 from webkitpy.tool import grammar
79
80 import port
81
82 _log = logging.getLogger("webkitpy.layout_tests.run_webkit_tests")
83
84 # Builder base URL where we have the archived test results.
85 BUILDER_BASE_URL = "http://build.chromium.org/buildbot/layout_test_results/"
86
87 LAYOUT_TESTS_DIRECTORY = "LayoutTests" + os.sep
88
89 TestExpectationsFile = test_expectations.TestExpectationsFile
90
91
92 class TestInput:
93     """Groups information about a test for easy passing of data."""
94
95     def __init__(self, filename, timeout):
96         """Holds the input parameters for a test.
97         Args:
98           filename: Full path to the test.
99           timeout: Timeout in msecs the driver should use while running the test
100           """
101         # FIXME: filename should really be test_name as a relative path.
102         self.filename = filename
103         self.timeout = timeout
104         # The image_hash is used to avoid doing an image dump if the
105         # checksums match. The image_hash is set later, and only if it is needed
106         # for the test.
107         self.image_hash = None
108
109
110 class ResultSummary(object):
111     """A class for partitioning the test results we get into buckets.
112
113     This class is basically a glorified struct and it's private to this file
114     so we don't bother with any information hiding."""
115
116     def __init__(self, expectations, test_files):
117         self.total = len(test_files)
118         self.remaining = self.total
119         self.expectations = expectations
120         self.expected = 0
121         self.unexpected = 0
122         self.tests_by_expectation = {}
123         self.tests_by_timeline = {}
124         self.results = {}
125         self.unexpected_results = {}
126         self.failures = {}
127         self.tests_by_expectation[test_expectations.SKIP] = set()
128         for expectation in TestExpectationsFile.EXPECTATIONS.values():
129             self.tests_by_expectation[expectation] = set()
130         for timeline in TestExpectationsFile.TIMELINES.values():
131             self.tests_by_timeline[timeline] = (
132                 expectations.get_tests_with_timeline(timeline))
133
134     def add(self, result, expected):
135         """Add a TestResult into the appropriate bin.
136
137         Args:
138           result: TestResult from dump_render_tree_thread.
139           expected: whether the result was what we expected it to be.
140         """
141
142         self.tests_by_expectation[result.type].add(result.filename)
143         self.results[result.filename] = result
144         self.remaining -= 1
145         if len(result.failures):
146             self.failures[result.filename] = result.failures
147         if expected:
148             self.expected += 1
149         else:
150             self.unexpected_results[result.filename] = result.type
151             self.unexpected += 1
152
153
154 def summarize_unexpected_results(port_obj, expectations, result_summary,
155                                  retry_summary):
156     """Summarize any unexpected results as a dict.
157
158     FIXME: split this data structure into a separate class?
159
160     Args:
161         port_obj: interface to port-specific hooks
162         expectations: test_expectations.TestExpectations object
163         result_summary: summary object from initial test runs
164         retry_summary: summary object from final test run of retried tests
165     Returns:
166         A dictionary containing a summary of the unexpected results from the
167         run, with the following fields:
168         'version': a version indicator (1 in this version)
169         'fixable': # of fixable tests (NOW - PASS)
170         'skipped': # of skipped tests (NOW & SKIPPED)
171         'num_regressions': # of non-flaky failures
172         'num_flaky': # of flaky failures
173         'num_passes': # of unexpected passes
174         'tests': a dict of tests -> {'expected': '...', 'actual': '...'}
175     """
176     results = {}
177     results['version'] = 1
178
179     tbe = result_summary.tests_by_expectation
180     tbt = result_summary.tests_by_timeline
181     results['fixable'] = len(tbt[test_expectations.NOW] -
182                                 tbe[test_expectations.PASS])
183     results['skipped'] = len(tbt[test_expectations.NOW] &
184                                 tbe[test_expectations.SKIP])
185
186     num_passes = 0
187     num_flaky = 0
188     num_regressions = 0
189     keywords = {}
190     for k, v in TestExpectationsFile.EXPECTATIONS.iteritems():
191         keywords[v] = k.upper()
192
193     tests = {}
194     for filename, result in result_summary.unexpected_results.iteritems():
195         # Note that if a test crashed in the original run, we ignore
196         # whether or not it crashed when we retried it (if we retried it),
197         # and always consider the result not flaky.
198         test = port_obj.relative_test_filename(filename)
199         expected = expectations.get_expectations_string(filename)
200         actual = [keywords[result]]
201
202         if result == test_expectations.PASS:
203             num_passes += 1
204         elif result == test_expectations.CRASH:
205             num_regressions += 1
206         else:
207             if filename not in retry_summary.unexpected_results:
208                 actual.extend(expectations.get_expectations_string(
209                     filename).split(" "))
210                 num_flaky += 1
211             else:
212                 retry_result = retry_summary.unexpected_results[filename]
213                 if result != retry_result:
214                     actual.append(keywords[retry_result])
215                     num_flaky += 1
216                 else:
217                     num_regressions += 1
218
219         tests[test] = {}
220         tests[test]['expected'] = expected
221         tests[test]['actual'] = " ".join(actual)
222
223     results['tests'] = tests
224     results['num_passes'] = num_passes
225     results['num_flaky'] = num_flaky
226     results['num_regressions'] = num_regressions
227
228     return results
229
230
231 class WorkerState(object):
232     """A class for the TestRunner/manager to use to track the current state
233     of the workers."""
234     def __init__(self, name, number, thread):
235         self.name = name
236         self.number = number
237         self.thread = thread
238
239
240 class TestRunner:
241     """A class for managing running a series of tests on a series of layout
242     test files."""
243
244     HTTP_SUBDIR = os.sep.join(['', 'http', ''])
245     WEBSOCKET_SUBDIR = os.sep.join(['', 'websocket', ''])
246
247     # The per-test timeout in milliseconds, if no --time-out-ms option was
248     # given to run_webkit_tests. This should correspond to the default timeout
249     # in DumpRenderTree.
250     DEFAULT_TEST_TIMEOUT_MS = 6 * 1000
251
252     def __init__(self, port, options, printer):
253         """Initialize test runner data structures.
254
255         Args:
256           port: an object implementing port-specific
257           options: a dictionary of command line options
258           printer: a Printer object to record updates to.
259         """
260         self._port = port
261         self._options = options
262         self._printer = printer
263
264         # This maps worker names to the state we are tracking for each of them.
265         self._workers = {}
266
267         # disable wss server. need to install pyOpenSSL on buildbots.
268         # self._websocket_secure_server = websocket_server.PyWebSocket(
269         #        options.results_directory, use_tls=True, port=9323)
270
271         # a set of test files, and the same tests as a list
272         self._test_files = set()
273         self._test_files_list = None
274         self._result_queue = Queue.Queue()
275         self._retrying = False
276
277     def collect_tests(self, args, last_unexpected_results):
278         """Find all the files to test.
279
280         Args:
281           args: list of test arguments from the command line
282           last_unexpected_results: list of unexpected results to retest, if any
283
284         """
285         paths = [self._strip_test_dir_prefix(arg) for arg in args if arg and arg != '']
286         paths += last_unexpected_results
287         if self._options.test_list:
288             paths += read_test_files(self._options.test_list)
289         self._test_files = self._port.tests(paths)
290
291     def _strip_test_dir_prefix(self, path):
292         if path.startswith(LAYOUT_TESTS_DIRECTORY):
293             return path[len(LAYOUT_TESTS_DIRECTORY):]
294         return path
295
296     def lint(self):
297         # Creating the expecations for each platform/configuration pair does
298         # all the test list parsing and ensures it's correct syntax (e.g. no
299         # dupes).
300         for platform_name in self._port.test_platform_names():
301             self.parse_expectations(platform_name, is_debug_mode=True)
302             self.parse_expectations(platform_name, is_debug_mode=False)
303         self._printer.write("")
304         _log.info("If there are no fail messages, errors or exceptions, "
305                   "then the lint succeeded.")
306         return 0
307
308     def parse_expectations(self, test_platform_name, is_debug_mode):
309         """Parse the expectations from the test_list files and return a data
310         structure holding them. Throws an error if the test_list files have
311         invalid syntax."""
312         if self._options.lint_test_files:
313             test_files = None
314         else:
315             test_files = self._test_files
316
317         try:
318             expectations_str = self._port.test_expectations()
319             overrides_str = self._port.test_expectations_overrides()
320             self._expectations = test_expectations.TestExpectations(
321                 self._port, test_files, expectations_str, test_platform_name,
322                 is_debug_mode, self._options.lint_test_files,
323                 overrides=overrides_str)
324             return self._expectations
325         except SyntaxError, err:
326             if self._options.lint_test_files:
327                 print str(err)
328             else:
329                 raise err
330
331     def prepare_lists_and_print_output(self):
332         """Create appropriate subsets of test lists and returns a
333         ResultSummary object. Also prints expected test counts.
334         """
335
336         # Remove skipped - both fixable and ignored - files from the
337         # top-level list of files to test.
338         num_all_test_files = len(self._test_files)
339         self._printer.print_expected("Found:  %d tests" %
340                                      (len(self._test_files)))
341         if not num_all_test_files:
342             _log.critical('No tests to run.')
343             return None
344
345         skipped = set()
346         if num_all_test_files > 1 and not self._options.force:
347             skipped = self._expectations.get_tests_with_result_type(
348                            test_expectations.SKIP)
349             self._test_files -= skipped
350
351         # Create a sorted list of test files so the subset chunk,
352         # if used, contains alphabetically consecutive tests.
353         self._test_files_list = list(self._test_files)
354         if self._options.randomize_order:
355             random.shuffle(self._test_files_list)
356         else:
357             self._test_files_list.sort()
358
359         # If the user specifies they just want to run a subset of the tests,
360         # just grab a subset of the non-skipped tests.
361         if self._options.run_chunk or self._options.run_part:
362             chunk_value = self._options.run_chunk or self._options.run_part
363             test_files = self._test_files_list
364             try:
365                 (chunk_num, chunk_len) = chunk_value.split(":")
366                 chunk_num = int(chunk_num)
367                 assert(chunk_num >= 0)
368                 test_size = int(chunk_len)
369                 assert(test_size > 0)
370             except:
371                 _log.critical("invalid chunk '%s'" % chunk_value)
372                 return None
373
374             # Get the number of tests
375             num_tests = len(test_files)
376
377             # Get the start offset of the slice.
378             if self._options.run_chunk:
379                 chunk_len = test_size
380                 # In this case chunk_num can be really large. We need
381                 # to make the slave fit in the current number of tests.
382                 slice_start = (chunk_num * chunk_len) % num_tests
383             else:
384                 # Validate the data.
385                 assert(test_size <= num_tests)
386                 assert(chunk_num <= test_size)
387
388                 # To count the chunk_len, and make sure we don't skip
389                 # some tests, we round to the next value that fits exactly
390                 # all the parts.
391                 rounded_tests = num_tests
392                 if rounded_tests % test_size != 0:
393                     rounded_tests = (num_tests + test_size -
394                                      (num_tests % test_size))
395
396                 chunk_len = rounded_tests / test_size
397                 slice_start = chunk_len * (chunk_num - 1)
398                 # It does not mind if we go over test_size.
399
400             # Get the end offset of the slice.
401             slice_end = min(num_tests, slice_start + chunk_len)
402
403             files = test_files[slice_start:slice_end]
404
405             tests_run_msg = 'Running: %d tests (chunk slice [%d:%d] of %d)' % (
406                 (slice_end - slice_start), slice_start, slice_end, num_tests)
407             self._printer.print_expected(tests_run_msg)
408
409             # If we reached the end and we don't have enough tests, we run some
410             # from the beginning.
411             if slice_end - slice_start < chunk_len:
412                 extra = chunk_len - (slice_end - slice_start)
413                 extra_msg = ('   last chunk is partial, appending [0:%d]' %
414                             extra)
415                 self._printer.print_expected(extra_msg)
416                 tests_run_msg += "\n" + extra_msg
417                 files.extend(test_files[0:extra])
418             tests_run_filename = os.path.join(self._options.results_directory,
419                                               "tests_run.txt")
420             with codecs.open(tests_run_filename, "w", "utf-8") as file:
421                 file.write(tests_run_msg + "\n")
422
423             len_skip_chunk = int(len(files) * len(skipped) /
424                                  float(len(self._test_files)))
425             skip_chunk_list = list(skipped)[0:len_skip_chunk]
426             skip_chunk = set(skip_chunk_list)
427
428             # Update expectations so that the stats are calculated correctly.
429             # We need to pass a list that includes the right # of skipped files
430             # to ParseExpectations so that ResultSummary() will get the correct
431             # stats. So, we add in the subset of skipped files, and then
432             # subtract them back out.
433             self._test_files_list = files + skip_chunk_list
434             self._test_files = set(self._test_files_list)
435
436             self._expectations = self.parse_expectations(
437                 self._port.test_platform_name(),
438                 self._options.configuration == 'Debug')
439
440             self._test_files = set(files)
441             self._test_files_list = files
442         else:
443             skip_chunk = skipped
444
445         result_summary = ResultSummary(self._expectations,
446             self._test_files | skip_chunk)
447         self._print_expected_results_of_type(result_summary,
448             test_expectations.PASS, "passes")
449         self._print_expected_results_of_type(result_summary,
450             test_expectations.FAIL, "failures")
451         self._print_expected_results_of_type(result_summary,
452             test_expectations.FLAKY, "flaky")
453         self._print_expected_results_of_type(result_summary,
454             test_expectations.SKIP, "skipped")
455
456         if self._options.force:
457             self._printer.print_expected('Running all tests, including '
458                                          'skips (--force)')
459         else:
460             # Note that we don't actually run the skipped tests (they were
461             # subtracted out of self._test_files, above), but we stub out the
462             # results here so the statistics can remain accurate.
463             for test in skip_chunk:
464                 result = test_results.TestResult(test,
465                     failures=[], test_run_time=0, total_time_for_all_diffs=0,
466                     time_for_diffs=0)
467                 result.type = test_expectations.SKIP
468                 result_summary.add(result, expected=True)
469         self._printer.print_expected('')
470
471         return result_summary
472
473     def _get_dir_for_test_file(self, test_file):
474         """Returns the highest-level directory by which to shard the given
475         test file."""
476         index = test_file.rfind(os.sep + LAYOUT_TESTS_DIRECTORY)
477
478         test_file = test_file[index + len(LAYOUT_TESTS_DIRECTORY):]
479         test_file_parts = test_file.split(os.sep, 1)
480         directory = test_file_parts[0]
481         test_file = test_file_parts[1]
482
483         # The http tests are very stable on mac/linux.
484         # TODO(ojan): Make the http server on Windows be apache so we can
485         # turn shard the http tests there as well. Switching to apache is
486         # what made them stable on linux/mac.
487         return_value = directory
488         while ((directory != 'http' or sys.platform in ('darwin', 'linux2'))
489                 and test_file.find(os.sep) >= 0):
490             test_file_parts = test_file.split(os.sep, 1)
491             directory = test_file_parts[0]
492             return_value = os.path.join(return_value, directory)
493             test_file = test_file_parts[1]
494
495         return return_value
496
497     def _get_test_input_for_file(self, test_file):
498         """Returns the appropriate TestInput object for the file. Mostly this
499         is used for looking up the timeout value (in ms) to use for the given
500         test."""
501         if self._test_is_slow(test_file):
502             return TestInput(test_file, self._options.slow_time_out_ms)
503         return TestInput(test_file, self._options.time_out_ms)
504
505     def _test_requires_lock(self, test_file):
506         """Return True if the test needs to be locked when
507         running multiple copies of NRWTs."""
508         split_path = test_file.split(os.sep)
509         return 'http' in split_path or 'websocket' in split_path
510
511     def _test_is_slow(self, test_file):
512         return self._expectations.has_modifier(test_file,
513                                                test_expectations.SLOW)
514
515     def _shard_tests(self, test_files, use_real_shards):
516         """Groups tests into batches.
517         This helps ensure that tests that depend on each other (aka bad tests!)
518         continue to run together as most cross-tests dependencies tend to
519         occur within the same directory. If use_real_shards is false, we
520         put each (non-HTTP/websocket) test into its own shard for maximum
521         concurrency instead of trying to do any sort of real sharding.
522
523         Return:
524             A list of lists of TestInput objects.
525         """
526         # FIXME: when we added http locking, we changed how this works such
527         # that we always lump all of the HTTP threads into a single shard.
528         # That will slow down experimental-fully-parallel, but it's unclear
529         # what the best alternative is completely revamping how we track
530         # when to grab the lock.
531
532         test_lists = []
533         tests_to_http_lock = []
534         if not use_real_shards:
535             for test_file in test_files:
536                 test_input = self._get_test_input_for_file(test_file)
537                 if self._test_requires_lock(test_file):
538                     tests_to_http_lock.append(test_input)
539                 else:
540                     test_lists.append((".", [test_input]))
541         else:
542             tests_by_dir = {}
543             for test_file in test_files:
544                 directory = self._get_dir_for_test_file(test_file)
545                 test_input = self._get_test_input_for_file(test_file)
546                 if self._test_requires_lock(test_file):
547                     tests_to_http_lock.append(test_input)
548                 else:
549                     tests_by_dir.setdefault(directory, [])
550                     tests_by_dir[directory].append(test_input)
551             # Sort by the number of tests in the dir so that the ones with the
552             # most tests get run first in order to maximize parallelization.
553             # Number of tests is a good enough, but not perfect, approximation
554             # of how long that set of tests will take to run. We can't just use
555             # a PriorityQueue until we move to Python 2.6.
556             for directory in tests_by_dir:
557                 test_list = tests_by_dir[directory]
558                 # Keep the tests in alphabetical order.
559                 # FIXME: Remove once tests are fixed so they can be run in any
560                 # order.
561                 test_list.reverse()
562                 test_list_tuple = (directory, test_list)
563                 test_lists.append(test_list_tuple)
564             test_lists.sort(lambda a, b: cmp(len(b[1]), len(a[1])))
565
566         # Put the http tests first. There are only a couple hundred of them,
567         # but each http test takes a very long time to run, so sorting by the
568         # number of tests doesn't accurately capture how long they take to run.
569         if tests_to_http_lock:
570             tests_to_http_lock.reverse()
571             test_lists.insert(0, ("tests_to_http_lock", tests_to_http_lock))
572
573         return test_lists
574
575     def _contains_tests(self, subdir):
576         for test_file in self._test_files:
577             if test_file.find(subdir) >= 0:
578                 return True
579         return False
580
581     def _num_workers(self):
582         return int(self._options.child_processes)
583
584     def _run_tests(self, file_list, result_summary):
585         """Runs the tests in the file_list.
586
587         Return: A tuple (keyboard_interrupted, thread_timings, test_timings,
588             individual_test_timings)
589             keyboard_interrupted is whether someone typed Ctrl^C
590             thread_timings is a list of dicts with the total runtime
591               of each thread with 'name', 'num_tests', 'total_time' properties
592             test_timings is a list of timings for each sharded subdirectory
593               of the form [time, directory_name, num_tests]
594             individual_test_timings is a list of run times for each test
595               in the form {filename:filename, test_run_time:test_run_time}
596             result_summary: summary object to populate with the results
597         """
598
599         self._workers = {}
600
601         self._printer.print_update('Sharding tests ...')
602         num_workers = self._num_workers()
603         test_lists = self._shard_tests(file_list,
604             num_workers > 1 and not self._options.experimental_fully_parallel)
605
606         broker = message_broker.get(self._port, self._options)
607         self._message_broker = broker
608
609         filename_queue = Queue.Queue()
610         for item in test_lists:
611             filename_queue.put(item)
612
613         self._printer.print_update('Starting %s ...' %
614                                    grammar.pluralize('worker', num_workers))
615         self._current_filename_queue = filename_queue
616         self._current_result_summary = result_summary
617
618         for worker_number in xrange(num_workers):
619             thread = broker.start_worker(self, worker_number)
620             w = WorkerState(thread.name(), worker_number, thread)
621             self._workers[thread.name()] = w
622
623         self._printer.print_update("Starting testing ...")
624         keyboard_interrupted = False
625         if not self._options.dry_run:
626             try:
627                 broker.run_message_loop(self)
628             except KeyboardInterrupt:
629                 _log.info("Interrupted, exiting")
630                 for worker_name in self._workers.keys():
631                     broker.cancel_worker(worker_name)
632                 keyboard_interrupted = True
633             except:
634                 # Unexpected exception; don't try to clean up workers.
635                 _log.info("Exception raised, exiting")
636                 raise
637
638         thread_timings, test_timings, individual_test_timings = \
639             self._collect_timing_info(self._workers)
640         self._message_broker = None
641
642         return (keyboard_interrupted, thread_timings, test_timings,
643                 individual_test_timings)
644
645     def _check_on_workers(self):
646         """Returns True iff all the workers have either completed or wedged."""
647
648         # Loop through all the threads waiting for them to finish.
649         some_thread_is_alive = True
650         while some_thread_is_alive:
651             some_thread_is_alive = False
652             t = time.time()
653             for worker in self._workers.values():
654                 thread = worker.thread
655                 exception_info = thread.exception_info()
656                 if exception_info is not None:
657                     # Re-raise the thread's exception here to make it
658                     # clear that testing was aborted. Otherwise,
659                     # the tests that did not run would be assumed
660                     # to have passed.
661                     raise exception_info[0], exception_info[1], exception_info[2]
662
663                 if thread.isAlive():
664                     some_thread_is_alive = True
665                     next_timeout = thread.next_timeout()
666                     if next_timeout and t > next_timeout:
667                         self._message_broker.log_wedged_worker(worker.name)
668                         thread.clear_next_timeout()
669
670             self.update_summary(self._current_result_summary)
671
672             if some_thread_is_alive:
673                 time.sleep(0.01)
674
675     def _collect_timing_info(self, workers):
676         test_timings = {}
677         individual_test_timings = []
678         thread_timings = []
679
680         for w in workers.values():
681             thread = w.thread
682             thread_timings.append({'name': thread.name(),
683                                    'num_tests': thread.get_num_tests(),
684                                    'total_time': thread.get_total_time()})
685             test_timings.update(thread.get_test_group_timing_stats())
686             individual_test_timings.extend(thread.get_test_results())
687
688         return (thread_timings, test_timings, individual_test_timings)
689
690     def needs_http(self):
691         """Returns whether the test runner needs an HTTP server."""
692         return self._contains_tests(self.HTTP_SUBDIR)
693
694     def needs_websocket(self):
695         """Returns whether the test runner needs a WEBSOCKET server."""
696         return self._contains_tests(self.WEBSOCKET_SUBDIR)
697
698     def set_up_run(self):
699         """Configures the system to be ready to run tests.
700
701         Returns a ResultSummary object if we should continue to run tests,
702         or None if we should abort.
703
704         """
705         # This must be started before we check the system dependencies,
706         # since the helper may do things to make the setup correct.
707         self._printer.print_update("Starting helper ...")
708         self._port.start_helper()
709
710         # Check that the system dependencies (themes, fonts, ...) are correct.
711         if not self._options.nocheck_sys_deps:
712             self._printer.print_update("Checking system dependencies ...")
713             if not self._port.check_sys_deps(self.needs_http()):
714                 self._port.stop_helper()
715                 return None
716
717         if self._options.clobber_old_results:
718             self._clobber_old_results()
719
720         # Create the output directory if it doesn't already exist.
721         self._port.maybe_make_directory(self._options.results_directory)
722
723         self._port.setup_test_run()
724
725         self._printer.print_update("Preparing tests ...")
726         result_summary = self.prepare_lists_and_print_output()
727         if not result_summary:
728             return None
729
730         return result_summary
731
732     def run(self, result_summary):
733         """Run all our tests on all our test files.
734
735         For each test file, we run each test type. If there are any failures,
736         we collect them for reporting.
737
738         Args:
739           result_summary: a summary object tracking the test results.
740
741         Return:
742           The number of unexpected results (0 == success)
743         """
744         # gather_test_files() must have been called first to initialize us.
745         # If we didn't find any files to test, we've errored out already in
746         # prepare_lists_and_print_output().
747         assert(len(self._test_files))
748
749         start_time = time.time()
750
751         keyboard_interrupted, thread_timings, test_timings, \
752             individual_test_timings = (
753             self._run_tests(self._test_files_list, result_summary))
754
755         # We exclude the crashes from the list of results to retry, because
756         # we want to treat even a potentially flaky crash as an error.
757         failures = self._get_failures(result_summary, include_crashes=False)
758         retry_summary = result_summary
759         while (len(failures) and self._options.retry_failures and
760             not self._retrying and not keyboard_interrupted):
761             _log.info('')
762             _log.info("Retrying %d unexpected failure(s) ..." % len(failures))
763             _log.info('')
764             self._retrying = True
765             retry_summary = ResultSummary(self._expectations, failures.keys())
766             # Note that we intentionally ignore the return value here.
767             self._run_tests(failures.keys(), retry_summary)
768             failures = self._get_failures(retry_summary, include_crashes=True)
769
770         end_time = time.time()
771
772         self._print_timing_statistics(end_time - start_time,
773                                       thread_timings, test_timings,
774                                       individual_test_timings,
775                                       result_summary)
776
777         self._print_result_summary(result_summary)
778
779         sys.stdout.flush()
780         sys.stderr.flush()
781
782         self._printer.print_one_line_summary(result_summary.total,
783                                              result_summary.expected,
784                                              result_summary.unexpected)
785
786         unexpected_results = summarize_unexpected_results(self._port,
787             self._expectations, result_summary, retry_summary)
788         self._printer.print_unexpected_results(unexpected_results)
789
790         if (self._options.record_results and not self._options.dry_run and
791             not keyboard_interrupted):
792             # Write the same data to log files and upload generated JSON files
793             # to appengine server.
794             self._upload_json_files(unexpected_results, result_summary,
795                                     individual_test_timings)
796
797         # Write the summary to disk (results.html) and display it if requested.
798         if not self._options.dry_run:
799             wrote_results = self._write_results_html_file(result_summary)
800             if self._options.show_results and wrote_results:
801                 self._show_results_html_file()
802
803         # Now that we've completed all the processing we can, we re-raise
804         # a KeyboardInterrupt if necessary so the caller can handle it.
805         if keyboard_interrupted:
806             raise KeyboardInterrupt
807
808         # Ignore flaky failures and unexpected passes so we don't turn the
809         # bot red for those.
810         return unexpected_results['num_regressions']
811
812     def clean_up_run(self):
813         """Restores the system after we're done running tests."""
814
815         _log.debug("flushing stdout")
816         sys.stdout.flush()
817         _log.debug("flushing stderr")
818         sys.stderr.flush()
819         _log.debug("stopping helper")
820         self._port.stop_helper()
821
822     def update_summary(self, result_summary):
823         """Update the summary and print results with any completed tests."""
824         while True:
825             try:
826                 result = test_results.TestResult.loads(self._result_queue.get_nowait())
827             except Queue.Empty:
828                 return
829
830             expected = self._expectations.matches_an_expected_result(
831                 result.filename, result.type, self._options.pixel_tests)
832             result_summary.add(result, expected)
833             exp_str = self._expectations.get_expectations_string(
834                 result.filename)
835             got_str = self._expectations.expectation_to_string(result.type)
836             self._printer.print_test_result(result, expected, exp_str, got_str)
837             self._printer.print_progress(result_summary, self._retrying,
838                                          self._test_files_list)
839
840     def _clobber_old_results(self):
841         # Just clobber the actual test results directories since the other
842         # files in the results directory are explicitly used for cross-run
843         # tracking.
844         self._printer.print_update("Clobbering old results in %s" %
845                                    self._options.results_directory)
846         layout_tests_dir = self._port.layout_tests_dir()
847         possible_dirs = self._port.test_dirs()
848         for dirname in possible_dirs:
849             if os.path.isdir(os.path.join(layout_tests_dir, dirname)):
850                 shutil.rmtree(os.path.join(self._options.results_directory,
851                                            dirname),
852                               ignore_errors=True)
853
854     def _get_failures(self, result_summary, include_crashes):
855         """Filters a dict of results and returns only the failures.
856
857         Args:
858           result_summary: the results of the test run
859           include_crashes: whether crashes are included in the output.
860             We use False when finding the list of failures to retry
861             to see if the results were flaky. Although the crashes may also be
862             flaky, we treat them as if they aren't so that they're not ignored.
863         Returns:
864           a dict of files -> results
865         """
866         failed_results = {}
867         for test, result in result_summary.unexpected_results.iteritems():
868             if (result == test_expectations.PASS or
869                 result == test_expectations.CRASH and not include_crashes):
870                 continue
871             failed_results[test] = result
872
873         return failed_results
874
875     def _upload_json_files(self, unexpected_results, result_summary,
876                         individual_test_timings):
877         """Writes the results of the test run as JSON files into the results
878         dir and upload the files to the appengine server.
879
880         There are three different files written into the results dir:
881           unexpected_results.json: A short list of any unexpected results.
882             This is used by the buildbots to display results.
883           expectations.json: This is used by the flakiness dashboard.
884           results.json: A full list of the results - used by the flakiness
885             dashboard and the aggregate results dashboard.
886
887         Args:
888           unexpected_results: dict of unexpected results
889           result_summary: full summary object
890           individual_test_timings: list of test times (used by the flakiness
891             dashboard).
892         """
893         results_directory = self._options.results_directory
894         _log.debug("Writing JSON files in %s." % results_directory)
895         unexpected_json_path = os.path.join(results_directory, "unexpected_results.json")
896         with codecs.open(unexpected_json_path, "w", "utf-8") as file:
897             simplejson.dump(unexpected_results, file, sort_keys=True, indent=2)
898
899         # Write a json file of the test_expectations.txt file for the layout
900         # tests dashboard.
901         expectations_path = os.path.join(results_directory, "expectations.json")
902         expectations_json = \
903             self._expectations.get_expectations_json_for_all_platforms()
904         with codecs.open(expectations_path, "w", "utf-8") as file:
905             file.write(u"ADD_EXPECTATIONS(%s);" % expectations_json)
906
907         generator = json_layout_results_generator.JSONLayoutResultsGenerator(
908             self._port, self._options.builder_name, self._options.build_name,
909             self._options.build_number, self._options.results_directory,
910             BUILDER_BASE_URL, individual_test_timings,
911             self._expectations, result_summary, self._test_files_list,
912             not self._options.upload_full_results,
913             self._options.test_results_server,
914             "layout-tests",
915             self._options.master_name)
916
917         _log.debug("Finished writing JSON files.")
918
919         json_files = ["expectations.json"]
920         if self._options.upload_full_results:
921             json_files.append("results.json")
922         else:
923             json_files.append("incremental_results.json")
924
925         generator.upload_json_files(json_files)
926
927     def _print_config(self):
928         """Prints the configuration for the test run."""
929         p = self._printer
930         p.print_config("Using port '%s'" % self._port.name())
931         p.print_config("Placing test results in %s" %
932                        self._options.results_directory)
933         if self._options.new_baseline:
934             p.print_config("Placing new baselines in %s" %
935                            self._port.baseline_path())
936         p.print_config("Using %s build" % self._options.configuration)
937         if self._options.pixel_tests:
938             p.print_config("Pixel tests enabled")
939         else:
940             p.print_config("Pixel tests disabled")
941
942         p.print_config("Regular timeout: %s, slow test timeout: %s" %
943                        (self._options.time_out_ms,
944                         self._options.slow_time_out_ms))
945
946         if self._num_workers() == 1:
947             p.print_config("Running one %s" % self._port.driver_name())
948         else:
949             p.print_config("Running %s %ss in parallel" %
950                            (self._options.child_processes,
951                             self._port.driver_name()))
952         p.print_config('Command line: ' +
953                        ' '.join(self._port.driver_cmd_line()))
954         p.print_config("Worker model: %s" % self._options.worker_model)
955         p.print_config("")
956
957     def _print_expected_results_of_type(self, result_summary,
958                                         result_type, result_type_str):
959         """Print the number of the tests in a given result class.
960
961         Args:
962           result_summary - the object containing all the results to report on
963           result_type - the particular result type to report in the summary.
964           result_type_str - a string description of the result_type.
965         """
966         tests = self._expectations.get_tests_with_result_type(result_type)
967         now = result_summary.tests_by_timeline[test_expectations.NOW]
968         wontfix = result_summary.tests_by_timeline[test_expectations.WONTFIX]
969
970         # We use a fancy format string in order to print the data out in a
971         # nicely-aligned table.
972         fmtstr = ("Expect: %%5d %%-8s (%%%dd now, %%%dd wontfix)"
973                   % (self._num_digits(now), self._num_digits(wontfix)))
974         self._printer.print_expected(fmtstr %
975             (len(tests), result_type_str, len(tests & now), len(tests & wontfix)))
976
977     def _num_digits(self, num):
978         """Returns the number of digits needed to represent the length of a
979         sequence."""
980         ndigits = 1
981         if len(num):
982             ndigits = int(math.log10(len(num))) + 1
983         return ndigits
984
985     def _print_timing_statistics(self, total_time, thread_timings,
986                                directory_test_timings, individual_test_timings,
987                                result_summary):
988         """Record timing-specific information for the test run.
989
990         Args:
991           total_time: total elapsed time (in seconds) for the test run
992           thread_timings: wall clock time each thread ran for
993           directory_test_timings: timing by directory
994           individual_test_timings: timing by file
995           result_summary: summary object for the test run
996         """
997         self._printer.print_timing("Test timing:")
998         self._printer.print_timing("  %6.2f total testing time" % total_time)
999         self._printer.print_timing("")
1000         self._printer.print_timing("Thread timing:")
1001         cuml_time = 0
1002         for t in thread_timings:
1003             self._printer.print_timing("    %10s: %5d tests, %6.2f secs" %
1004                   (t['name'], t['num_tests'], t['total_time']))
1005             cuml_time += t['total_time']
1006         self._printer.print_timing("   %6.2f cumulative, %6.2f optimal" %
1007               (cuml_time, cuml_time / int(self._options.child_processes)))
1008         self._printer.print_timing("")
1009
1010         self._print_aggregate_test_statistics(individual_test_timings)
1011         self._print_individual_test_times(individual_test_timings,
1012                                           result_summary)
1013         self._print_directory_timings(directory_test_timings)
1014
1015     def _print_aggregate_test_statistics(self, individual_test_timings):
1016         """Prints aggregate statistics (e.g. median, mean, etc.) for all tests.
1017         Args:
1018           individual_test_timings: List of dump_render_tree_thread.TestStats
1019               for all tests.
1020         """
1021         test_types = []  # Unit tests don't actually produce any timings.
1022         if individual_test_timings:
1023             test_types = individual_test_timings[0].time_for_diffs.keys()
1024         times_for_dump_render_tree = []
1025         times_for_diff_processing = []
1026         times_per_test_type = {}
1027         for test_type in test_types:
1028             times_per_test_type[test_type] = []
1029
1030         for test_stats in individual_test_timings:
1031             times_for_dump_render_tree.append(test_stats.test_run_time)
1032             times_for_diff_processing.append(
1033                 test_stats.total_time_for_all_diffs)
1034             time_for_diffs = test_stats.time_for_diffs
1035             for test_type in test_types:
1036                 times_per_test_type[test_type].append(
1037                     time_for_diffs[test_type])
1038
1039         self._print_statistics_for_test_timings(
1040             "PER TEST TIME IN TESTSHELL (seconds):",
1041             times_for_dump_render_tree)
1042         self._print_statistics_for_test_timings(
1043             "PER TEST DIFF PROCESSING TIMES (seconds):",
1044             times_for_diff_processing)
1045         for test_type in test_types:
1046             self._print_statistics_for_test_timings(
1047                 "PER TEST TIMES BY TEST TYPE: %s" % test_type,
1048                 times_per_test_type[test_type])
1049
1050     def _print_individual_test_times(self, individual_test_timings,
1051                                   result_summary):
1052         """Prints the run times for slow, timeout and crash tests.
1053         Args:
1054           individual_test_timings: List of TestStats for all tests.
1055           result_summary: summary object for test run
1056         """
1057         # Reverse-sort by the time spent in DumpRenderTree.
1058         individual_test_timings.sort(lambda a, b:
1059             cmp(b.test_run_time, a.test_run_time))
1060
1061         num_printed = 0
1062         slow_tests = []
1063         timeout_or_crash_tests = []
1064         unexpected_slow_tests = []
1065         for test_tuple in individual_test_timings:
1066             filename = test_tuple.filename
1067             is_timeout_crash_or_slow = False
1068             if self._test_is_slow(filename):
1069                 is_timeout_crash_or_slow = True
1070                 slow_tests.append(test_tuple)
1071
1072             if filename in result_summary.failures:
1073                 result = result_summary.results[filename].type
1074                 if (result == test_expectations.TIMEOUT or
1075                     result == test_expectations.CRASH):
1076                     is_timeout_crash_or_slow = True
1077                     timeout_or_crash_tests.append(test_tuple)
1078
1079             if (not is_timeout_crash_or_slow and
1080                 num_printed < printing.NUM_SLOW_TESTS_TO_LOG):
1081                 num_printed = num_printed + 1
1082                 unexpected_slow_tests.append(test_tuple)
1083
1084         self._printer.print_timing("")
1085         self._print_test_list_timing("%s slowest tests that are not "
1086             "marked as SLOW and did not timeout/crash:" %
1087             printing.NUM_SLOW_TESTS_TO_LOG, unexpected_slow_tests)
1088         self._printer.print_timing("")
1089         self._print_test_list_timing("Tests marked as SLOW:", slow_tests)
1090         self._printer.print_timing("")
1091         self._print_test_list_timing("Tests that timed out or crashed:",
1092                                      timeout_or_crash_tests)
1093         self._printer.print_timing("")
1094
1095     def _print_test_list_timing(self, title, test_list):
1096         """Print timing info for each test.
1097
1098         Args:
1099           title: section heading
1100           test_list: tests that fall in this section
1101         """
1102         if self._printer.disabled('slowest'):
1103             return
1104
1105         self._printer.print_timing(title)
1106         for test_tuple in test_list:
1107             filename = test_tuple.filename[len(
1108                 self._port.layout_tests_dir()) + 1:]
1109             filename = filename.replace('\\', '/')
1110             test_run_time = round(test_tuple.test_run_time, 1)
1111             self._printer.print_timing("  %s took %s seconds" %
1112                                        (filename, test_run_time))
1113
1114     def _print_directory_timings(self, directory_test_timings):
1115         """Print timing info by directory for any directories that
1116         take > 10 seconds to run.
1117
1118         Args:
1119           directory_test_timing: time info for each directory
1120         """
1121         timings = []
1122         for directory in directory_test_timings:
1123             num_tests, time_for_directory = directory_test_timings[directory]
1124             timings.append((round(time_for_directory, 1), directory,
1125                             num_tests))
1126         timings.sort()
1127
1128         self._printer.print_timing("Time to process slowest subdirectories:")
1129         min_seconds_to_print = 10
1130         for timing in timings:
1131             if timing[0] > min_seconds_to_print:
1132                 self._printer.print_timing(
1133                     "  %s took %s seconds to run %s tests." % (timing[1],
1134                     timing[0], timing[2]))
1135         self._printer.print_timing("")
1136
1137     def _print_statistics_for_test_timings(self, title, timings):
1138         """Prints the median, mean and standard deviation of the values in
1139         timings.
1140
1141         Args:
1142           title: Title for these timings.
1143           timings: A list of floats representing times.
1144         """
1145         self._printer.print_timing(title)
1146         timings.sort()
1147
1148         num_tests = len(timings)
1149         if not num_tests:
1150             return
1151         percentile90 = timings[int(.9 * num_tests)]
1152         percentile99 = timings[int(.99 * num_tests)]
1153
1154         if num_tests % 2 == 1:
1155             median = timings[((num_tests - 1) / 2) - 1]
1156         else:
1157             lower = timings[num_tests / 2 - 1]
1158             upper = timings[num_tests / 2]
1159             median = (float(lower + upper)) / 2
1160
1161         mean = sum(timings) / num_tests
1162
1163         for time in timings:
1164             sum_of_deviations = math.pow(time - mean, 2)
1165
1166         std_deviation = math.sqrt(sum_of_deviations / num_tests)
1167         self._printer.print_timing("  Median:          %6.3f" % median)
1168         self._printer.print_timing("  Mean:            %6.3f" % mean)
1169         self._printer.print_timing("  90th percentile: %6.3f" % percentile90)
1170         self._printer.print_timing("  99th percentile: %6.3f" % percentile99)
1171         self._printer.print_timing("  Standard dev:    %6.3f" % std_deviation)
1172         self._printer.print_timing("")
1173
1174     def _print_result_summary(self, result_summary):
1175         """Print a short summary about how many tests passed.
1176
1177         Args:
1178           result_summary: information to log
1179         """
1180         failed = len(result_summary.failures)
1181         skipped = len(
1182             result_summary.tests_by_expectation[test_expectations.SKIP])
1183         total = result_summary.total
1184         passed = total - failed - skipped
1185         pct_passed = 0.0
1186         if total > 0:
1187             pct_passed = float(passed) * 100 / total
1188
1189         self._printer.print_actual("")
1190         self._printer.print_actual("=> Results: %d/%d tests passed (%.1f%%)" %
1191                      (passed, total, pct_passed))
1192         self._printer.print_actual("")
1193         self._print_result_summary_entry(result_summary,
1194             test_expectations.NOW, "Tests to be fixed")
1195
1196         self._printer.print_actual("")
1197         self._print_result_summary_entry(result_summary,
1198             test_expectations.WONTFIX,
1199             "Tests that will only be fixed if they crash (WONTFIX)")
1200         self._printer.print_actual("")
1201
1202     def _print_result_summary_entry(self, result_summary, timeline,
1203                                     heading):
1204         """Print a summary block of results for a particular timeline of test.
1205
1206         Args:
1207           result_summary: summary to print results for
1208           timeline: the timeline to print results for (NOT, WONTFIX, etc.)
1209           heading: a textual description of the timeline
1210         """
1211         total = len(result_summary.tests_by_timeline[timeline])
1212         not_passing = (total -
1213            len(result_summary.tests_by_expectation[test_expectations.PASS] &
1214                result_summary.tests_by_timeline[timeline]))
1215         self._printer.print_actual("=> %s (%d):" % (heading, not_passing))
1216
1217         for result in TestExpectationsFile.EXPECTATION_ORDER:
1218             if result == test_expectations.PASS:
1219                 continue
1220             results = (result_summary.tests_by_expectation[result] &
1221                        result_summary.tests_by_timeline[timeline])
1222             desc = TestExpectationsFile.EXPECTATION_DESCRIPTIONS[result]
1223             if not_passing and len(results):
1224                 pct = len(results) * 100.0 / not_passing
1225                 self._printer.print_actual("  %5d %-24s (%4.1f%%)" %
1226                     (len(results), desc[len(results) != 1], pct))
1227
1228     def _results_html(self, test_files, failures, title="Test Failures", override_time=None):
1229         """
1230         test_files = a list of file paths
1231         failures = dictionary mapping test paths to failure objects
1232         title = title printed at top of test
1233         override_time = current time (used by unit tests)
1234         """
1235         page = """<html>
1236   <head>
1237     <title>Layout Test Results (%(time)s)</title>
1238   </head>
1239   <body>
1240     <h2>%(title)s (%(time)s)</h2>
1241         """ % {'title': title, 'time': override_time or time.asctime()}
1242
1243         for test_file in sorted(test_files):
1244             test_name = self._port.relative_test_filename(test_file)
1245             test_url = self._port.filename_to_uri(test_file)
1246             page += u"<p><a href='%s'>%s</a><br />\n" % (test_url, test_name)
1247             test_failures = failures.get(test_file, [])
1248             for failure in test_failures:
1249                 page += (u"&nbsp;&nbsp;%s<br/>" %
1250                          failure.result_html_output(test_name))
1251             page += "</p>\n"
1252         page += "</body></html>\n"
1253         return page
1254
1255     def _write_results_html_file(self, result_summary):
1256         """Write results.html which is a summary of tests that failed.
1257
1258         Args:
1259           result_summary: a summary of the results :)
1260
1261         Returns:
1262           True if any results were written (since expected failures may be
1263           omitted)
1264         """
1265         # test failures
1266         if self._options.full_results_html:
1267             results_title = "Test Failures"
1268             test_files = result_summary.failures.keys()
1269         else:
1270             results_title = "Unexpected Test Failures"
1271             unexpected_failures = self._get_failures(result_summary,
1272                 include_crashes=True)
1273             test_files = unexpected_failures.keys()
1274         if not len(test_files):
1275             return False
1276
1277         out_filename = os.path.join(self._options.results_directory,
1278                                     "results.html")
1279         with codecs.open(out_filename, "w", "utf-8") as results_file:
1280             html = self._results_html(test_files, result_summary.failures, results_title)
1281             results_file.write(html)
1282
1283         return True
1284
1285     def _show_results_html_file(self):
1286         """Shows the results.html page."""
1287         results_filename = os.path.join(self._options.results_directory,
1288                                         "results.html")
1289         self._port.show_results_html_file(results_filename)
1290
1291
1292 def read_test_files(files):
1293     tests = []
1294     for file in files:
1295         try:
1296             with codecs.open(file, 'r', 'utf-8') as file_contents:
1297                 # FIXME: This could be cleaner using a list comprehension.
1298                 for line in file_contents:
1299                     line = test_expectations.strip_comments(line)
1300                     if line:
1301                         tests.append(line)
1302         except IOError, e:
1303             if e.errno == errno.ENOENT:
1304                 _log.critical('')
1305                 _log.critical('--test-list file "%s" not found' % file)
1306             raise
1307     return tests
1308
1309
1310 def run(port, options, args, regular_output=sys.stderr,
1311         buildbot_output=sys.stdout):
1312     """Run the tests.
1313
1314     Args:
1315       port: Port object for port-specific behavior
1316       options: a dictionary of command line options
1317       args: a list of sub directories or files to test
1318       regular_output: a stream-like object that we can send logging/debug
1319           output to
1320       buildbot_output: a stream-like object that we can write all output that
1321           is intended to be parsed by the buildbot to
1322     Returns:
1323       the number of unexpected results that occurred, or -1 if there is an
1324           error.
1325
1326     """
1327     _set_up_derived_options(port, options)
1328
1329     printer = printing.Printer(port, options, regular_output, buildbot_output,
1330         int(options.child_processes), options.experimental_fully_parallel)
1331     if options.help_printing:
1332         printer.help_printing()
1333         printer.cleanup()
1334         return 0
1335
1336     last_unexpected_results = _gather_unexpected_results(options)
1337     if options.print_last_failures:
1338         printer.write("\n".join(last_unexpected_results) + "\n")
1339         printer.cleanup()
1340         return 0
1341
1342     # We wrap any parts of the run that are slow or likely to raise exceptions
1343     # in a try/finally to ensure that we clean up the logging configuration.
1344     num_unexpected_results = -1
1345     try:
1346         test_runner = TestRunner(port, options, printer)
1347         test_runner._print_config()
1348
1349         printer.print_update("Collecting tests ...")
1350         try:
1351             test_runner.collect_tests(args, last_unexpected_results)
1352         except IOError, e:
1353             if e.errno == errno.ENOENT:
1354                 return -1
1355             raise
1356
1357         printer.print_update("Parsing expectations ...")
1358         if options.lint_test_files:
1359             return test_runner.lint()
1360         test_runner.parse_expectations(port.test_platform_name(),
1361                                        options.configuration == 'Debug')
1362
1363         printer.print_update("Checking build ...")
1364         if not port.check_build(test_runner.needs_http()):
1365             _log.error("Build check failed")
1366             return -1
1367
1368         result_summary = test_runner.set_up_run()
1369         if result_summary:
1370             num_unexpected_results = test_runner.run(result_summary)
1371             test_runner.clean_up_run()
1372             _log.debug("Testing completed, Exit status: %d" %
1373                        num_unexpected_results)
1374     finally:
1375         printer.cleanup()
1376
1377     return num_unexpected_results
1378
1379
1380 def _set_up_derived_options(port_obj, options):
1381     """Sets the options values that depend on other options values."""
1382     if options.worker_model is None:
1383         options.worker_model = port_obj.default_worker_model()
1384     if options.worker_model == 'inline':
1385         if options.child_processes and int(options.child_processes) > 1:
1386             _log.warning("--worker-model=inline overrides --child-processes")
1387         options.child_processes = "1"
1388     if not options.child_processes:
1389         options.child_processes = os.environ.get("WEBKIT_TEST_CHILD_PROCESSES",
1390                                                  str(port_obj.default_child_processes()))
1391
1392     if not options.configuration:
1393         options.configuration = port_obj.default_configuration()
1394
1395     if options.pixel_tests is None:
1396         options.pixel_tests = True
1397
1398     if not options.use_apache:
1399         options.use_apache = sys.platform in ('darwin', 'linux2')
1400
1401     if not os.path.isabs(options.results_directory):
1402         # This normalizes the path to the build dir.
1403         # FIXME: how this happens is not at all obvious; this is a dumb
1404         # interface and should be cleaned up.
1405         options.results_directory = port_obj.results_directory()
1406
1407     if not options.time_out_ms:
1408         if options.configuration == "Debug":
1409             options.time_out_ms = str(2 * TestRunner.DEFAULT_TEST_TIMEOUT_MS)
1410         else:
1411             options.time_out_ms = str(TestRunner.DEFAULT_TEST_TIMEOUT_MS)
1412
1413     options.slow_time_out_ms = str(5 * int(options.time_out_ms))
1414
1415
1416 def _gather_unexpected_results(options):
1417     """Returns the unexpected results from the previous run, if any."""
1418     last_unexpected_results = []
1419     if options.print_last_failures or options.retest_last_failures:
1420         unexpected_results_filename = os.path.join(
1421         options.results_directory, "unexpected_results.json")
1422         with codecs.open(unexpected_results_filename, "r", "utf-8") as file:
1423             results = simplejson.load(file)
1424         last_unexpected_results = results['tests'].keys()
1425     return last_unexpected_results
1426
1427
1428 def _compat_shim_callback(option, opt_str, value, parser):
1429     print "Ignoring unsupported option: %s" % opt_str
1430
1431
1432 def _compat_shim_option(option_name, **kwargs):
1433     return optparse.make_option(option_name, action="callback",
1434         callback=_compat_shim_callback,
1435         help="Ignored, for old-run-webkit-tests compat only.", **kwargs)
1436
1437
1438 def parse_args(args=None):
1439     """Provides a default set of command line args.
1440
1441     Returns a tuple of options, args from optparse"""
1442
1443     # FIXME: All of these options should be stored closer to the code which
1444     # FIXME: actually uses them. configuration_options should move
1445     # FIXME: to WebKitPort and be shared across all scripts.
1446     configuration_options = [
1447         optparse.make_option("-t", "--target", dest="configuration",
1448                              help="(DEPRECATED)"),
1449         # FIXME: --help should display which configuration is default.
1450         optparse.make_option('--debug', action='store_const', const='Debug',
1451                              dest="configuration",
1452                              help='Set the configuration to Debug'),
1453         optparse.make_option('--release', action='store_const',
1454                              const='Release', dest="configuration",
1455                              help='Set the configuration to Release'),
1456         # old-run-webkit-tests also accepts -c, --configuration CONFIGURATION.
1457     ]
1458
1459     print_options = printing.print_options()
1460
1461     # FIXME: These options should move onto the ChromiumPort.
1462     chromium_options = [
1463         optparse.make_option("--chromium", action="store_true", default=False,
1464             help="use the Chromium port"),
1465         optparse.make_option("--startup-dialog", action="store_true",
1466             default=False, help="create a dialog on DumpRenderTree startup"),
1467         optparse.make_option("--gp-fault-error-box", action="store_true",
1468             default=False, help="enable Windows GP fault error box"),
1469         optparse.make_option("--multiple-loads",
1470             type="int", help="turn on multiple loads of each test"),
1471         optparse.make_option("--js-flags",
1472             type="string", help="JavaScript flags to pass to tests"),
1473         optparse.make_option("--nocheck-sys-deps", action="store_true",
1474             default=False,
1475             help="Don't check the system dependencies (themes)"),
1476         optparse.make_option("--use-drt", action="store_true",
1477             default=None,
1478             help="Use DumpRenderTree instead of test_shell"),
1479         optparse.make_option("--accelerated-compositing",
1480             action="store_true",
1481             help="Use hardware-accelated compositing for rendering"),
1482         optparse.make_option("--no-accelerated-compositing",
1483             action="store_false",
1484             dest="accelerated_compositing",
1485             help="Don't use hardware-accelerated compositing for rendering"),
1486         optparse.make_option("--accelerated-2d-canvas",
1487             action="store_true",
1488             help="Use hardware-accelerated 2D Canvas calls"),
1489         optparse.make_option("--no-accelerated-2d-canvas",
1490             action="store_false",
1491             dest="accelerated_2d_canvas",
1492             help="Don't use hardware-accelerated 2D Canvas calls"),
1493     ]
1494
1495     # Missing Mac-specific old-run-webkit-tests options:
1496     # FIXME: Need: -g, --guard for guard malloc support on Mac.
1497     # FIXME: Need: -l --leaks    Enable leaks checking.
1498     # FIXME: Need: --sample-on-timeout Run sample on timeout
1499
1500     old_run_webkit_tests_compat = [
1501         # NRWT doesn't generate results by default anyway.
1502         _compat_shim_option("--no-new-test-results"),
1503         # NRWT doesn't sample on timeout yet anyway.
1504         _compat_shim_option("--no-sample-on-timeout"),
1505         # FIXME: NRWT needs to support remote links eventually.
1506         _compat_shim_option("--use-remote-links-to-tests"),
1507         # FIXME: NRWT doesn't need this option as much since failures are
1508         # designed to be cheap.  We eventually plan to add this support.
1509         _compat_shim_option("--exit-after-n-failures", nargs=1, type="int"),
1510     ]
1511
1512     results_options = [
1513         # NEED for bots: --use-remote-links-to-tests Link to test files
1514         # within the SVN repository in the results.
1515         optparse.make_option("-p", "--pixel-tests", action="store_true",
1516             dest="pixel_tests", help="Enable pixel-to-pixel PNG comparisons"),
1517         optparse.make_option("--no-pixel-tests", action="store_false",
1518             dest="pixel_tests", help="Disable pixel-to-pixel PNG comparisons"),
1519         optparse.make_option("--tolerance",
1520             help="Ignore image differences less than this percentage (some "
1521                 "ports may ignore this option)", type="float"),
1522         optparse.make_option("--results-directory",
1523             default="layout-test-results",
1524             help="Output results directory source dir, relative to Debug or "
1525                  "Release"),
1526         optparse.make_option("--new-baseline", action="store_true",
1527             default=False, help="Save all generated results as new baselines "
1528                  "into the platform directory, overwriting whatever's "
1529                  "already there."),
1530         optparse.make_option("--reset-results", action="store_true",
1531             default=False, help="Reset any existing baselines to the "
1532                  "generated results"),
1533         optparse.make_option("--no-show-results", action="store_false",
1534             default=True, dest="show_results",
1535             help="Don't launch a browser with results after the tests "
1536                  "are done"),
1537         # FIXME: We should have a helper function to do this sort of
1538         # deprectated mapping and automatically log, etc.
1539         optparse.make_option("--noshow-results", action="store_false",
1540             dest="show_results",
1541             help="Deprecated, same as --no-show-results."),
1542         optparse.make_option("--no-launch-safari", action="store_false",
1543             dest="show_results",
1544             help="old-run-webkit-tests compat, same as --noshow-results."),
1545         # old-run-webkit-tests:
1546         # --[no-]launch-safari    Launch (or do not launch) Safari to display
1547         #                         test results (default: launch)
1548         optparse.make_option("--full-results-html", action="store_true",
1549             default=False,
1550             help="Show all failures in results.html, rather than only "
1551                  "regressions"),
1552         optparse.make_option("--clobber-old-results", action="store_true",
1553             default=False, help="Clobbers test results from previous runs."),
1554         optparse.make_option("--platform",
1555             help="Override the platform for expected results"),
1556         optparse.make_option("--no-record-results", action="store_false",
1557             default=True, dest="record_results",
1558             help="Don't record the results."),
1559         # old-run-webkit-tests also has HTTP toggle options:
1560         # --[no-]http                     Run (or do not run) http tests
1561         #                                 (default: run)
1562     ]
1563
1564     test_options = [
1565         optparse.make_option("--build", dest="build",
1566             action="store_true", default=True,
1567             help="Check to ensure the DumpRenderTree build is up-to-date "
1568                  "(default)."),
1569         optparse.make_option("--no-build", dest="build",
1570             action="store_false", help="Don't check to see if the "
1571                                        "DumpRenderTree build is up-to-date."),
1572         optparse.make_option("-n", "--dry-run", action="store_true",
1573             default=False,
1574             help="Do everything but actually run the tests or upload results."),
1575         # old-run-webkit-tests has --valgrind instead of wrapper.
1576         optparse.make_option("--wrapper",
1577             help="wrapper command to insert before invocations of "
1578                  "DumpRenderTree; option is split on whitespace before "
1579                  "running. (Example: --wrapper='valgrind --smc-check=all')"),
1580         # old-run-webkit-tests:
1581         # -i|--ignore-tests               Comma-separated list of directories
1582         #                                 or tests to ignore
1583         optparse.make_option("--test-list", action="append",
1584             help="read list of tests to run from file", metavar="FILE"),
1585         # old-run-webkit-tests uses --skipped==[default|ignore|only]
1586         # instead of --force:
1587         optparse.make_option("--force", action="store_true", default=False,
1588             help="Run all tests, even those marked SKIP in the test list"),
1589         optparse.make_option("--use-apache", action="store_true",
1590             default=False, help="Whether to use apache instead of lighttpd."),
1591         optparse.make_option("--time-out-ms",
1592             help="Set the timeout for each test"),
1593         # old-run-webkit-tests calls --randomize-order --random:
1594         optparse.make_option("--randomize-order", action="store_true",
1595             default=False, help=("Run tests in random order (useful "
1596                                 "for tracking down corruption)")),
1597         optparse.make_option("--run-chunk",
1598             help=("Run a specified chunk (n:l), the nth of len l, "
1599                  "of the layout tests")),
1600         optparse.make_option("--run-part", help=("Run a specified part (n:m), "
1601                   "the nth of m parts, of the layout tests")),
1602         # old-run-webkit-tests calls --batch-size: --nthly n
1603         #   Restart DumpRenderTree every n tests (default: 1000)
1604         optparse.make_option("--batch-size",
1605             help=("Run a the tests in batches (n), after every n tests, "
1606                   "DumpRenderTree is relaunched."), type="int", default=0),
1607         # old-run-webkit-tests calls --run-singly: -1|--singly
1608         # Isolate each test case run (implies --nthly 1 --verbose)
1609         optparse.make_option("--run-singly", action="store_true",
1610             default=False, help="run a separate DumpRenderTree for each test"),
1611         optparse.make_option("--child-processes",
1612             help="Number of DumpRenderTrees to run in parallel."),
1613         # FIXME: Display default number of child processes that will run.
1614         # FIXME: Display default threading model (will be port-specific).
1615         optparse.make_option("--worker-model", action="store",
1616             help=("controls worker model. Valid values are "
1617             "'inline' and 'threads'.")),
1618         optparse.make_option("--experimental-fully-parallel",
1619             action="store_true", default=False,
1620             help="run all tests in parallel"),
1621         # FIXME: Need --exit-after-n-failures N
1622         #      Exit after the first N failures instead of running all tests
1623         # FIXME: Need --exit-after-n-crashes N
1624         #      Exit after the first N crashes instead of running all tests
1625         # FIXME: consider: --iterations n
1626         #      Number of times to run the set of tests (e.g. ABCABCABC)
1627         optparse.make_option("--print-last-failures", action="store_true",
1628             default=False, help="Print the tests in the last run that "
1629             "had unexpected failures (or passes) and then exit."),
1630         optparse.make_option("--retest-last-failures", action="store_true",
1631             default=False, help="re-test the tests in the last run that "
1632             "had unexpected failures (or passes)."),
1633         optparse.make_option("--retry-failures", action="store_true",
1634             default=True,
1635             help="Re-try any tests that produce unexpected results (default)"),
1636         optparse.make_option("--no-retry-failures", action="store_false",
1637             dest="retry_failures",
1638             help="Don't re-try any tests that produce unexpected results."),
1639     ]
1640
1641     misc_options = [
1642         optparse.make_option("--lint-test-files", action="store_true",
1643         default=False, help=("Makes sure the test files parse for all "
1644                             "configurations. Does not run any tests.")),
1645     ]
1646
1647     # FIXME: Move these into json_results_generator.py
1648     results_json_options = [
1649         optparse.make_option("--master-name", help="The name of the buildbot master."),
1650         optparse.make_option("--builder-name", default="DUMMY_BUILDER_NAME",
1651             help=("The name of the builder shown on the waterfall running "
1652                   "this script e.g. WebKit.")),
1653         optparse.make_option("--build-name", default="DUMMY_BUILD_NAME",
1654             help=("The name of the builder used in its path, e.g. "
1655                   "webkit-rel.")),
1656         optparse.make_option("--build-number", default="DUMMY_BUILD_NUMBER",
1657             help=("The build number of the builder running this script.")),
1658         optparse.make_option("--test-results-server", default="",
1659             help=("If specified, upload results json files to this appengine "
1660                   "server.")),
1661         optparse.make_option("--upload-full-results",
1662             action="store_true",
1663             default=False,
1664             help="If true, upload full json results to server."),
1665     ]
1666
1667     option_list = (configuration_options + print_options +
1668                    chromium_options + results_options + test_options +
1669                    misc_options + results_json_options +
1670                    old_run_webkit_tests_compat)
1671     option_parser = optparse.OptionParser(option_list=option_list)
1672
1673     return option_parser.parse_args(args)
1674
1675
1676 def main():
1677     options, args = parse_args()
1678     port_obj = port.get(options.platform, options)
1679     return run(port_obj, options, args)
1680
1681
1682 if '__main__' == __name__:
1683     try:
1684         sys.exit(main())
1685     except KeyboardInterrupt:
1686         # this mirrors what the shell normally does
1687         sys.exit(signal.SIGINT + 128)