426d39d9f3936465bc149f719a16055f181fa424
[WebKit.git] / Tools / Scripts / webkitpy / layout_tests / run_webkit_tests.py
1 #!/usr/bin/env python
2 # Copyright (C) 2010 Google Inc. All rights reserved.
3 # Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Szeged
4 #
5 # Redistribution and use in source and binary forms, with or without
6 # modification, are permitted provided that the following conditions are
7 # met:
8 #
9 #     * Redistributions of source code must retain the above copyright
10 # notice, this list of conditions and the following disclaimer.
11 #     * Redistributions in binary form must reproduce the above
12 # copyright notice, this list of conditions and the following disclaimer
13 # in the documentation and/or other materials provided with the
14 # distribution.
15 #     * Neither the name of Google Inc. nor the names of its
16 # contributors may be used to endorse or promote products derived from
17 # this software without specific prior written permission.
18 #
19 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30
31 """Run layout tests.
32
33 This is a port of the existing webkit test script run-webkit-tests.
34
35 The TestRunner class runs a series of tests (TestType interface) against a set
36 of test files.  If a test file fails a TestType, it returns a list TestFailure
37 objects to the TestRunner.  The TestRunner then aggregates the TestFailures to
38 create a final report.
39
40 This script reads several files, if they exist in the test_lists subdirectory
41 next to this script itself.  Each should contain a list of paths to individual
42 tests or entire subdirectories of tests, relative to the outermost test
43 directory.  Entire lines starting with '//' (comments) will be ignored.
44
45 For details of the files' contents and purposes, see test_lists/README.
46 """
47
48 from __future__ import with_statement
49
50 import codecs
51 import errno
52 import glob
53 import logging
54 import math
55 import optparse
56 import os
57 import platform
58 import Queue
59 import random
60 import re
61 import shutil
62 import signal
63 import sys
64 import time
65 import traceback
66
67 from layout_package import dump_render_tree_thread
68 from layout_package import json_layout_results_generator
69 from layout_package import message_broker
70 from layout_package import printing
71 from layout_package import test_expectations
72 from layout_package import test_failures
73 from layout_package import test_results
74 from layout_package import test_results_uploader
75
76 from webkitpy.common.system import user
77 from webkitpy.thirdparty import simplejson
78 from webkitpy.tool import grammar
79
80 import port
81
82 _log = logging.getLogger("webkitpy.layout_tests.run_webkit_tests")
83
84 # Builder base URL where we have the archived test results.
85 BUILDER_BASE_URL = "http://build.chromium.org/buildbot/layout_test_results/"
86
87 LAYOUT_TESTS_DIRECTORY = "LayoutTests" + os.sep
88
89 TestExpectationsFile = test_expectations.TestExpectationsFile
90
91
92 class TestInput:
93     """Groups information about a test for easy passing of data."""
94
95     def __init__(self, filename, timeout):
96         """Holds the input parameters for a test.
97         Args:
98           filename: Full path to the test.
99           timeout: Timeout in msecs the driver should use while running the test
100           """
101         # FIXME: filename should really be test_name as a relative path.
102         self.filename = filename
103         self.timeout = timeout
104         # The image_hash is used to avoid doing an image dump if the
105         # checksums match. The image_hash is set later, and only if it is needed
106         # for the test.
107         self.image_hash = None
108
109
110 class TestRunInterruptedException(Exception):
111     """Raised when a test run should be stopped immediately."""
112     def __init__(self, reason):
113         self.reason = reason
114
115
116 class ResultSummary(object):
117     """A class for partitioning the test results we get into buckets.
118
119     This class is basically a glorified struct and it's private to this file
120     so we don't bother with any information hiding."""
121
122     def __init__(self, expectations, test_files):
123         self.total = len(test_files)
124         self.remaining = self.total
125         self.expectations = expectations
126         self.expected = 0
127         self.unexpected = 0
128         self.unexpected_failures = 0
129         self.unexpected_crashes_or_timeouts = 0
130         self.tests_by_expectation = {}
131         self.tests_by_timeline = {}
132         self.results = {}
133         self.unexpected_results = {}
134         self.failures = {}
135         self.tests_by_expectation[test_expectations.SKIP] = set()
136         for expectation in TestExpectationsFile.EXPECTATIONS.values():
137             self.tests_by_expectation[expectation] = set()
138         for timeline in TestExpectationsFile.TIMELINES.values():
139             self.tests_by_timeline[timeline] = (
140                 expectations.get_tests_with_timeline(timeline))
141
142     def add(self, result, expected):
143         """Add a TestResult into the appropriate bin.
144
145         Args:
146           result: TestResult
147           expected: whether the result was what we expected it to be.
148         """
149
150         self.tests_by_expectation[result.type].add(result.filename)
151         self.results[result.filename] = result
152         self.remaining -= 1
153         if len(result.failures):
154             self.failures[result.filename] = result.failures
155         if expected:
156             self.expected += 1
157         else:
158             self.unexpected_results[result.filename] = result.type
159             self.unexpected += 1
160             if len(result.failures):
161                 self.unexpected_failures += 1
162             if result.type == test_expectations.CRASH or result.type == test_expectations.TIMEOUT:
163                 self.unexpected_crashes_or_timeouts += 1
164
165
166 def summarize_unexpected_results(port_obj, expectations, result_summary,
167                                  retry_summary):
168     """Summarize any unexpected results as a dict.
169
170     FIXME: split this data structure into a separate class?
171
172     Args:
173         port_obj: interface to port-specific hooks
174         expectations: test_expectations.TestExpectations object
175         result_summary: summary object from initial test runs
176         retry_summary: summary object from final test run of retried tests
177     Returns:
178         A dictionary containing a summary of the unexpected results from the
179         run, with the following fields:
180         'version': a version indicator (1 in this version)
181         'fixable': # of fixable tests (NOW - PASS)
182         'skipped': # of skipped tests (NOW & SKIPPED)
183         'num_regressions': # of non-flaky failures
184         'num_flaky': # of flaky failures
185         'num_passes': # of unexpected passes
186         'tests': a dict of tests -> {'expected': '...', 'actual': '...'}
187     """
188     results = {}
189     results['version'] = 1
190
191     tbe = result_summary.tests_by_expectation
192     tbt = result_summary.tests_by_timeline
193     results['fixable'] = len(tbt[test_expectations.NOW] -
194                                 tbe[test_expectations.PASS])
195     results['skipped'] = len(tbt[test_expectations.NOW] &
196                                 tbe[test_expectations.SKIP])
197
198     num_passes = 0
199     num_flaky = 0
200     num_regressions = 0
201     keywords = {}
202     for k, v in TestExpectationsFile.EXPECTATIONS.iteritems():
203         keywords[v] = k.upper()
204
205     tests = {}
206     for filename, result in result_summary.unexpected_results.iteritems():
207         # Note that if a test crashed in the original run, we ignore
208         # whether or not it crashed when we retried it (if we retried it),
209         # and always consider the result not flaky.
210         test = port_obj.relative_test_filename(filename)
211         expected = expectations.get_expectations_string(filename)
212         actual = [keywords[result]]
213
214         if result == test_expectations.PASS:
215             num_passes += 1
216         elif result == test_expectations.CRASH:
217             num_regressions += 1
218         else:
219             if filename not in retry_summary.unexpected_results:
220                 actual.extend(expectations.get_expectations_string(
221                     filename).split(" "))
222                 num_flaky += 1
223             else:
224                 retry_result = retry_summary.unexpected_results[filename]
225                 if result != retry_result:
226                     actual.append(keywords[retry_result])
227                     num_flaky += 1
228                 else:
229                     num_regressions += 1
230
231         tests[test] = {}
232         tests[test]['expected'] = expected
233         tests[test]['actual'] = " ".join(actual)
234
235     results['tests'] = tests
236     results['num_passes'] = num_passes
237     results['num_flaky'] = num_flaky
238     results['num_regressions'] = num_regressions
239
240     return results
241
242
243 class TestRunner:
244     """A class for managing running a series of tests on a series of layout
245     test files."""
246
247     HTTP_SUBDIR = os.sep.join(['', 'http', ''])
248     WEBSOCKET_SUBDIR = os.sep.join(['', 'websocket', ''])
249
250     # The per-test timeout in milliseconds, if no --time-out-ms option was
251     # given to run_webkit_tests. This should correspond to the default timeout
252     # in DumpRenderTree.
253     DEFAULT_TEST_TIMEOUT_MS = 6 * 1000
254
255     def __init__(self, port, options, printer):
256         """Initialize test runner data structures.
257
258         Args:
259           port: an object implementing port-specific
260           options: a dictionary of command line options
261           printer: a Printer object to record updates to.
262         """
263         self._port = port
264         self._options = options
265         self._printer = printer
266         self._message_broker = None
267
268         # disable wss server. need to install pyOpenSSL on buildbots.
269         # self._websocket_secure_server = websocket_server.PyWebSocket(
270         #        options.results_directory, use_tls=True, port=9323)
271
272         # a set of test files, and the same tests as a list
273         self._test_files = set()
274         self._test_files_list = None
275         self._result_queue = Queue.Queue()
276         self._retrying = False
277
278     def collect_tests(self, args, last_unexpected_results):
279         """Find all the files to test.
280
281         Args:
282           args: list of test arguments from the command line
283           last_unexpected_results: list of unexpected results to retest, if any
284
285         """
286         paths = [self._strip_test_dir_prefix(arg) for arg in args if arg and arg != '']
287         paths += last_unexpected_results
288         if self._options.test_list:
289             paths += read_test_files(self._options.test_list)
290         self._test_files = self._port.tests(paths)
291
292     def _strip_test_dir_prefix(self, path):
293         if path.startswith(LAYOUT_TESTS_DIRECTORY):
294             return path[len(LAYOUT_TESTS_DIRECTORY):]
295         return path
296
297     def lint(self):
298         lint_failed = False
299
300         # Creating the expecations for each platform/configuration pair does
301         # all the test list parsing and ensures it's correct syntax (e.g. no
302         # dupes).
303         for platform_name in self._port.test_platform_names():
304             try:
305                 self.parse_expectations(platform_name, is_debug_mode=True)
306             except test_expectations.ParseError:
307                 lint_failed = True
308             try:
309                 self.parse_expectations(platform_name, is_debug_mode=False)
310             except test_expectations.ParseError:
311                 lint_failed = True
312
313         self._printer.write("")
314         if lint_failed:
315             _log.error("Lint failed.")
316             return -1
317
318         _log.info("Lint succeeded.")
319         return 0
320
321     def parse_expectations(self, test_platform_name, is_debug_mode):
322         """Parse the expectations from the test_list files and return a data
323         structure holding them. Throws an error if the test_list files have
324         invalid syntax."""
325         if self._options.lint_test_files:
326             test_files = None
327         else:
328             test_files = self._test_files
329
330         expectations_str = self._port.test_expectations()
331         overrides_str = self._port.test_expectations_overrides()
332         self._expectations = test_expectations.TestExpectations(
333             self._port, test_files, expectations_str, test_platform_name,
334             is_debug_mode, self._options.lint_test_files,
335             overrides=overrides_str)
336         return self._expectations
337
338
339     def prepare_lists_and_print_output(self):
340         """Create appropriate subsets of test lists and returns a
341         ResultSummary object. Also prints expected test counts.
342         """
343
344         # Remove skipped - both fixable and ignored - files from the
345         # top-level list of files to test.
346         num_all_test_files = len(self._test_files)
347         self._printer.print_expected("Found:  %d tests" %
348                                      (len(self._test_files)))
349         if not num_all_test_files:
350             _log.critical('No tests to run.')
351             return None
352
353         skipped = set()
354         if num_all_test_files > 1 and not self._options.force:
355             skipped = self._expectations.get_tests_with_result_type(
356                            test_expectations.SKIP)
357             self._test_files -= skipped
358
359         # Create a sorted list of test files so the subset chunk,
360         # if used, contains alphabetically consecutive tests.
361         self._test_files_list = list(self._test_files)
362         if self._options.randomize_order:
363             random.shuffle(self._test_files_list)
364         else:
365             self._test_files_list.sort()
366
367         # If the user specifies they just want to run a subset of the tests,
368         # just grab a subset of the non-skipped tests.
369         if self._options.run_chunk or self._options.run_part:
370             chunk_value = self._options.run_chunk or self._options.run_part
371             test_files = self._test_files_list
372             try:
373                 (chunk_num, chunk_len) = chunk_value.split(":")
374                 chunk_num = int(chunk_num)
375                 assert(chunk_num >= 0)
376                 test_size = int(chunk_len)
377                 assert(test_size > 0)
378             except:
379                 _log.critical("invalid chunk '%s'" % chunk_value)
380                 return None
381
382             # Get the number of tests
383             num_tests = len(test_files)
384
385             # Get the start offset of the slice.
386             if self._options.run_chunk:
387                 chunk_len = test_size
388                 # In this case chunk_num can be really large. We need
389                 # to make the slave fit in the current number of tests.
390                 slice_start = (chunk_num * chunk_len) % num_tests
391             else:
392                 # Validate the data.
393                 assert(test_size <= num_tests)
394                 assert(chunk_num <= test_size)
395
396                 # To count the chunk_len, and make sure we don't skip
397                 # some tests, we round to the next value that fits exactly
398                 # all the parts.
399                 rounded_tests = num_tests
400                 if rounded_tests % test_size != 0:
401                     rounded_tests = (num_tests + test_size -
402                                      (num_tests % test_size))
403
404                 chunk_len = rounded_tests / test_size
405                 slice_start = chunk_len * (chunk_num - 1)
406                 # It does not mind if we go over test_size.
407
408             # Get the end offset of the slice.
409             slice_end = min(num_tests, slice_start + chunk_len)
410
411             files = test_files[slice_start:slice_end]
412
413             tests_run_msg = 'Running: %d tests (chunk slice [%d:%d] of %d)' % (
414                 (slice_end - slice_start), slice_start, slice_end, num_tests)
415             self._printer.print_expected(tests_run_msg)
416
417             # If we reached the end and we don't have enough tests, we run some
418             # from the beginning.
419             if slice_end - slice_start < chunk_len:
420                 extra = chunk_len - (slice_end - slice_start)
421                 extra_msg = ('   last chunk is partial, appending [0:%d]' %
422                             extra)
423                 self._printer.print_expected(extra_msg)
424                 tests_run_msg += "\n" + extra_msg
425                 files.extend(test_files[0:extra])
426             tests_run_filename = os.path.join(self._options.results_directory,
427                                               "tests_run.txt")
428             with codecs.open(tests_run_filename, "w", "utf-8") as file:
429                 file.write(tests_run_msg + "\n")
430
431             len_skip_chunk = int(len(files) * len(skipped) /
432                                  float(len(self._test_files)))
433             skip_chunk_list = list(skipped)[0:len_skip_chunk]
434             skip_chunk = set(skip_chunk_list)
435
436             # Update expectations so that the stats are calculated correctly.
437             # We need to pass a list that includes the right # of skipped files
438             # to ParseExpectations so that ResultSummary() will get the correct
439             # stats. So, we add in the subset of skipped files, and then
440             # subtract them back out.
441             self._test_files_list = files + skip_chunk_list
442             self._test_files = set(self._test_files_list)
443
444             self._expectations = self.parse_expectations(
445                 self._port.test_platform_name(),
446                 self._options.configuration == 'Debug')
447
448             self._test_files = set(files)
449             self._test_files_list = files
450         else:
451             skip_chunk = skipped
452
453         result_summary = ResultSummary(self._expectations,
454             self._test_files | skip_chunk)
455         self._print_expected_results_of_type(result_summary,
456             test_expectations.PASS, "passes")
457         self._print_expected_results_of_type(result_summary,
458             test_expectations.FAIL, "failures")
459         self._print_expected_results_of_type(result_summary,
460             test_expectations.FLAKY, "flaky")
461         self._print_expected_results_of_type(result_summary,
462             test_expectations.SKIP, "skipped")
463
464         if self._options.force:
465             self._printer.print_expected('Running all tests, including '
466                                          'skips (--force)')
467         else:
468             # Note that we don't actually run the skipped tests (they were
469             # subtracted out of self._test_files, above), but we stub out the
470             # results here so the statistics can remain accurate.
471             for test in skip_chunk:
472                 result = test_results.TestResult(test,
473                     failures=[], test_run_time=0, total_time_for_all_diffs=0,
474                     time_for_diffs=0)
475                 result.type = test_expectations.SKIP
476                 result_summary.add(result, expected=True)
477         self._printer.print_expected('')
478
479         return result_summary
480
481     def _get_dir_for_test_file(self, test_file):
482         """Returns the highest-level directory by which to shard the given
483         test file."""
484         index = test_file.rfind(os.sep + LAYOUT_TESTS_DIRECTORY)
485
486         test_file = test_file[index + len(LAYOUT_TESTS_DIRECTORY):]
487         test_file_parts = test_file.split(os.sep, 1)
488         directory = test_file_parts[0]
489         test_file = test_file_parts[1]
490
491         # The http tests are very stable on mac/linux.
492         # TODO(ojan): Make the http server on Windows be apache so we can
493         # turn shard the http tests there as well. Switching to apache is
494         # what made them stable on linux/mac.
495         return_value = directory
496         while ((directory != 'http' or sys.platform in ('darwin', 'linux2'))
497                 and test_file.find(os.sep) >= 0):
498             test_file_parts = test_file.split(os.sep, 1)
499             directory = test_file_parts[0]
500             return_value = os.path.join(return_value, directory)
501             test_file = test_file_parts[1]
502
503         return return_value
504
505     def _get_test_input_for_file(self, test_file):
506         """Returns the appropriate TestInput object for the file. Mostly this
507         is used for looking up the timeout value (in ms) to use for the given
508         test."""
509         if self._test_is_slow(test_file):
510             return TestInput(test_file, self._options.slow_time_out_ms)
511         return TestInput(test_file, self._options.time_out_ms)
512
513     def _test_requires_lock(self, test_file):
514         """Return True if the test needs to be locked when
515         running multiple copies of NRWTs."""
516         split_path = test_file.split(os.sep)
517         return 'http' in split_path or 'websocket' in split_path
518
519     def _test_is_slow(self, test_file):
520         return self._expectations.has_modifier(test_file,
521                                                test_expectations.SLOW)
522
523     def _shard_tests(self, test_files, use_real_shards):
524         """Groups tests into batches.
525         This helps ensure that tests that depend on each other (aka bad tests!)
526         continue to run together as most cross-tests dependencies tend to
527         occur within the same directory. If use_real_shards is False, we
528         put each (non-HTTP/websocket) test into its own shard for maximum
529         concurrency instead of trying to do any sort of real sharding.
530
531         Return:
532             A list of lists of TestInput objects.
533         """
534         # FIXME: when we added http locking, we changed how this works such
535         # that we always lump all of the HTTP threads into a single shard.
536         # That will slow down experimental-fully-parallel, but it's unclear
537         # what the best alternative is completely revamping how we track
538         # when to grab the lock.
539
540         test_lists = []
541         tests_to_http_lock = []
542         if not use_real_shards:
543             for test_file in test_files:
544                 test_input = self._get_test_input_for_file(test_file)
545                 if self._test_requires_lock(test_file):
546                     tests_to_http_lock.append(test_input)
547                 else:
548                     test_lists.append((".", [test_input]))
549         else:
550             tests_by_dir = {}
551             for test_file in test_files:
552                 directory = self._get_dir_for_test_file(test_file)
553                 test_input = self._get_test_input_for_file(test_file)
554                 if self._test_requires_lock(test_file):
555                     tests_to_http_lock.append(test_input)
556                 else:
557                     tests_by_dir.setdefault(directory, [])
558                     tests_by_dir[directory].append(test_input)
559             # Sort by the number of tests in the dir so that the ones with the
560             # most tests get run first in order to maximize parallelization.
561             # Number of tests is a good enough, but not perfect, approximation
562             # of how long that set of tests will take to run. We can't just use
563             # a PriorityQueue until we move to Python 2.6.
564             for directory in tests_by_dir:
565                 test_list = tests_by_dir[directory]
566                 # Keep the tests in alphabetical order.
567                 # FIXME: Remove once tests are fixed so they can be run in any
568                 # order.
569                 test_list.reverse()
570                 test_list_tuple = (directory, test_list)
571                 test_lists.append(test_list_tuple)
572             test_lists.sort(lambda a, b: cmp(len(b[1]), len(a[1])))
573
574         # Put the http tests first. There are only a couple hundred of them,
575         # but each http test takes a very long time to run, so sorting by the
576         # number of tests doesn't accurately capture how long they take to run.
577         if tests_to_http_lock:
578             tests_to_http_lock.reverse()
579             test_lists.insert(0, ("tests_to_http_lock", tests_to_http_lock))
580
581         return test_lists
582
583     def _contains_tests(self, subdir):
584         for test_file in self._test_files:
585             if test_file.find(subdir) >= 0:
586                 return True
587         return False
588
589     def _num_workers(self):
590         return int(self._options.child_processes)
591
592     def _run_tests(self, file_list, result_summary):
593         """Runs the tests in the file_list.
594
595         Return: A tuple (interrupted, keyboard_interrupted, thread_timings,
596             test_timings, individual_test_timings)
597             interrupted is whether the run was interrupted
598             keyboard_interrupted is whether the interruption was because someone
599               typed Ctrl^C
600             thread_timings is a list of dicts with the total runtime
601               of each thread with 'name', 'num_tests', 'total_time' properties
602             test_timings is a list of timings for each sharded subdirectory
603               of the form [time, directory_name, num_tests]
604             individual_test_timings is a list of run times for each test
605               in the form {filename:filename, test_run_time:test_run_time}
606             result_summary: summary object to populate with the results
607         """
608
609         self._printer.print_update('Sharding tests ...')
610         num_workers = self._num_workers()
611         test_lists = self._shard_tests(file_list,
612             num_workers > 1 and not self._options.experimental_fully_parallel)
613         filename_queue = Queue.Queue()
614         for item in test_lists:
615             filename_queue.put(item)
616
617         self._printer.print_update('Starting %s ...' %
618                                    grammar.pluralize('worker', num_workers))
619         self._message_broker = message_broker.get(self._port, self._options)
620         broker = self._message_broker
621         self._current_filename_queue = filename_queue
622         self._current_result_summary = result_summary
623
624         if not self._options.dry_run:
625             threads = broker.start_workers(self)
626         else:
627             threads = {}
628
629         self._printer.print_update("Starting testing ...")
630         keyboard_interrupted = False
631         interrupted = False
632         if not self._options.dry_run:
633             try:
634                 broker.run_message_loop()
635             except KeyboardInterrupt:
636                 _log.info("Interrupted, exiting")
637                 broker.cancel_workers()
638                 keyboard_interrupted = True
639                 interrupted = True
640             except TestRunInterruptedException, e:
641                 _log.info(e.reason)
642                 broker.cancel_workers()
643                 interrupted = True
644             except:
645                 # Unexpected exception; don't try to clean up workers.
646                 _log.info("Exception raised, exiting")
647                 raise
648
649         thread_timings, test_timings, individual_test_timings = \
650             self._collect_timing_info(threads)
651
652         broker.cleanup()
653         self._message_broker = None
654         return (interrupted, keyboard_interrupted, thread_timings, test_timings,
655                 individual_test_timings)
656
657     def update(self):
658         self.update_summary(self._current_result_summary)
659
660     def _collect_timing_info(self, threads):
661         test_timings = {}
662         individual_test_timings = []
663         thread_timings = []
664
665         for thread in threads:
666             thread_timings.append({'name': thread.getName(),
667                                    'num_tests': thread.get_num_tests(),
668                                    'total_time': thread.get_total_time()})
669             test_timings.update(thread.get_test_group_timing_stats())
670             individual_test_timings.extend(thread.get_test_results())
671
672         return (thread_timings, test_timings, individual_test_timings)
673
674     def needs_http(self):
675         """Returns whether the test runner needs an HTTP server."""
676         return self._contains_tests(self.HTTP_SUBDIR)
677
678     def needs_websocket(self):
679         """Returns whether the test runner needs a WEBSOCKET server."""
680         return self._contains_tests(self.WEBSOCKET_SUBDIR)
681
682     def set_up_run(self):
683         """Configures the system to be ready to run tests.
684
685         Returns a ResultSummary object if we should continue to run tests,
686         or None if we should abort.
687
688         """
689         # This must be started before we check the system dependencies,
690         # since the helper may do things to make the setup correct.
691         self._printer.print_update("Starting helper ...")
692         self._port.start_helper()
693
694         # Check that the system dependencies (themes, fonts, ...) are correct.
695         if not self._options.nocheck_sys_deps:
696             self._printer.print_update("Checking system dependencies ...")
697             if not self._port.check_sys_deps(self.needs_http()):
698                 self._port.stop_helper()
699                 return None
700
701         if self._options.clobber_old_results:
702             self._clobber_old_results()
703
704         # Create the output directory if it doesn't already exist.
705         self._port.maybe_make_directory(self._options.results_directory)
706
707         self._port.setup_test_run()
708
709         self._printer.print_update("Preparing tests ...")
710         result_summary = self.prepare_lists_and_print_output()
711         if not result_summary:
712             return None
713
714         return result_summary
715
716     def run(self, result_summary):
717         """Run all our tests on all our test files.
718
719         For each test file, we run each test type. If there are any failures,
720         we collect them for reporting.
721
722         Args:
723           result_summary: a summary object tracking the test results.
724
725         Return:
726           The number of unexpected results (0 == success)
727         """
728         # gather_test_files() must have been called first to initialize us.
729         # If we didn't find any files to test, we've errored out already in
730         # prepare_lists_and_print_output().
731         assert(len(self._test_files))
732
733         start_time = time.time()
734
735         interrupted, keyboard_interrupted, thread_timings, test_timings, \
736             individual_test_timings = (
737             self._run_tests(self._test_files_list, result_summary))
738
739         # We exclude the crashes from the list of results to retry, because
740         # we want to treat even a potentially flaky crash as an error.
741         failures = self._get_failures(result_summary, include_crashes=False)
742         retry_summary = result_summary
743         while (len(failures) and self._options.retry_failures and
744             not self._retrying and not interrupted):
745             _log.info('')
746             _log.info("Retrying %d unexpected failure(s) ..." % len(failures))
747             _log.info('')
748             self._retrying = True
749             retry_summary = ResultSummary(self._expectations, failures.keys())
750             # Note that we intentionally ignore the return value here.
751             self._run_tests(failures.keys(), retry_summary)
752             failures = self._get_failures(retry_summary, include_crashes=True)
753
754         end_time = time.time()
755
756         self._print_timing_statistics(end_time - start_time,
757                                       thread_timings, test_timings,
758                                       individual_test_timings,
759                                       result_summary)
760
761         self._print_result_summary(result_summary)
762
763         sys.stdout.flush()
764         sys.stderr.flush()
765
766         self._printer.print_one_line_summary(result_summary.total,
767                                              result_summary.expected,
768                                              result_summary.unexpected)
769
770         unexpected_results = summarize_unexpected_results(self._port,
771             self._expectations, result_summary, retry_summary)
772         self._printer.print_unexpected_results(unexpected_results)
773
774         if (self._options.record_results and not self._options.dry_run and
775             not interrupted):
776             # Write the same data to log files and upload generated JSON files
777             # to appengine server.
778             self._upload_json_files(unexpected_results, result_summary,
779                                     individual_test_timings)
780
781         # Write the summary to disk (results.html) and display it if requested.
782         if not self._options.dry_run:
783             wrote_results = self._write_results_html_file(result_summary)
784             if self._options.show_results and wrote_results:
785                 self._show_results_html_file()
786
787         # Now that we've completed all the processing we can, we re-raise
788         # a KeyboardInterrupt if necessary so the caller can handle it.
789         if keyboard_interrupted:
790             raise KeyboardInterrupt
791
792         # Ignore flaky failures and unexpected passes so we don't turn the
793         # bot red for those.
794         return unexpected_results['num_regressions']
795
796     def clean_up_run(self):
797         """Restores the system after we're done running tests."""
798
799         _log.debug("flushing stdout")
800         sys.stdout.flush()
801         _log.debug("flushing stderr")
802         sys.stderr.flush()
803         _log.debug("stopping helper")
804         self._port.stop_helper()
805
806     def update_summary(self, result_summary):
807         """Update the summary and print results with any completed tests."""
808         while True:
809             try:
810                 result = test_results.TestResult.loads(self._result_queue.get_nowait())
811             except Queue.Empty:
812                 return
813
814             expected = self._expectations.matches_an_expected_result(
815                 result.filename, result.type, self._options.pixel_tests)
816             result_summary.add(result, expected)
817             exp_str = self._expectations.get_expectations_string(
818                 result.filename)
819             got_str = self._expectations.expectation_to_string(result.type)
820             self._printer.print_test_result(result, expected, exp_str, got_str)
821             self._printer.print_progress(result_summary, self._retrying,
822                                          self._test_files_list)
823
824             def interrupt_if_at_failure_limit(limit, count, message):
825                 if limit and count >= limit:
826                     raise TestRunInterruptedException(message % count)
827
828             interrupt_if_at_failure_limit(
829                 self._options.exit_after_n_failures,
830                 result_summary.unexpected_failures,
831                 "Aborting run since %d failures were reached")
832             interrupt_if_at_failure_limit(
833                 self._options.exit_after_n_crashes_or_timeouts,
834                 result_summary.unexpected_crashes_or_timeouts,
835                 "Aborting run since %d crashes or timeouts were reached")
836
837
838     def _clobber_old_results(self):
839         # Just clobber the actual test results directories since the other
840         # files in the results directory are explicitly used for cross-run
841         # tracking.
842         self._printer.print_update("Clobbering old results in %s" %
843                                    self._options.results_directory)
844         layout_tests_dir = self._port.layout_tests_dir()
845         possible_dirs = self._port.test_dirs()
846         for dirname in possible_dirs:
847             if os.path.isdir(os.path.join(layout_tests_dir, dirname)):
848                 shutil.rmtree(os.path.join(self._options.results_directory,
849                                            dirname),
850                               ignore_errors=True)
851
852     def _get_failures(self, result_summary, include_crashes):
853         """Filters a dict of results and returns only the failures.
854
855         Args:
856           result_summary: the results of the test run
857           include_crashes: whether crashes are included in the output.
858             We use False when finding the list of failures to retry
859             to see if the results were flaky. Although the crashes may also be
860             flaky, we treat them as if they aren't so that they're not ignored.
861         Returns:
862           a dict of files -> results
863         """
864         failed_results = {}
865         for test, result in result_summary.unexpected_results.iteritems():
866             if (result == test_expectations.PASS or
867                 result == test_expectations.CRASH and not include_crashes):
868                 continue
869             failed_results[test] = result
870
871         return failed_results
872
873     def _upload_json_files(self, unexpected_results, result_summary,
874                         individual_test_timings):
875         """Writes the results of the test run as JSON files into the results
876         dir and upload the files to the appengine server.
877
878         There are three different files written into the results dir:
879           unexpected_results.json: A short list of any unexpected results.
880             This is used by the buildbots to display results.
881           expectations.json: This is used by the flakiness dashboard.
882           results.json: A full list of the results - used by the flakiness
883             dashboard and the aggregate results dashboard.
884
885         Args:
886           unexpected_results: dict of unexpected results
887           result_summary: full summary object
888           individual_test_timings: list of test times (used by the flakiness
889             dashboard).
890         """
891         results_directory = self._options.results_directory
892         _log.debug("Writing JSON files in %s." % results_directory)
893         unexpected_json_path = os.path.join(results_directory, "unexpected_results.json")
894         with codecs.open(unexpected_json_path, "w", "utf-8") as file:
895             simplejson.dump(unexpected_results, file, sort_keys=True, indent=2)
896
897         # Write a json file of the test_expectations.txt file for the layout
898         # tests dashboard.
899         expectations_path = os.path.join(results_directory, "expectations.json")
900         expectations_json = \
901             self._expectations.get_expectations_json_for_all_platforms()
902         with codecs.open(expectations_path, "w", "utf-8") as file:
903             file.write(u"ADD_EXPECTATIONS(%s);" % expectations_json)
904
905         generator = json_layout_results_generator.JSONLayoutResultsGenerator(
906             self._port, self._options.builder_name, self._options.build_name,
907             self._options.build_number, self._options.results_directory,
908             BUILDER_BASE_URL, individual_test_timings,
909             self._expectations, result_summary, self._test_files_list,
910             not self._options.upload_full_results,
911             self._options.test_results_server,
912             "layout-tests",
913             self._options.master_name)
914
915         _log.debug("Finished writing JSON files.")
916
917         json_files = ["expectations.json"]
918         if self._options.upload_full_results:
919             json_files.append("results.json")
920         else:
921             json_files.append("incremental_results.json")
922
923         generator.upload_json_files(json_files)
924
925     def _print_config(self):
926         """Prints the configuration for the test run."""
927         p = self._printer
928         p.print_config("Using port '%s'" % self._port.name())
929         p.print_config("Placing test results in %s" %
930                        self._options.results_directory)
931         if self._options.new_baseline:
932             p.print_config("Placing new baselines in %s" %
933                            self._port.baseline_path())
934         p.print_config("Using %s build" % self._options.configuration)
935         if self._options.pixel_tests:
936             p.print_config("Pixel tests enabled")
937         else:
938             p.print_config("Pixel tests disabled")
939
940         p.print_config("Regular timeout: %s, slow test timeout: %s" %
941                        (self._options.time_out_ms,
942                         self._options.slow_time_out_ms))
943
944         if self._num_workers() == 1:
945             p.print_config("Running one %s" % self._port.driver_name())
946         else:
947             p.print_config("Running %s %ss in parallel" %
948                            (self._options.child_processes,
949                             self._port.driver_name()))
950         p.print_config('Command line: ' +
951                        ' '.join(self._port.driver_cmd_line()))
952         p.print_config("Worker model: %s" % self._options.worker_model)
953         p.print_config("")
954
955     def _print_expected_results_of_type(self, result_summary,
956                                         result_type, result_type_str):
957         """Print the number of the tests in a given result class.
958
959         Args:
960           result_summary - the object containing all the results to report on
961           result_type - the particular result type to report in the summary.
962           result_type_str - a string description of the result_type.
963         """
964         tests = self._expectations.get_tests_with_result_type(result_type)
965         now = result_summary.tests_by_timeline[test_expectations.NOW]
966         wontfix = result_summary.tests_by_timeline[test_expectations.WONTFIX]
967
968         # We use a fancy format string in order to print the data out in a
969         # nicely-aligned table.
970         fmtstr = ("Expect: %%5d %%-8s (%%%dd now, %%%dd wontfix)"
971                   % (self._num_digits(now), self._num_digits(wontfix)))
972         self._printer.print_expected(fmtstr %
973             (len(tests), result_type_str, len(tests & now), len(tests & wontfix)))
974
975     def _num_digits(self, num):
976         """Returns the number of digits needed to represent the length of a
977         sequence."""
978         ndigits = 1
979         if len(num):
980             ndigits = int(math.log10(len(num))) + 1
981         return ndigits
982
983     def _print_timing_statistics(self, total_time, thread_timings,
984                                directory_test_timings, individual_test_timings,
985                                result_summary):
986         """Record timing-specific information for the test run.
987
988         Args:
989           total_time: total elapsed time (in seconds) for the test run
990           thread_timings: wall clock time each thread ran for
991           directory_test_timings: timing by directory
992           individual_test_timings: timing by file
993           result_summary: summary object for the test run
994         """
995         self._printer.print_timing("Test timing:")
996         self._printer.print_timing("  %6.2f total testing time" % total_time)
997         self._printer.print_timing("")
998         self._printer.print_timing("Thread timing:")
999         cuml_time = 0
1000         for t in thread_timings:
1001             self._printer.print_timing("    %10s: %5d tests, %6.2f secs" %
1002                   (t['name'], t['num_tests'], t['total_time']))
1003             cuml_time += t['total_time']
1004         self._printer.print_timing("   %6.2f cumulative, %6.2f optimal" %
1005               (cuml_time, cuml_time / int(self._options.child_processes)))
1006         self._printer.print_timing("")
1007
1008         self._print_aggregate_test_statistics(individual_test_timings)
1009         self._print_individual_test_times(individual_test_timings,
1010                                           result_summary)
1011         self._print_directory_timings(directory_test_timings)
1012
1013     def _print_aggregate_test_statistics(self, individual_test_timings):
1014         """Prints aggregate statistics (e.g. median, mean, etc.) for all tests.
1015         Args:
1016           individual_test_timings: List of TestResults for all tests.
1017         """
1018         test_types = []  # Unit tests don't actually produce any timings.
1019         if individual_test_timings:
1020             test_types = individual_test_timings[0].time_for_diffs.keys()
1021         times_for_dump_render_tree = []
1022         times_for_diff_processing = []
1023         times_per_test_type = {}
1024         for test_type in test_types:
1025             times_per_test_type[test_type] = []
1026
1027         for test_stats in individual_test_timings:
1028             times_for_dump_render_tree.append(test_stats.test_run_time)
1029             times_for_diff_processing.append(
1030                 test_stats.total_time_for_all_diffs)
1031             time_for_diffs = test_stats.time_for_diffs
1032             for test_type in test_types:
1033                 times_per_test_type[test_type].append(
1034                     time_for_diffs[test_type])
1035
1036         self._print_statistics_for_test_timings(
1037             "PER TEST TIME IN TESTSHELL (seconds):",
1038             times_for_dump_render_tree)
1039         self._print_statistics_for_test_timings(
1040             "PER TEST DIFF PROCESSING TIMES (seconds):",
1041             times_for_diff_processing)
1042         for test_type in test_types:
1043             self._print_statistics_for_test_timings(
1044                 "PER TEST TIMES BY TEST TYPE: %s" % test_type,
1045                 times_per_test_type[test_type])
1046
1047     def _print_individual_test_times(self, individual_test_timings,
1048                                   result_summary):
1049         """Prints the run times for slow, timeout and crash tests.
1050         Args:
1051           individual_test_timings: List of TestStats for all tests.
1052           result_summary: summary object for test run
1053         """
1054         # Reverse-sort by the time spent in DumpRenderTree.
1055         individual_test_timings.sort(lambda a, b:
1056             cmp(b.test_run_time, a.test_run_time))
1057
1058         num_printed = 0
1059         slow_tests = []
1060         timeout_or_crash_tests = []
1061         unexpected_slow_tests = []
1062         for test_tuple in individual_test_timings:
1063             filename = test_tuple.filename
1064             is_timeout_crash_or_slow = False
1065             if self._test_is_slow(filename):
1066                 is_timeout_crash_or_slow = True
1067                 slow_tests.append(test_tuple)
1068
1069             if filename in result_summary.failures:
1070                 result = result_summary.results[filename].type
1071                 if (result == test_expectations.TIMEOUT or
1072                     result == test_expectations.CRASH):
1073                     is_timeout_crash_or_slow = True
1074                     timeout_or_crash_tests.append(test_tuple)
1075
1076             if (not is_timeout_crash_or_slow and
1077                 num_printed < printing.NUM_SLOW_TESTS_TO_LOG):
1078                 num_printed = num_printed + 1
1079                 unexpected_slow_tests.append(test_tuple)
1080
1081         self._printer.print_timing("")
1082         self._print_test_list_timing("%s slowest tests that are not "
1083             "marked as SLOW and did not timeout/crash:" %
1084             printing.NUM_SLOW_TESTS_TO_LOG, unexpected_slow_tests)
1085         self._printer.print_timing("")
1086         self._print_test_list_timing("Tests marked as SLOW:", slow_tests)
1087         self._printer.print_timing("")
1088         self._print_test_list_timing("Tests that timed out or crashed:",
1089                                      timeout_or_crash_tests)
1090         self._printer.print_timing("")
1091
1092     def _print_test_list_timing(self, title, test_list):
1093         """Print timing info for each test.
1094
1095         Args:
1096           title: section heading
1097           test_list: tests that fall in this section
1098         """
1099         if self._printer.disabled('slowest'):
1100             return
1101
1102         self._printer.print_timing(title)
1103         for test_tuple in test_list:
1104             filename = test_tuple.filename[len(
1105                 self._port.layout_tests_dir()) + 1:]
1106             filename = filename.replace('\\', '/')
1107             test_run_time = round(test_tuple.test_run_time, 1)
1108             self._printer.print_timing("  %s took %s seconds" %
1109                                        (filename, test_run_time))
1110
1111     def _print_directory_timings(self, directory_test_timings):
1112         """Print timing info by directory for any directories that
1113         take > 10 seconds to run.
1114
1115         Args:
1116           directory_test_timing: time info for each directory
1117         """
1118         timings = []
1119         for directory in directory_test_timings:
1120             num_tests, time_for_directory = directory_test_timings[directory]
1121             timings.append((round(time_for_directory, 1), directory,
1122                             num_tests))
1123         timings.sort()
1124
1125         self._printer.print_timing("Time to process slowest subdirectories:")
1126         min_seconds_to_print = 10
1127         for timing in timings:
1128             if timing[0] > min_seconds_to_print:
1129                 self._printer.print_timing(
1130                     "  %s took %s seconds to run %s tests." % (timing[1],
1131                     timing[0], timing[2]))
1132         self._printer.print_timing("")
1133
1134     def _print_statistics_for_test_timings(self, title, timings):
1135         """Prints the median, mean and standard deviation of the values in
1136         timings.
1137
1138         Args:
1139           title: Title for these timings.
1140           timings: A list of floats representing times.
1141         """
1142         self._printer.print_timing(title)
1143         timings.sort()
1144
1145         num_tests = len(timings)
1146         if not num_tests:
1147             return
1148         percentile90 = timings[int(.9 * num_tests)]
1149         percentile99 = timings[int(.99 * num_tests)]
1150
1151         if num_tests % 2 == 1:
1152             median = timings[((num_tests - 1) / 2) - 1]
1153         else:
1154             lower = timings[num_tests / 2 - 1]
1155             upper = timings[num_tests / 2]
1156             median = (float(lower + upper)) / 2
1157
1158         mean = sum(timings) / num_tests
1159
1160         for time in timings:
1161             sum_of_deviations = math.pow(time - mean, 2)
1162
1163         std_deviation = math.sqrt(sum_of_deviations / num_tests)
1164         self._printer.print_timing("  Median:          %6.3f" % median)
1165         self._printer.print_timing("  Mean:            %6.3f" % mean)
1166         self._printer.print_timing("  90th percentile: %6.3f" % percentile90)
1167         self._printer.print_timing("  99th percentile: %6.3f" % percentile99)
1168         self._printer.print_timing("  Standard dev:    %6.3f" % std_deviation)
1169         self._printer.print_timing("")
1170
1171     def _print_result_summary(self, result_summary):
1172         """Print a short summary about how many tests passed.
1173
1174         Args:
1175           result_summary: information to log
1176         """
1177         failed = len(result_summary.failures)
1178         skipped = len(
1179             result_summary.tests_by_expectation[test_expectations.SKIP])
1180         total = result_summary.total
1181         passed = total - failed - skipped
1182         pct_passed = 0.0
1183         if total > 0:
1184             pct_passed = float(passed) * 100 / total
1185
1186         self._printer.print_actual("")
1187         self._printer.print_actual("=> Results: %d/%d tests passed (%.1f%%)" %
1188                      (passed, total, pct_passed))
1189         self._printer.print_actual("")
1190         self._print_result_summary_entry(result_summary,
1191             test_expectations.NOW, "Tests to be fixed")
1192
1193         self._printer.print_actual("")
1194         self._print_result_summary_entry(result_summary,
1195             test_expectations.WONTFIX,
1196             "Tests that will only be fixed if they crash (WONTFIX)")
1197         self._printer.print_actual("")
1198
1199     def _print_result_summary_entry(self, result_summary, timeline,
1200                                     heading):
1201         """Print a summary block of results for a particular timeline of test.
1202
1203         Args:
1204           result_summary: summary to print results for
1205           timeline: the timeline to print results for (NOT, WONTFIX, etc.)
1206           heading: a textual description of the timeline
1207         """
1208         total = len(result_summary.tests_by_timeline[timeline])
1209         not_passing = (total -
1210            len(result_summary.tests_by_expectation[test_expectations.PASS] &
1211                result_summary.tests_by_timeline[timeline]))
1212         self._printer.print_actual("=> %s (%d):" % (heading, not_passing))
1213
1214         for result in TestExpectationsFile.EXPECTATION_ORDER:
1215             if result == test_expectations.PASS:
1216                 continue
1217             results = (result_summary.tests_by_expectation[result] &
1218                        result_summary.tests_by_timeline[timeline])
1219             desc = TestExpectationsFile.EXPECTATION_DESCRIPTIONS[result]
1220             if not_passing and len(results):
1221                 pct = len(results) * 100.0 / not_passing
1222                 self._printer.print_actual("  %5d %-24s (%4.1f%%)" %
1223                     (len(results), desc[len(results) != 1], pct))
1224
1225     def _results_html(self, test_files, failures, title="Test Failures", override_time=None):
1226         """
1227         test_files = a list of file paths
1228         failures = dictionary mapping test paths to failure objects
1229         title = title printed at top of test
1230         override_time = current time (used by unit tests)
1231         """
1232         page = """<html>
1233   <head>
1234     <title>Layout Test Results (%(time)s)</title>
1235   </head>
1236   <body>
1237     <h2>%(title)s (%(time)s)</h2>
1238         """ % {'title': title, 'time': override_time or time.asctime()}
1239
1240         for test_file in sorted(test_files):
1241             test_name = self._port.relative_test_filename(test_file)
1242             test_url = self._port.filename_to_uri(test_file)
1243             page += u"<p><a href='%s'>%s</a><br />\n" % (test_url, test_name)
1244             test_failures = failures.get(test_file, [])
1245             for failure in test_failures:
1246                 page += (u"&nbsp;&nbsp;%s<br/>" %
1247                          failure.result_html_output(test_name))
1248             page += "</p>\n"
1249         page += "</body></html>\n"
1250         return page
1251
1252     def _write_results_html_file(self, result_summary):
1253         """Write results.html which is a summary of tests that failed.
1254
1255         Args:
1256           result_summary: a summary of the results :)
1257
1258         Returns:
1259           True if any results were written (since expected failures may be
1260           omitted)
1261         """
1262         # test failures
1263         if self._options.full_results_html:
1264             results_title = "Test Failures"
1265             test_files = result_summary.failures.keys()
1266         else:
1267             results_title = "Unexpected Test Failures"
1268             unexpected_failures = self._get_failures(result_summary,
1269                 include_crashes=True)
1270             test_files = unexpected_failures.keys()
1271         if not len(test_files):
1272             return False
1273
1274         out_filename = os.path.join(self._options.results_directory,
1275                                     "results.html")
1276         with codecs.open(out_filename, "w", "utf-8") as results_file:
1277             html = self._results_html(test_files, result_summary.failures, results_title)
1278             results_file.write(html)
1279
1280         return True
1281
1282     def _show_results_html_file(self):
1283         """Shows the results.html page."""
1284         results_filename = os.path.join(self._options.results_directory,
1285                                         "results.html")
1286         self._port.show_results_html_file(results_filename)
1287
1288
1289 def read_test_files(files):
1290     tests = []
1291     for file in files:
1292         try:
1293             with codecs.open(file, 'r', 'utf-8') as file_contents:
1294                 # FIXME: This could be cleaner using a list comprehension.
1295                 for line in file_contents:
1296                     line = test_expectations.strip_comments(line)
1297                     if line:
1298                         tests.append(line)
1299         except IOError, e:
1300             if e.errno == errno.ENOENT:
1301                 _log.critical('')
1302                 _log.critical('--test-list file "%s" not found' % file)
1303             raise
1304     return tests
1305
1306
1307 def run(port, options, args, regular_output=sys.stderr,
1308         buildbot_output=sys.stdout):
1309     """Run the tests.
1310
1311     Args:
1312       port: Port object for port-specific behavior
1313       options: a dictionary of command line options
1314       args: a list of sub directories or files to test
1315       regular_output: a stream-like object that we can send logging/debug
1316           output to
1317       buildbot_output: a stream-like object that we can write all output that
1318           is intended to be parsed by the buildbot to
1319     Returns:
1320       the number of unexpected results that occurred, or -1 if there is an
1321           error.
1322
1323     """
1324     warnings = _set_up_derived_options(port, options)
1325
1326     printer = printing.Printer(port, options, regular_output, buildbot_output,
1327         int(options.child_processes), options.experimental_fully_parallel)
1328     for w in warnings:
1329         _log.warning(w)
1330
1331     if options.help_printing:
1332         printer.help_printing()
1333         printer.cleanup()
1334         return 0
1335
1336     last_unexpected_results = _gather_unexpected_results(options)
1337     if options.print_last_failures:
1338         printer.write("\n".join(last_unexpected_results) + "\n")
1339         printer.cleanup()
1340         return 0
1341
1342     # We wrap any parts of the run that are slow or likely to raise exceptions
1343     # in a try/finally to ensure that we clean up the logging configuration.
1344     num_unexpected_results = -1
1345     try:
1346         test_runner = TestRunner(port, options, printer)
1347         test_runner._print_config()
1348
1349         printer.print_update("Collecting tests ...")
1350         try:
1351             test_runner.collect_tests(args, last_unexpected_results)
1352         except IOError, e:
1353             if e.errno == errno.ENOENT:
1354                 return -1
1355             raise
1356
1357         printer.print_update("Parsing expectations ...")
1358         if options.lint_test_files:
1359             return test_runner.lint()
1360         test_runner.parse_expectations(port.test_platform_name(),
1361                                        options.configuration == 'Debug')
1362
1363         printer.print_update("Checking build ...")
1364         if not port.check_build(test_runner.needs_http()):
1365             _log.error("Build check failed")
1366             return -1
1367
1368         result_summary = test_runner.set_up_run()
1369         if result_summary:
1370             num_unexpected_results = test_runner.run(result_summary)
1371             test_runner.clean_up_run()
1372             _log.debug("Testing completed, Exit status: %d" %
1373                        num_unexpected_results)
1374     finally:
1375         printer.cleanup()
1376
1377     return num_unexpected_results
1378
1379
1380 def _set_up_derived_options(port_obj, options):
1381     """Sets the options values that depend on other options values."""
1382     # We return a list of warnings to print after the printer is initialized.
1383     warnings = []
1384
1385     if options.worker_model == 'old-inline':
1386         if options.child_processes and int(options.child_processes) > 1:
1387             warnings.append("--worker-model=old-inline overrides --child-processes")
1388         options.child_processes = "1"
1389     if not options.child_processes:
1390         options.child_processes = os.environ.get("WEBKIT_TEST_CHILD_PROCESSES",
1391                                                  str(port_obj.default_child_processes()))
1392
1393     if not options.configuration:
1394         options.configuration = port_obj.default_configuration()
1395
1396     if options.pixel_tests is None:
1397         options.pixel_tests = True
1398
1399     if not options.use_apache:
1400         options.use_apache = sys.platform in ('darwin', 'linux2')
1401
1402     if not os.path.isabs(options.results_directory):
1403         # This normalizes the path to the build dir.
1404         # FIXME: how this happens is not at all obvious; this is a dumb
1405         # interface and should be cleaned up.
1406         options.results_directory = port_obj.results_directory()
1407
1408     if not options.time_out_ms:
1409         if options.configuration == "Debug":
1410             options.time_out_ms = str(2 * TestRunner.DEFAULT_TEST_TIMEOUT_MS)
1411         else:
1412             options.time_out_ms = str(TestRunner.DEFAULT_TEST_TIMEOUT_MS)
1413
1414     options.slow_time_out_ms = str(5 * int(options.time_out_ms))
1415     return warnings
1416
1417
1418 def _gather_unexpected_results(options):
1419     """Returns the unexpected results from the previous run, if any."""
1420     last_unexpected_results = []
1421     if options.print_last_failures or options.retest_last_failures:
1422         unexpected_results_filename = os.path.join(
1423         options.results_directory, "unexpected_results.json")
1424         with codecs.open(unexpected_results_filename, "r", "utf-8") as file:
1425             results = simplejson.load(file)
1426         last_unexpected_results = results['tests'].keys()
1427     return last_unexpected_results
1428
1429
1430 def _compat_shim_callback(option, opt_str, value, parser):
1431     print "Ignoring unsupported option: %s" % opt_str
1432
1433
1434 def _compat_shim_option(option_name, **kwargs):
1435     return optparse.make_option(option_name, action="callback",
1436         callback=_compat_shim_callback,
1437         help="Ignored, for old-run-webkit-tests compat only.", **kwargs)
1438
1439
1440 def parse_args(args=None):
1441     """Provides a default set of command line args.
1442
1443     Returns a tuple of options, args from optparse"""
1444
1445     # FIXME: All of these options should be stored closer to the code which
1446     # FIXME: actually uses them. configuration_options should move
1447     # FIXME: to WebKitPort and be shared across all scripts.
1448     configuration_options = [
1449         optparse.make_option("-t", "--target", dest="configuration",
1450                              help="(DEPRECATED)"),
1451         # FIXME: --help should display which configuration is default.
1452         optparse.make_option('--debug', action='store_const', const='Debug',
1453                              dest="configuration",
1454                              help='Set the configuration to Debug'),
1455         optparse.make_option('--release', action='store_const',
1456                              const='Release', dest="configuration",
1457                              help='Set the configuration to Release'),
1458         # old-run-webkit-tests also accepts -c, --configuration CONFIGURATION.
1459     ]
1460
1461     print_options = printing.print_options()
1462
1463     # FIXME: These options should move onto the ChromiumPort.
1464     chromium_options = [
1465         optparse.make_option("--chromium", action="store_true", default=False,
1466             help="use the Chromium port"),
1467         optparse.make_option("--startup-dialog", action="store_true",
1468             default=False, help="create a dialog on DumpRenderTree startup"),
1469         optparse.make_option("--gp-fault-error-box", action="store_true",
1470             default=False, help="enable Windows GP fault error box"),
1471         optparse.make_option("--multiple-loads",
1472             type="int", help="turn on multiple loads of each test"),
1473         optparse.make_option("--js-flags",
1474             type="string", help="JavaScript flags to pass to tests"),
1475         optparse.make_option("--nocheck-sys-deps", action="store_true",
1476             default=False,
1477             help="Don't check the system dependencies (themes)"),
1478         optparse.make_option("--use-test-shell", action="store_true",
1479             default=False,
1480             help="Use test_shell instead of DRT"),
1481         optparse.make_option("--accelerated-compositing",
1482             action="store_true",
1483             help="Use hardware-accelated compositing for rendering"),
1484         optparse.make_option("--no-accelerated-compositing",
1485             action="store_false",
1486             dest="accelerated_compositing",
1487             help="Don't use hardware-accelerated compositing for rendering"),
1488         optparse.make_option("--accelerated-2d-canvas",
1489             action="store_true",
1490             help="Use hardware-accelerated 2D Canvas calls"),
1491         optparse.make_option("--no-accelerated-2d-canvas",
1492             action="store_false",
1493             dest="accelerated_2d_canvas",
1494             help="Don't use hardware-accelerated 2D Canvas calls"),
1495     ]
1496
1497     # Missing Mac-specific old-run-webkit-tests options:
1498     # FIXME: Need: -g, --guard for guard malloc support on Mac.
1499     # FIXME: Need: -l --leaks    Enable leaks checking.
1500     # FIXME: Need: --sample-on-timeout Run sample on timeout
1501
1502     old_run_webkit_tests_compat = [
1503         # NRWT doesn't generate results by default anyway.
1504         _compat_shim_option("--no-new-test-results"),
1505         # NRWT doesn't sample on timeout yet anyway.
1506         _compat_shim_option("--no-sample-on-timeout"),
1507         # FIXME: NRWT needs to support remote links eventually.
1508         _compat_shim_option("--use-remote-links-to-tests"),
1509     ]
1510
1511     results_options = [
1512         # NEED for bots: --use-remote-links-to-tests Link to test files
1513         # within the SVN repository in the results.
1514         optparse.make_option("-p", "--pixel-tests", action="store_true",
1515             dest="pixel_tests", help="Enable pixel-to-pixel PNG comparisons"),
1516         optparse.make_option("--no-pixel-tests", action="store_false",
1517             dest="pixel_tests", help="Disable pixel-to-pixel PNG comparisons"),
1518         optparse.make_option("--tolerance",
1519             help="Ignore image differences less than this percentage (some "
1520                 "ports may ignore this option)", type="float"),
1521         optparse.make_option("--results-directory",
1522             default="layout-test-results",
1523             help="Output results directory source dir, relative to Debug or "
1524                  "Release"),
1525         optparse.make_option("--new-baseline", action="store_true",
1526             default=False, help="Save all generated results as new baselines "
1527                  "into the platform directory, overwriting whatever's "
1528                  "already there."),
1529         optparse.make_option("--reset-results", action="store_true",
1530             default=False, help="Reset any existing baselines to the "
1531                  "generated results"),
1532         optparse.make_option("--no-show-results", action="store_false",
1533             default=True, dest="show_results",
1534             help="Don't launch a browser with results after the tests "
1535                  "are done"),
1536         # FIXME: We should have a helper function to do this sort of
1537         # deprectated mapping and automatically log, etc.
1538         optparse.make_option("--noshow-results", action="store_false",
1539             dest="show_results",
1540             help="Deprecated, same as --no-show-results."),
1541         optparse.make_option("--no-launch-safari", action="store_false",
1542             dest="show_results",
1543             help="old-run-webkit-tests compat, same as --noshow-results."),
1544         # old-run-webkit-tests:
1545         # --[no-]launch-safari    Launch (or do not launch) Safari to display
1546         #                         test results (default: launch)
1547         optparse.make_option("--full-results-html", action="store_true",
1548             default=False,
1549             help="Show all failures in results.html, rather than only "
1550                  "regressions"),
1551         optparse.make_option("--clobber-old-results", action="store_true",
1552             default=False, help="Clobbers test results from previous runs."),
1553         optparse.make_option("--platform",
1554             help="Override the platform for expected results"),
1555         optparse.make_option("--no-record-results", action="store_false",
1556             default=True, dest="record_results",
1557             help="Don't record the results."),
1558         # old-run-webkit-tests also has HTTP toggle options:
1559         # --[no-]http                     Run (or do not run) http tests
1560         #                                 (default: run)
1561     ]
1562
1563     test_options = [
1564         optparse.make_option("--build", dest="build",
1565             action="store_true", default=True,
1566             help="Check to ensure the DumpRenderTree build is up-to-date "
1567                  "(default)."),
1568         optparse.make_option("--no-build", dest="build",
1569             action="store_false", help="Don't check to see if the "
1570                                        "DumpRenderTree build is up-to-date."),
1571         optparse.make_option("-n", "--dry-run", action="store_true",
1572             default=False,
1573             help="Do everything but actually run the tests or upload results."),
1574         # old-run-webkit-tests has --valgrind instead of wrapper.
1575         optparse.make_option("--wrapper",
1576             help="wrapper command to insert before invocations of "
1577                  "DumpRenderTree; option is split on whitespace before "
1578                  "running. (Example: --wrapper='valgrind --smc-check=all')"),
1579         # old-run-webkit-tests:
1580         # -i|--ignore-tests               Comma-separated list of directories
1581         #                                 or tests to ignore
1582         optparse.make_option("--test-list", action="append",
1583             help="read list of tests to run from file", metavar="FILE"),
1584         # old-run-webkit-tests uses --skipped==[default|ignore|only]
1585         # instead of --force:
1586         optparse.make_option("--force", action="store_true", default=False,
1587             help="Run all tests, even those marked SKIP in the test list"),
1588         optparse.make_option("--use-apache", action="store_true",
1589             default=False, help="Whether to use apache instead of lighttpd."),
1590         optparse.make_option("--time-out-ms",
1591             help="Set the timeout for each test"),
1592         # old-run-webkit-tests calls --randomize-order --random:
1593         optparse.make_option("--randomize-order", action="store_true",
1594             default=False, help=("Run tests in random order (useful "
1595                                 "for tracking down corruption)")),
1596         optparse.make_option("--run-chunk",
1597             help=("Run a specified chunk (n:l), the nth of len l, "
1598                  "of the layout tests")),
1599         optparse.make_option("--run-part", help=("Run a specified part (n:m), "
1600                   "the nth of m parts, of the layout tests")),
1601         # old-run-webkit-tests calls --batch-size: --nthly n
1602         #   Restart DumpRenderTree every n tests (default: 1000)
1603         optparse.make_option("--batch-size",
1604             help=("Run a the tests in batches (n), after every n tests, "
1605                   "DumpRenderTree is relaunched."), type="int", default=0),
1606         # old-run-webkit-tests calls --run-singly: -1|--singly
1607         # Isolate each test case run (implies --nthly 1 --verbose)
1608         optparse.make_option("--run-singly", action="store_true",
1609             default=False, help="run a separate DumpRenderTree for each test"),
1610         optparse.make_option("--child-processes",
1611             help="Number of DumpRenderTrees to run in parallel."),
1612         # FIXME: Display default number of child processes that will run.
1613         optparse.make_option("--worker-model", action="store",
1614             default="old-threads", help=("controls worker model. Valid values "
1615             "are 'old-inline', 'old-threads'.")),
1616         optparse.make_option("--experimental-fully-parallel",
1617             action="store_true", default=False,
1618             help="run all tests in parallel"),
1619         optparse.make_option("--exit-after-n-failures", type="int", nargs=1,
1620             help="Exit after the first N failures instead of running all "
1621             "tests"),
1622         optparse.make_option("--exit-after-n-crashes-or-timeouts", type="int",
1623             nargs=1, help="Exit after the first N crashes instead of running "
1624             "all tests"),
1625         # FIXME: consider: --iterations n
1626         #      Number of times to run the set of tests (e.g. ABCABCABC)
1627         optparse.make_option("--print-last-failures", action="store_true",
1628             default=False, help="Print the tests in the last run that "
1629             "had unexpected failures (or passes) and then exit."),
1630         optparse.make_option("--retest-last-failures", action="store_true",
1631             default=False, help="re-test the tests in the last run that "
1632             "had unexpected failures (or passes)."),
1633         optparse.make_option("--retry-failures", action="store_true",
1634             default=True,
1635             help="Re-try any tests that produce unexpected results (default)"),
1636         optparse.make_option("--no-retry-failures", action="store_false",
1637             dest="retry_failures",
1638             help="Don't re-try any tests that produce unexpected results."),
1639     ]
1640
1641     misc_options = [
1642         optparse.make_option("--lint-test-files", action="store_true",
1643         default=False, help=("Makes sure the test files parse for all "
1644                             "configurations. Does not run any tests.")),
1645     ]
1646
1647     # FIXME: Move these into json_results_generator.py
1648     results_json_options = [
1649         optparse.make_option("--master-name", help="The name of the buildbot master."),
1650         optparse.make_option("--builder-name", default="DUMMY_BUILDER_NAME",
1651             help=("The name of the builder shown on the waterfall running "
1652                   "this script e.g. WebKit.")),
1653         optparse.make_option("--build-name", default="DUMMY_BUILD_NAME",
1654             help=("The name of the builder used in its path, e.g. "
1655                   "webkit-rel.")),
1656         optparse.make_option("--build-number", default="DUMMY_BUILD_NUMBER",
1657             help=("The build number of the builder running this script.")),
1658         optparse.make_option("--test-results-server", default="",
1659             help=("If specified, upload results json files to this appengine "
1660                   "server.")),
1661         optparse.make_option("--upload-full-results",
1662             action="store_true",
1663             default=False,
1664             help="If true, upload full json results to server."),
1665     ]
1666
1667     option_list = (configuration_options + print_options +
1668                    chromium_options + results_options + test_options +
1669                    misc_options + results_json_options +
1670                    old_run_webkit_tests_compat)
1671     option_parser = optparse.OptionParser(option_list=option_list)
1672
1673     return option_parser.parse_args(args)
1674
1675
1676 def main():
1677     options, args = parse_args()
1678     port_obj = port.get(options.platform, options)
1679     return run(port_obj, options, args)
1680
1681
1682 if '__main__' == __name__:
1683     try:
1684         sys.exit(main())
1685     except KeyboardInterrupt:
1686         # this mirrors what the shell normally does
1687         sys.exit(signal.SIGINT + 128)