2010-11-24 Dirk Pranke <dpranke@chromium.org>
[WebKit.git] / WebKitTools / Scripts / webkitpy / layout_tests / run_webkit_tests.py
1 #!/usr/bin/env python
2 # Copyright (C) 2010 Google Inc. All rights reserved.
3 # Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Szeged
4 #
5 # Redistribution and use in source and binary forms, with or without
6 # modification, are permitted provided that the following conditions are
7 # met:
8 #
9 #     * Redistributions of source code must retain the above copyright
10 # notice, this list of conditions and the following disclaimer.
11 #     * Redistributions in binary form must reproduce the above
12 # copyright notice, this list of conditions and the following disclaimer
13 # in the documentation and/or other materials provided with the
14 # distribution.
15 #     * Neither the name of Google Inc. nor the names of its
16 # contributors may be used to endorse or promote products derived from
17 # this software without specific prior written permission.
18 #
19 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30
31 """Run layout tests.
32
33 This is a port of the existing webkit test script run-webkit-tests.
34
35 The TestRunner class runs a series of tests (TestType interface) against a set
36 of test files.  If a test file fails a TestType, it returns a list TestFailure
37 objects to the TestRunner.  The TestRunner then aggregates the TestFailures to
38 create a final report.
39
40 This script reads several files, if they exist in the test_lists subdirectory
41 next to this script itself.  Each should contain a list of paths to individual
42 tests or entire subdirectories of tests, relative to the outermost test
43 directory.  Entire lines starting with '//' (comments) will be ignored.
44
45 For details of the files' contents and purposes, see test_lists/README.
46 """
47
48 from __future__ import with_statement
49
50 import codecs
51 import errno
52 import glob
53 import logging
54 import math
55 import optparse
56 import os
57 import platform
58 import Queue
59 import random
60 import re
61 import shutil
62 import signal
63 import sys
64 import time
65 import traceback
66
67 from layout_package import dump_render_tree_thread
68 from layout_package import json_layout_results_generator
69 from layout_package import message_broker
70 from layout_package import printing
71 from layout_package import test_expectations
72 from layout_package import test_failures
73 from layout_package import test_results
74 from layout_package import test_results_uploader
75
76 from webkitpy.common.system import user
77 from webkitpy.thirdparty import simplejson
78 from webkitpy.tool import grammar
79
80 import port
81
82 _log = logging.getLogger("webkitpy.layout_tests.run_webkit_tests")
83
84 # Builder base URL where we have the archived test results.
85 BUILDER_BASE_URL = "http://build.chromium.org/buildbot/layout_test_results/"
86
87 LAYOUT_TESTS_DIRECTORY = "LayoutTests" + os.sep
88
89 TestExpectationsFile = test_expectations.TestExpectationsFile
90
91
92 class TestInput:
93     """Groups information about a test for easy passing of data."""
94
95     def __init__(self, filename, timeout):
96         """Holds the input parameters for a test.
97         Args:
98           filename: Full path to the test.
99           timeout: Timeout in msecs the driver should use while running the test
100           """
101         # FIXME: filename should really be test_name as a relative path.
102         self.filename = filename
103         self.timeout = timeout
104         # The image_hash is used to avoid doing an image dump if the
105         # checksums match. The image_hash is set later, and only if it is needed
106         # for the test.
107         self.image_hash = None
108
109
110 class ResultSummary(object):
111     """A class for partitioning the test results we get into buckets.
112
113     This class is basically a glorified struct and it's private to this file
114     so we don't bother with any information hiding."""
115
116     def __init__(self, expectations, test_files):
117         self.total = len(test_files)
118         self.remaining = self.total
119         self.expectations = expectations
120         self.expected = 0
121         self.unexpected = 0
122         self.tests_by_expectation = {}
123         self.tests_by_timeline = {}
124         self.results = {}
125         self.unexpected_results = {}
126         self.failures = {}
127         self.tests_by_expectation[test_expectations.SKIP] = set()
128         for expectation in TestExpectationsFile.EXPECTATIONS.values():
129             self.tests_by_expectation[expectation] = set()
130         for timeline in TestExpectationsFile.TIMELINES.values():
131             self.tests_by_timeline[timeline] = (
132                 expectations.get_tests_with_timeline(timeline))
133
134     def add(self, result, expected):
135         """Add a TestResult into the appropriate bin.
136
137         Args:
138           result: TestResult from dump_render_tree_thread.
139           expected: whether the result was what we expected it to be.
140         """
141
142         self.tests_by_expectation[result.type].add(result.filename)
143         self.results[result.filename] = result
144         self.remaining -= 1
145         if len(result.failures):
146             self.failures[result.filename] = result.failures
147         if expected:
148             self.expected += 1
149         else:
150             self.unexpected_results[result.filename] = result.type
151             self.unexpected += 1
152
153
154 def summarize_unexpected_results(port_obj, expectations, result_summary,
155                                  retry_summary):
156     """Summarize any unexpected results as a dict.
157
158     FIXME: split this data structure into a separate class?
159
160     Args:
161         port_obj: interface to port-specific hooks
162         expectations: test_expectations.TestExpectations object
163         result_summary: summary object from initial test runs
164         retry_summary: summary object from final test run of retried tests
165     Returns:
166         A dictionary containing a summary of the unexpected results from the
167         run, with the following fields:
168         'version': a version indicator (1 in this version)
169         'fixable': # of fixable tests (NOW - PASS)
170         'skipped': # of skipped tests (NOW & SKIPPED)
171         'num_regressions': # of non-flaky failures
172         'num_flaky': # of flaky failures
173         'num_passes': # of unexpected passes
174         'tests': a dict of tests -> {'expected': '...', 'actual': '...'}
175     """
176     results = {}
177     results['version'] = 1
178
179     tbe = result_summary.tests_by_expectation
180     tbt = result_summary.tests_by_timeline
181     results['fixable'] = len(tbt[test_expectations.NOW] -
182                                 tbe[test_expectations.PASS])
183     results['skipped'] = len(tbt[test_expectations.NOW] &
184                                 tbe[test_expectations.SKIP])
185
186     num_passes = 0
187     num_flaky = 0
188     num_regressions = 0
189     keywords = {}
190     for k, v in TestExpectationsFile.EXPECTATIONS.iteritems():
191         keywords[v] = k.upper()
192
193     tests = {}
194     for filename, result in result_summary.unexpected_results.iteritems():
195         # Note that if a test crashed in the original run, we ignore
196         # whether or not it crashed when we retried it (if we retried it),
197         # and always consider the result not flaky.
198         test = port_obj.relative_test_filename(filename)
199         expected = expectations.get_expectations_string(filename)
200         actual = [keywords[result]]
201
202         if result == test_expectations.PASS:
203             num_passes += 1
204         elif result == test_expectations.CRASH:
205             num_regressions += 1
206         else:
207             if filename not in retry_summary.unexpected_results:
208                 actual.extend(expectations.get_expectations_string(
209                     filename).split(" "))
210                 num_flaky += 1
211             else:
212                 retry_result = retry_summary.unexpected_results[filename]
213                 if result != retry_result:
214                     actual.append(keywords[retry_result])
215                     num_flaky += 1
216                 else:
217                     num_regressions += 1
218
219         tests[test] = {}
220         tests[test]['expected'] = expected
221         tests[test]['actual'] = " ".join(actual)
222
223     results['tests'] = tests
224     results['num_passes'] = num_passes
225     results['num_flaky'] = num_flaky
226     results['num_regressions'] = num_regressions
227
228     return results
229
230
231 class TestRunner:
232     """A class for managing running a series of tests on a series of layout
233     test files."""
234
235     HTTP_SUBDIR = os.sep.join(['', 'http', ''])
236     WEBSOCKET_SUBDIR = os.sep.join(['', 'websocket', ''])
237
238     # The per-test timeout in milliseconds, if no --time-out-ms option was
239     # given to run_webkit_tests. This should correspond to the default timeout
240     # in DumpRenderTree.
241     DEFAULT_TEST_TIMEOUT_MS = 6 * 1000
242
243     def __init__(self, port, options, printer, message_broker):
244         """Initialize test runner data structures.
245
246         Args:
247           port: an object implementing port-specific
248           options: a dictionary of command line options
249           printer: a Printer object to record updates to.
250           message_broker: object used to communicate with workers.
251         """
252         self._port = port
253         self._options = options
254         self._printer = printer
255         self._message_broker = message_broker
256
257         # disable wss server. need to install pyOpenSSL on buildbots.
258         # self._websocket_secure_server = websocket_server.PyWebSocket(
259         #        options.results_directory, use_tls=True, port=9323)
260
261         # a set of test files, and the same tests as a list
262         self._test_files = set()
263         self._test_files_list = None
264         self._result_queue = Queue.Queue()
265         self._retrying = False
266
267     def collect_tests(self, args, last_unexpected_results):
268         """Find all the files to test.
269
270         Args:
271           args: list of test arguments from the command line
272           last_unexpected_results: list of unexpected results to retest, if any
273
274         """
275         paths = [self._strip_test_dir_prefix(arg) for arg in args if arg and arg != '']
276         paths += last_unexpected_results
277         if self._options.test_list:
278             paths += read_test_files(self._options.test_list)
279         self._test_files = self._port.tests(paths)
280
281     def _strip_test_dir_prefix(self, path):
282         if path.startswith(LAYOUT_TESTS_DIRECTORY):
283             return path[len(LAYOUT_TESTS_DIRECTORY):]
284         return path
285
286     def lint(self):
287         # Creating the expecations for each platform/configuration pair does
288         # all the test list parsing and ensures it's correct syntax (e.g. no
289         # dupes).
290         for platform_name in self._port.test_platform_names():
291             self.parse_expectations(platform_name, is_debug_mode=True)
292             self.parse_expectations(platform_name, is_debug_mode=False)
293         self._printer.write("")
294         _log.info("If there are no fail messages, errors or exceptions, "
295                   "then the lint succeeded.")
296         return 0
297
298     def parse_expectations(self, test_platform_name, is_debug_mode):
299         """Parse the expectations from the test_list files and return a data
300         structure holding them. Throws an error if the test_list files have
301         invalid syntax."""
302         if self._options.lint_test_files:
303             test_files = None
304         else:
305             test_files = self._test_files
306
307         try:
308             expectations_str = self._port.test_expectations()
309             overrides_str = self._port.test_expectations_overrides()
310             self._expectations = test_expectations.TestExpectations(
311                 self._port, test_files, expectations_str, test_platform_name,
312                 is_debug_mode, self._options.lint_test_files,
313                 overrides=overrides_str)
314             return self._expectations
315         except SyntaxError, err:
316             if self._options.lint_test_files:
317                 print str(err)
318             else:
319                 raise err
320
321     def prepare_lists_and_print_output(self):
322         """Create appropriate subsets of test lists and returns a
323         ResultSummary object. Also prints expected test counts.
324         """
325
326         # Remove skipped - both fixable and ignored - files from the
327         # top-level list of files to test.
328         num_all_test_files = len(self._test_files)
329         self._printer.print_expected("Found:  %d tests" %
330                                      (len(self._test_files)))
331         if not num_all_test_files:
332             _log.critical('No tests to run.')
333             return None
334
335         skipped = set()
336         if num_all_test_files > 1 and not self._options.force:
337             skipped = self._expectations.get_tests_with_result_type(
338                            test_expectations.SKIP)
339             self._test_files -= skipped
340
341         # Create a sorted list of test files so the subset chunk,
342         # if used, contains alphabetically consecutive tests.
343         self._test_files_list = list(self._test_files)
344         if self._options.randomize_order:
345             random.shuffle(self._test_files_list)
346         else:
347             self._test_files_list.sort()
348
349         # If the user specifies they just want to run a subset of the tests,
350         # just grab a subset of the non-skipped tests.
351         if self._options.run_chunk or self._options.run_part:
352             chunk_value = self._options.run_chunk or self._options.run_part
353             test_files = self._test_files_list
354             try:
355                 (chunk_num, chunk_len) = chunk_value.split(":")
356                 chunk_num = int(chunk_num)
357                 assert(chunk_num >= 0)
358                 test_size = int(chunk_len)
359                 assert(test_size > 0)
360             except:
361                 _log.critical("invalid chunk '%s'" % chunk_value)
362                 return None
363
364             # Get the number of tests
365             num_tests = len(test_files)
366
367             # Get the start offset of the slice.
368             if self._options.run_chunk:
369                 chunk_len = test_size
370                 # In this case chunk_num can be really large. We need
371                 # to make the slave fit in the current number of tests.
372                 slice_start = (chunk_num * chunk_len) % num_tests
373             else:
374                 # Validate the data.
375                 assert(test_size <= num_tests)
376                 assert(chunk_num <= test_size)
377
378                 # To count the chunk_len, and make sure we don't skip
379                 # some tests, we round to the next value that fits exactly
380                 # all the parts.
381                 rounded_tests = num_tests
382                 if rounded_tests % test_size != 0:
383                     rounded_tests = (num_tests + test_size -
384                                      (num_tests % test_size))
385
386                 chunk_len = rounded_tests / test_size
387                 slice_start = chunk_len * (chunk_num - 1)
388                 # It does not mind if we go over test_size.
389
390             # Get the end offset of the slice.
391             slice_end = min(num_tests, slice_start + chunk_len)
392
393             files = test_files[slice_start:slice_end]
394
395             tests_run_msg = 'Running: %d tests (chunk slice [%d:%d] of %d)' % (
396                 (slice_end - slice_start), slice_start, slice_end, num_tests)
397             self._printer.print_expected(tests_run_msg)
398
399             # If we reached the end and we don't have enough tests, we run some
400             # from the beginning.
401             if slice_end - slice_start < chunk_len:
402                 extra = chunk_len - (slice_end - slice_start)
403                 extra_msg = ('   last chunk is partial, appending [0:%d]' %
404                             extra)
405                 self._printer.print_expected(extra_msg)
406                 tests_run_msg += "\n" + extra_msg
407                 files.extend(test_files[0:extra])
408             tests_run_filename = os.path.join(self._options.results_directory,
409                                               "tests_run.txt")
410             with codecs.open(tests_run_filename, "w", "utf-8") as file:
411                 file.write(tests_run_msg + "\n")
412
413             len_skip_chunk = int(len(files) * len(skipped) /
414                                  float(len(self._test_files)))
415             skip_chunk_list = list(skipped)[0:len_skip_chunk]
416             skip_chunk = set(skip_chunk_list)
417
418             # Update expectations so that the stats are calculated correctly.
419             # We need to pass a list that includes the right # of skipped files
420             # to ParseExpectations so that ResultSummary() will get the correct
421             # stats. So, we add in the subset of skipped files, and then
422             # subtract them back out.
423             self._test_files_list = files + skip_chunk_list
424             self._test_files = set(self._test_files_list)
425
426             self._expectations = self.parse_expectations(
427                 self._port.test_platform_name(),
428                 self._options.configuration == 'Debug')
429
430             self._test_files = set(files)
431             self._test_files_list = files
432         else:
433             skip_chunk = skipped
434
435         result_summary = ResultSummary(self._expectations,
436             self._test_files | skip_chunk)
437         self._print_expected_results_of_type(result_summary,
438             test_expectations.PASS, "passes")
439         self._print_expected_results_of_type(result_summary,
440             test_expectations.FAIL, "failures")
441         self._print_expected_results_of_type(result_summary,
442             test_expectations.FLAKY, "flaky")
443         self._print_expected_results_of_type(result_summary,
444             test_expectations.SKIP, "skipped")
445
446         if self._options.force:
447             self._printer.print_expected('Running all tests, including '
448                                          'skips (--force)')
449         else:
450             # Note that we don't actually run the skipped tests (they were
451             # subtracted out of self._test_files, above), but we stub out the
452             # results here so the statistics can remain accurate.
453             for test in skip_chunk:
454                 result = test_results.TestResult(test,
455                     failures=[], test_run_time=0, total_time_for_all_diffs=0,
456                     time_for_diffs=0)
457                 result.type = test_expectations.SKIP
458                 result_summary.add(result, expected=True)
459         self._printer.print_expected('')
460
461         return result_summary
462
463     def _get_dir_for_test_file(self, test_file):
464         """Returns the highest-level directory by which to shard the given
465         test file."""
466         index = test_file.rfind(os.sep + LAYOUT_TESTS_DIRECTORY)
467
468         test_file = test_file[index + len(LAYOUT_TESTS_DIRECTORY):]
469         test_file_parts = test_file.split(os.sep, 1)
470         directory = test_file_parts[0]
471         test_file = test_file_parts[1]
472
473         # The http tests are very stable on mac/linux.
474         # TODO(ojan): Make the http server on Windows be apache so we can
475         # turn shard the http tests there as well. Switching to apache is
476         # what made them stable on linux/mac.
477         return_value = directory
478         while ((directory != 'http' or sys.platform in ('darwin', 'linux2'))
479                 and test_file.find(os.sep) >= 0):
480             test_file_parts = test_file.split(os.sep, 1)
481             directory = test_file_parts[0]
482             return_value = os.path.join(return_value, directory)
483             test_file = test_file_parts[1]
484
485         return return_value
486
487     def _get_test_input_for_file(self, test_file):
488         """Returns the appropriate TestInput object for the file. Mostly this
489         is used for looking up the timeout value (in ms) to use for the given
490         test."""
491         if self._test_is_slow(test_file):
492             return TestInput(test_file, self._options.slow_time_out_ms)
493         return TestInput(test_file, self._options.time_out_ms)
494
495     def _test_requires_lock(self, test_file):
496         """Return True if the test needs to be locked when
497         running multiple copies of NRWTs."""
498         split_path = test_file.split(os.sep)
499         return 'http' in split_path or 'websocket' in split_path
500
501     def _test_is_slow(self, test_file):
502         return self._expectations.has_modifier(test_file,
503                                                test_expectations.SLOW)
504
505     def _shard_tests(self, test_files, use_real_shards):
506         """Groups tests into batches.
507         This helps ensure that tests that depend on each other (aka bad tests!)
508         continue to run together as most cross-tests dependencies tend to
509         occur within the same directory. If use_real_shards is false, we
510         put each (non-HTTP/websocket) test into its own shard for maximum
511         concurrency instead of trying to do any sort of real sharding.
512
513         Return:
514             A list of lists of TestInput objects.
515         """
516         # FIXME: when we added http locking, we changed how this works such
517         # that we always lump all of the HTTP threads into a single shard.
518         # That will slow down experimental-fully-parallel, but it's unclear
519         # what the best alternative is completely revamping how we track
520         # when to grab the lock.
521
522         test_lists = []
523         tests_to_http_lock = []
524         if not use_real_shards:
525             for test_file in test_files:
526                 test_input = self._get_test_input_for_file(test_file)
527                 if self._test_requires_lock(test_file):
528                     tests_to_http_lock.append(test_input)
529                 else:
530                     test_lists.append((".", [test_input]))
531         else:
532             tests_by_dir = {}
533             for test_file in test_files:
534                 directory = self._get_dir_for_test_file(test_file)
535                 test_input = self._get_test_input_for_file(test_file)
536                 if self._test_requires_lock(test_file):
537                     tests_to_http_lock.append(test_input)
538                 else:
539                     tests_by_dir.setdefault(directory, [])
540                     tests_by_dir[directory].append(test_input)
541             # Sort by the number of tests in the dir so that the ones with the
542             # most tests get run first in order to maximize parallelization.
543             # Number of tests is a good enough, but not perfect, approximation
544             # of how long that set of tests will take to run. We can't just use
545             # a PriorityQueue until we move to Python 2.6.
546             for directory in tests_by_dir:
547                 test_list = tests_by_dir[directory]
548                 # Keep the tests in alphabetical order.
549                 # FIXME: Remove once tests are fixed so they can be run in any
550                 # order.
551                 test_list.reverse()
552                 test_list_tuple = (directory, test_list)
553                 test_lists.append(test_list_tuple)
554             test_lists.sort(lambda a, b: cmp(len(b[1]), len(a[1])))
555
556         # Put the http tests first. There are only a couple hundred of them,
557         # but each http test takes a very long time to run, so sorting by the
558         # number of tests doesn't accurately capture how long they take to run.
559         if tests_to_http_lock:
560             tests_to_http_lock.reverse()
561             test_lists.insert(0, ("tests_to_http_lock", tests_to_http_lock))
562
563         return test_lists
564
565     def _contains_tests(self, subdir):
566         for test_file in self._test_files:
567             if test_file.find(subdir) >= 0:
568                 return True
569         return False
570
571     def _num_workers(self):
572         return int(self._options.child_processes)
573
574     def _run_tests(self, file_list, result_summary):
575         """Runs the tests in the file_list.
576
577         Return: A tuple (keyboard_interrupted, thread_timings, test_timings,
578             individual_test_timings)
579             keyboard_interrupted is whether someone typed Ctrl^C
580             thread_timings is a list of dicts with the total runtime
581               of each thread with 'name', 'num_tests', 'total_time' properties
582             test_timings is a list of timings for each sharded subdirectory
583               of the form [time, directory_name, num_tests]
584             individual_test_timings is a list of run times for each test
585               in the form {filename:filename, test_run_time:test_run_time}
586             result_summary: summary object to populate with the results
587         """
588
589         self._printer.print_update('Sharding tests ...')
590         num_workers = self._num_workers()
591         test_lists = self._shard_tests(file_list,
592             num_workers > 1 and not self._options.experimental_fully_parallel)
593         filename_queue = Queue.Queue()
594         for item in test_lists:
595             filename_queue.put(item)
596
597         self._printer.print_update('Starting %s ...' %
598                                    grammar.pluralize('worker', num_workers))
599         message_broker = self._message_broker
600         self._current_filename_queue = filename_queue
601         self._current_result_summary = result_summary
602         threads = message_broker.start_workers(self)
603
604         self._printer.print_update("Starting testing ...")
605         keyboard_interrupted = False
606         try:
607             message_broker.run_message_loop()
608         except KeyboardInterrupt:
609             _log.info("Interrupted, exiting")
610             message_broker.cancel_workers()
611             keyboard_interrupted = True
612         except:
613             # Unexpected exception; don't try to clean up workers.
614             _log.info("Exception raised, exiting")
615             raise
616
617         thread_timings, test_timings, individual_test_timings = \
618             self._collect_timing_info(threads)
619
620         return (keyboard_interrupted, thread_timings, test_timings,
621                 individual_test_timings)
622
623     def update(self):
624         self.update_summary(self._current_result_summary)
625
626     def _collect_timing_info(self, threads):
627         test_timings = {}
628         individual_test_timings = []
629         thread_timings = []
630
631         for thread in threads:
632             thread_timings.append({'name': thread.getName(),
633                                    'num_tests': thread.get_num_tests(),
634                                    'total_time': thread.get_total_time()})
635             test_timings.update(thread.get_test_group_timing_stats())
636             individual_test_timings.extend(thread.get_test_results())
637
638         return (thread_timings, test_timings, individual_test_timings)
639
640     def needs_http(self):
641         """Returns whether the test runner needs an HTTP server."""
642         return self._contains_tests(self.HTTP_SUBDIR)
643
644     def needs_websocket(self):
645         """Returns whether the test runner needs a WEBSOCKET server."""
646         return self._contains_tests(self.WEBSOCKET_SUBDIR)
647
648     def set_up_run(self):
649         """Configures the system to be ready to run tests.
650
651         Returns a ResultSummary object if we should continue to run tests,
652         or None if we should abort.
653
654         """
655         # This must be started before we check the system dependencies,
656         # since the helper may do things to make the setup correct.
657         self._printer.print_update("Starting helper ...")
658         self._port.start_helper()
659
660         # Check that the system dependencies (themes, fonts, ...) are correct.
661         if not self._options.nocheck_sys_deps:
662             self._printer.print_update("Checking system dependencies ...")
663             if not self._port.check_sys_deps(self.needs_http()):
664                 self._port.stop_helper()
665                 return None
666
667         if self._options.clobber_old_results:
668             self._clobber_old_results()
669
670         # Create the output directory if it doesn't already exist.
671         self._port.maybe_make_directory(self._options.results_directory)
672
673         self._port.setup_test_run()
674
675         self._printer.print_update("Preparing tests ...")
676         result_summary = self.prepare_lists_and_print_output()
677         if not result_summary:
678             return None
679
680         return result_summary
681
682     def run(self, result_summary):
683         """Run all our tests on all our test files.
684
685         For each test file, we run each test type. If there are any failures,
686         we collect them for reporting.
687
688         Args:
689           result_summary: a summary object tracking the test results.
690
691         Return:
692           The number of unexpected results (0 == success)
693         """
694         # gather_test_files() must have been called first to initialize us.
695         # If we didn't find any files to test, we've errored out already in
696         # prepare_lists_and_print_output().
697         assert(len(self._test_files))
698
699         start_time = time.time()
700
701         keyboard_interrupted, thread_timings, test_timings, \
702             individual_test_timings = (
703             self._run_tests(self._test_files_list, result_summary))
704
705         # We exclude the crashes from the list of results to retry, because
706         # we want to treat even a potentially flaky crash as an error.
707         failures = self._get_failures(result_summary, include_crashes=False)
708         retry_summary = result_summary
709         while (len(failures) and self._options.retry_failures and
710             not self._retrying and not keyboard_interrupted):
711             _log.info('')
712             _log.info("Retrying %d unexpected failure(s) ..." % len(failures))
713             _log.info('')
714             self._retrying = True
715             retry_summary = ResultSummary(self._expectations, failures.keys())
716             # Note that we intentionally ignore the return value here.
717             self._run_tests(failures.keys(), retry_summary)
718             failures = self._get_failures(retry_summary, include_crashes=True)
719
720         end_time = time.time()
721
722         self._print_timing_statistics(end_time - start_time,
723                                       thread_timings, test_timings,
724                                       individual_test_timings,
725                                       result_summary)
726
727         self._print_result_summary(result_summary)
728
729         sys.stdout.flush()
730         sys.stderr.flush()
731
732         self._printer.print_one_line_summary(result_summary.total,
733                                              result_summary.expected,
734                                              result_summary.unexpected)
735
736         unexpected_results = summarize_unexpected_results(self._port,
737             self._expectations, result_summary, retry_summary)
738         self._printer.print_unexpected_results(unexpected_results)
739
740         if self._options.record_results:
741             # Write the same data to log files and upload generated JSON files
742             # to appengine server.
743             self._upload_json_files(unexpected_results, result_summary,
744                                     individual_test_timings)
745
746         # Write the summary to disk (results.html) and display it if requested.
747         wrote_results = self._write_results_html_file(result_summary)
748         if self._options.show_results and wrote_results:
749             self._show_results_html_file()
750
751         # Now that we've completed all the processing we can, we re-raise
752         # a KeyboardInterrupt if necessary so the caller can handle it.
753         if keyboard_interrupted:
754             raise KeyboardInterrupt
755
756         # Ignore flaky failures and unexpected passes so we don't turn the
757         # bot red for those.
758         return unexpected_results['num_regressions']
759
760     def clean_up_run(self):
761         """Restores the system after we're done running tests."""
762
763         _log.debug("flushing stdout")
764         sys.stdout.flush()
765         _log.debug("flushing stderr")
766         sys.stderr.flush()
767         _log.debug("stopping helper")
768         self._port.stop_helper()
769
770     def update_summary(self, result_summary):
771         """Update the summary and print results with any completed tests."""
772         while True:
773             try:
774                 result = test_results.TestResult.loads(self._result_queue.get_nowait())
775             except Queue.Empty:
776                 return
777
778             expected = self._expectations.matches_an_expected_result(
779                 result.filename, result.type, self._options.pixel_tests)
780             result_summary.add(result, expected)
781             exp_str = self._expectations.get_expectations_string(
782                 result.filename)
783             got_str = self._expectations.expectation_to_string(result.type)
784             self._printer.print_test_result(result, expected, exp_str, got_str)
785             self._printer.print_progress(result_summary, self._retrying,
786                                          self._test_files_list)
787
788     def _clobber_old_results(self):
789         # Just clobber the actual test results directories since the other
790         # files in the results directory are explicitly used for cross-run
791         # tracking.
792         self._printer.print_update("Clobbering old results in %s" %
793                                    self._options.results_directory)
794         layout_tests_dir = self._port.layout_tests_dir()
795         possible_dirs = self._port.test_dirs()
796         for dirname in possible_dirs:
797             if os.path.isdir(os.path.join(layout_tests_dir, dirname)):
798                 shutil.rmtree(os.path.join(self._options.results_directory,
799                                            dirname),
800                               ignore_errors=True)
801
802     def _get_failures(self, result_summary, include_crashes):
803         """Filters a dict of results and returns only the failures.
804
805         Args:
806           result_summary: the results of the test run
807           include_crashes: whether crashes are included in the output.
808             We use False when finding the list of failures to retry
809             to see if the results were flaky. Although the crashes may also be
810             flaky, we treat them as if they aren't so that they're not ignored.
811         Returns:
812           a dict of files -> results
813         """
814         failed_results = {}
815         for test, result in result_summary.unexpected_results.iteritems():
816             if (result == test_expectations.PASS or
817                 result == test_expectations.CRASH and not include_crashes):
818                 continue
819             failed_results[test] = result
820
821         return failed_results
822
823     def _upload_json_files(self, unexpected_results, result_summary,
824                         individual_test_timings):
825         """Writes the results of the test run as JSON files into the results
826         dir and upload the files to the appengine server.
827
828         There are three different files written into the results dir:
829           unexpected_results.json: A short list of any unexpected results.
830             This is used by the buildbots to display results.
831           expectations.json: This is used by the flakiness dashboard.
832           results.json: A full list of the results - used by the flakiness
833             dashboard and the aggregate results dashboard.
834
835         Args:
836           unexpected_results: dict of unexpected results
837           result_summary: full summary object
838           individual_test_timings: list of test times (used by the flakiness
839             dashboard).
840         """
841         results_directory = self._options.results_directory
842         _log.debug("Writing JSON files in %s." % results_directory)
843         unexpected_json_path = os.path.join(results_directory, "unexpected_results.json")
844         with codecs.open(unexpected_json_path, "w", "utf-8") as file:
845             simplejson.dump(unexpected_results, file, sort_keys=True, indent=2)
846
847         # Write a json file of the test_expectations.txt file for the layout
848         # tests dashboard.
849         expectations_path = os.path.join(results_directory, "expectations.json")
850         expectations_json = \
851             self._expectations.get_expectations_json_for_all_platforms()
852         with codecs.open(expectations_path, "w", "utf-8") as file:
853             file.write(u"ADD_EXPECTATIONS(%s);" % expectations_json)
854
855         generator = json_layout_results_generator.JSONLayoutResultsGenerator(
856             self._port, self._options.builder_name, self._options.build_name,
857             self._options.build_number, self._options.results_directory,
858             BUILDER_BASE_URL, individual_test_timings,
859             self._expectations, result_summary, self._test_files_list,
860             not self._options.upload_full_results,
861             self._options.test_results_server,
862             "layout-tests",
863             self._options.master_name)
864
865         _log.debug("Finished writing JSON files.")
866
867         json_files = ["expectations.json"]
868         if self._options.upload_full_results:
869             json_files.append("results.json")
870         else:
871             json_files.append("incremental_results.json")
872
873         generator.upload_json_files(json_files)
874
875     def _print_config(self):
876         """Prints the configuration for the test run."""
877         p = self._printer
878         p.print_config("Using port '%s'" % self._port.name())
879         p.print_config("Placing test results in %s" %
880                        self._options.results_directory)
881         if self._options.new_baseline:
882             p.print_config("Placing new baselines in %s" %
883                            self._port.baseline_path())
884         p.print_config("Using %s build" % self._options.configuration)
885         if self._options.pixel_tests:
886             p.print_config("Pixel tests enabled")
887         else:
888             p.print_config("Pixel tests disabled")
889
890         p.print_config("Regular timeout: %s, slow test timeout: %s" %
891                        (self._options.time_out_ms,
892                         self._options.slow_time_out_ms))
893
894         if self._num_workers() == 1:
895             p.print_config("Running one %s" % self._port.driver_name())
896         else:
897             p.print_config("Running %s %ss in parallel" %
898                            (self._options.child_processes,
899                             self._port.driver_name()))
900         p.print_config("Worker model: %s" % self._options.worker_model)
901         p.print_config("")
902
903     def _print_expected_results_of_type(self, result_summary,
904                                         result_type, result_type_str):
905         """Print the number of the tests in a given result class.
906
907         Args:
908           result_summary - the object containing all the results to report on
909           result_type - the particular result type to report in the summary.
910           result_type_str - a string description of the result_type.
911         """
912         tests = self._expectations.get_tests_with_result_type(result_type)
913         now = result_summary.tests_by_timeline[test_expectations.NOW]
914         wontfix = result_summary.tests_by_timeline[test_expectations.WONTFIX]
915
916         # We use a fancy format string in order to print the data out in a
917         # nicely-aligned table.
918         fmtstr = ("Expect: %%5d %%-8s (%%%dd now, %%%dd wontfix)"
919                   % (self._num_digits(now), self._num_digits(wontfix)))
920         self._printer.print_expected(fmtstr %
921             (len(tests), result_type_str, len(tests & now), len(tests & wontfix)))
922
923     def _num_digits(self, num):
924         """Returns the number of digits needed to represent the length of a
925         sequence."""
926         ndigits = 1
927         if len(num):
928             ndigits = int(math.log10(len(num))) + 1
929         return ndigits
930
931     def _print_timing_statistics(self, total_time, thread_timings,
932                                directory_test_timings, individual_test_timings,
933                                result_summary):
934         """Record timing-specific information for the test run.
935
936         Args:
937           total_time: total elapsed time (in seconds) for the test run
938           thread_timings: wall clock time each thread ran for
939           directory_test_timings: timing by directory
940           individual_test_timings: timing by file
941           result_summary: summary object for the test run
942         """
943         self._printer.print_timing("Test timing:")
944         self._printer.print_timing("  %6.2f total testing time" % total_time)
945         self._printer.print_timing("")
946         self._printer.print_timing("Thread timing:")
947         cuml_time = 0
948         for t in thread_timings:
949             self._printer.print_timing("    %10s: %5d tests, %6.2f secs" %
950                   (t['name'], t['num_tests'], t['total_time']))
951             cuml_time += t['total_time']
952         self._printer.print_timing("   %6.2f cumulative, %6.2f optimal" %
953               (cuml_time, cuml_time / int(self._options.child_processes)))
954         self._printer.print_timing("")
955
956         self._print_aggregate_test_statistics(individual_test_timings)
957         self._print_individual_test_times(individual_test_timings,
958                                           result_summary)
959         self._print_directory_timings(directory_test_timings)
960
961     def _print_aggregate_test_statistics(self, individual_test_timings):
962         """Prints aggregate statistics (e.g. median, mean, etc.) for all tests.
963         Args:
964           individual_test_timings: List of dump_render_tree_thread.TestStats
965               for all tests.
966         """
967         test_types = []  # Unit tests don't actually produce any timings.
968         if individual_test_timings:
969             test_types = individual_test_timings[0].time_for_diffs.keys()
970         times_for_dump_render_tree = []
971         times_for_diff_processing = []
972         times_per_test_type = {}
973         for test_type in test_types:
974             times_per_test_type[test_type] = []
975
976         for test_stats in individual_test_timings:
977             times_for_dump_render_tree.append(test_stats.test_run_time)
978             times_for_diff_processing.append(
979                 test_stats.total_time_for_all_diffs)
980             time_for_diffs = test_stats.time_for_diffs
981             for test_type in test_types:
982                 times_per_test_type[test_type].append(
983                     time_for_diffs[test_type])
984
985         self._print_statistics_for_test_timings(
986             "PER TEST TIME IN TESTSHELL (seconds):",
987             times_for_dump_render_tree)
988         self._print_statistics_for_test_timings(
989             "PER TEST DIFF PROCESSING TIMES (seconds):",
990             times_for_diff_processing)
991         for test_type in test_types:
992             self._print_statistics_for_test_timings(
993                 "PER TEST TIMES BY TEST TYPE: %s" % test_type,
994                 times_per_test_type[test_type])
995
996     def _print_individual_test_times(self, individual_test_timings,
997                                   result_summary):
998         """Prints the run times for slow, timeout and crash tests.
999         Args:
1000           individual_test_timings: List of dump_render_tree_thread.TestStats
1001               for all tests.
1002           result_summary: summary object for test run
1003         """
1004         # Reverse-sort by the time spent in DumpRenderTree.
1005         individual_test_timings.sort(lambda a, b:
1006             cmp(b.test_run_time, a.test_run_time))
1007
1008         num_printed = 0
1009         slow_tests = []
1010         timeout_or_crash_tests = []
1011         unexpected_slow_tests = []
1012         for test_tuple in individual_test_timings:
1013             filename = test_tuple.filename
1014             is_timeout_crash_or_slow = False
1015             if self._test_is_slow(filename):
1016                 is_timeout_crash_or_slow = True
1017                 slow_tests.append(test_tuple)
1018
1019             if filename in result_summary.failures:
1020                 result = result_summary.results[filename].type
1021                 if (result == test_expectations.TIMEOUT or
1022                     result == test_expectations.CRASH):
1023                     is_timeout_crash_or_slow = True
1024                     timeout_or_crash_tests.append(test_tuple)
1025
1026             if (not is_timeout_crash_or_slow and
1027                 num_printed < printing.NUM_SLOW_TESTS_TO_LOG):
1028                 num_printed = num_printed + 1
1029                 unexpected_slow_tests.append(test_tuple)
1030
1031         self._printer.print_timing("")
1032         self._print_test_list_timing("%s slowest tests that are not "
1033             "marked as SLOW and did not timeout/crash:" %
1034             printing.NUM_SLOW_TESTS_TO_LOG, unexpected_slow_tests)
1035         self._printer.print_timing("")
1036         self._print_test_list_timing("Tests marked as SLOW:", slow_tests)
1037         self._printer.print_timing("")
1038         self._print_test_list_timing("Tests that timed out or crashed:",
1039                                      timeout_or_crash_tests)
1040         self._printer.print_timing("")
1041
1042     def _print_test_list_timing(self, title, test_list):
1043         """Print timing info for each test.
1044
1045         Args:
1046           title: section heading
1047           test_list: tests that fall in this section
1048         """
1049         if self._printer.disabled('slowest'):
1050             return
1051
1052         self._printer.print_timing(title)
1053         for test_tuple in test_list:
1054             filename = test_tuple.filename[len(
1055                 self._port.layout_tests_dir()) + 1:]
1056             filename = filename.replace('\\', '/')
1057             test_run_time = round(test_tuple.test_run_time, 1)
1058             self._printer.print_timing("  %s took %s seconds" %
1059                                        (filename, test_run_time))
1060
1061     def _print_directory_timings(self, directory_test_timings):
1062         """Print timing info by directory for any directories that
1063         take > 10 seconds to run.
1064
1065         Args:
1066           directory_test_timing: time info for each directory
1067         """
1068         timings = []
1069         for directory in directory_test_timings:
1070             num_tests, time_for_directory = directory_test_timings[directory]
1071             timings.append((round(time_for_directory, 1), directory,
1072                             num_tests))
1073         timings.sort()
1074
1075         self._printer.print_timing("Time to process slowest subdirectories:")
1076         min_seconds_to_print = 10
1077         for timing in timings:
1078             if timing[0] > min_seconds_to_print:
1079                 self._printer.print_timing(
1080                     "  %s took %s seconds to run %s tests." % (timing[1],
1081                     timing[0], timing[2]))
1082         self._printer.print_timing("")
1083
1084     def _print_statistics_for_test_timings(self, title, timings):
1085         """Prints the median, mean and standard deviation of the values in
1086         timings.
1087
1088         Args:
1089           title: Title for these timings.
1090           timings: A list of floats representing times.
1091         """
1092         self._printer.print_timing(title)
1093         timings.sort()
1094
1095         num_tests = len(timings)
1096         if not num_tests:
1097             return
1098         percentile90 = timings[int(.9 * num_tests)]
1099         percentile99 = timings[int(.99 * num_tests)]
1100
1101         if num_tests % 2 == 1:
1102             median = timings[((num_tests - 1) / 2) - 1]
1103         else:
1104             lower = timings[num_tests / 2 - 1]
1105             upper = timings[num_tests / 2]
1106             median = (float(lower + upper)) / 2
1107
1108         mean = sum(timings) / num_tests
1109
1110         for time in timings:
1111             sum_of_deviations = math.pow(time - mean, 2)
1112
1113         std_deviation = math.sqrt(sum_of_deviations / num_tests)
1114         self._printer.print_timing("  Median:          %6.3f" % median)
1115         self._printer.print_timing("  Mean:            %6.3f" % mean)
1116         self._printer.print_timing("  90th percentile: %6.3f" % percentile90)
1117         self._printer.print_timing("  99th percentile: %6.3f" % percentile99)
1118         self._printer.print_timing("  Standard dev:    %6.3f" % std_deviation)
1119         self._printer.print_timing("")
1120
1121     def _print_result_summary(self, result_summary):
1122         """Print a short summary about how many tests passed.
1123
1124         Args:
1125           result_summary: information to log
1126         """
1127         failed = len(result_summary.failures)
1128         skipped = len(
1129             result_summary.tests_by_expectation[test_expectations.SKIP])
1130         total = result_summary.total
1131         passed = total - failed - skipped
1132         pct_passed = 0.0
1133         if total > 0:
1134             pct_passed = float(passed) * 100 / total
1135
1136         self._printer.print_actual("")
1137         self._printer.print_actual("=> Results: %d/%d tests passed (%.1f%%)" %
1138                      (passed, total, pct_passed))
1139         self._printer.print_actual("")
1140         self._print_result_summary_entry(result_summary,
1141             test_expectations.NOW, "Tests to be fixed")
1142
1143         self._printer.print_actual("")
1144         self._print_result_summary_entry(result_summary,
1145             test_expectations.WONTFIX,
1146             "Tests that will only be fixed if they crash (WONTFIX)")
1147         self._printer.print_actual("")
1148
1149     def _print_result_summary_entry(self, result_summary, timeline,
1150                                     heading):
1151         """Print a summary block of results for a particular timeline of test.
1152
1153         Args:
1154           result_summary: summary to print results for
1155           timeline: the timeline to print results for (NOT, WONTFIX, etc.)
1156           heading: a textual description of the timeline
1157         """
1158         total = len(result_summary.tests_by_timeline[timeline])
1159         not_passing = (total -
1160            len(result_summary.tests_by_expectation[test_expectations.PASS] &
1161                result_summary.tests_by_timeline[timeline]))
1162         self._printer.print_actual("=> %s (%d):" % (heading, not_passing))
1163
1164         for result in TestExpectationsFile.EXPECTATION_ORDER:
1165             if result == test_expectations.PASS:
1166                 continue
1167             results = (result_summary.tests_by_expectation[result] &
1168                        result_summary.tests_by_timeline[timeline])
1169             desc = TestExpectationsFile.EXPECTATION_DESCRIPTIONS[result]
1170             if not_passing and len(results):
1171                 pct = len(results) * 100.0 / not_passing
1172                 self._printer.print_actual("  %5d %-24s (%4.1f%%)" %
1173                     (len(results), desc[len(results) != 1], pct))
1174
1175     def _results_html(self, test_files, failures, title="Test Failures", override_time=None):
1176         """
1177         test_files = a list of file paths
1178         failures = dictionary mapping test paths to failure objects
1179         title = title printed at top of test
1180         override_time = current time (used by unit tests)
1181         """
1182         page = """<html>
1183   <head>
1184     <title>Layout Test Results (%(time)s)</title>
1185   </head>
1186   <body>
1187     <h2>%(title)s (%(time)s)</h2>
1188         """ % {'title': title, 'time': override_time or time.asctime()}
1189
1190         for test_file in sorted(test_files):
1191             test_name = self._port.relative_test_filename(test_file)
1192             test_url = self._port.filename_to_uri(test_file)
1193             page += u"<p><a href='%s'>%s</a><br />\n" % (test_url, test_name)
1194             test_failures = failures.get(test_file, [])
1195             for failure in test_failures:
1196                 page += (u"&nbsp;&nbsp;%s<br/>" %
1197                          failure.result_html_output(test_name))
1198             page += "</p>\n"
1199         page += "</body></html>\n"
1200         return page
1201
1202     def _write_results_html_file(self, result_summary):
1203         """Write results.html which is a summary of tests that failed.
1204
1205         Args:
1206           result_summary: a summary of the results :)
1207
1208         Returns:
1209           True if any results were written (since expected failures may be
1210           omitted)
1211         """
1212         # test failures
1213         if self._options.full_results_html:
1214             results_title = "Test Failures"
1215             test_files = result_summary.failures.keys()
1216         else:
1217             results_title = "Unexpected Test Failures"
1218             unexpected_failures = self._get_failures(result_summary,
1219                 include_crashes=True)
1220             test_files = unexpected_failures.keys()
1221         if not len(test_files):
1222             return False
1223
1224         out_filename = os.path.join(self._options.results_directory,
1225                                     "results.html")
1226         with codecs.open(out_filename, "w", "utf-8") as results_file:
1227             html = self._results_html(test_files, result_summary.failures, results_title)
1228             results_file.write(html)
1229
1230         return True
1231
1232     def _show_results_html_file(self):
1233         """Shows the results.html page."""
1234         results_filename = os.path.join(self._options.results_directory,
1235                                         "results.html")
1236         self._port.show_results_html_file(results_filename)
1237
1238
1239 def read_test_files(files):
1240     tests = []
1241     for file in files:
1242         try:
1243             with codecs.open(file, 'r', 'utf-8') as file_contents:
1244                 # FIXME: This could be cleaner using a list comprehension.
1245                 for line in file_contents:
1246                     line = test_expectations.strip_comments(line)
1247                     if line:
1248                         tests.append(line)
1249         except IOError, e:
1250             if e.errno == errno.ENOENT:
1251                 _log.critical('')
1252                 _log.critical('--test-list file "%s" not found' % file)
1253             raise
1254     return tests
1255
1256
1257 def run(port, options, args, regular_output=sys.stderr,
1258         buildbot_output=sys.stdout):
1259     """Run the tests.
1260
1261     Args:
1262       port: Port object for port-specific behavior
1263       options: a dictionary of command line options
1264       args: a list of sub directories or files to test
1265       regular_output: a stream-like object that we can send logging/debug
1266           output to
1267       buildbot_output: a stream-like object that we can write all output that
1268           is intended to be parsed by the buildbot to
1269     Returns:
1270       the number of unexpected results that occurred, or -1 if there is an
1271           error.
1272
1273     """
1274     _set_up_derived_options(port, options)
1275
1276     printer = printing.Printer(port, options, regular_output, buildbot_output,
1277         int(options.child_processes), options.experimental_fully_parallel)
1278     if options.help_printing:
1279         printer.help_printing()
1280         printer.cleanup()
1281         return 0
1282
1283     last_unexpected_results = _gather_unexpected_results(options)
1284     if options.print_last_failures:
1285         printer.write("\n".join(last_unexpected_results) + "\n")
1286         printer.cleanup()
1287         return 0
1288
1289     broker = message_broker.get(port, options)
1290
1291     # We wrap any parts of the run that are slow or likely to raise exceptions
1292     # in a try/finally to ensure that we clean up the logging configuration.
1293     num_unexpected_results = -1
1294     try:
1295         test_runner = TestRunner(port, options, printer, broker)
1296         test_runner._print_config()
1297
1298         printer.print_update("Collecting tests ...")
1299         try:
1300             test_runner.collect_tests(args, last_unexpected_results)
1301         except IOError, e:
1302             if e.errno == errno.ENOENT:
1303                 return -1
1304             raise
1305
1306         printer.print_update("Parsing expectations ...")
1307         if options.lint_test_files:
1308             return test_runner.lint()
1309         test_runner.parse_expectations(port.test_platform_name(),
1310                                        options.configuration == 'Debug')
1311
1312         printer.print_update("Checking build ...")
1313         if not port.check_build(test_runner.needs_http()):
1314             _log.error("Build check failed")
1315             return -1
1316
1317         result_summary = test_runner.set_up_run()
1318         if result_summary:
1319             num_unexpected_results = test_runner.run(result_summary)
1320             test_runner.clean_up_run()
1321             _log.debug("Testing completed, Exit status: %d" %
1322                        num_unexpected_results)
1323     finally:
1324         broker.cleanup()
1325         printer.cleanup()
1326
1327     return num_unexpected_results
1328
1329
1330 def _set_up_derived_options(port_obj, options):
1331     """Sets the options values that depend on other options values."""
1332
1333     if options.worker_model == 'inline':
1334         if options.child_processes and int(options.child_processes) > 1:
1335             _log.warning("--worker-model=inline overrides --child-processes")
1336         options.child_processes = "1"
1337     if not options.child_processes:
1338         options.child_processes = os.environ.get("WEBKIT_TEST_CHILD_PROCESSES",
1339                                                  str(port_obj.default_child_processes()))
1340
1341     if not options.configuration:
1342         options.configuration = port_obj.default_configuration()
1343
1344     if options.pixel_tests is None:
1345         options.pixel_tests = True
1346
1347     if not options.use_apache:
1348         options.use_apache = sys.platform in ('darwin', 'linux2')
1349
1350     if not os.path.isabs(options.results_directory):
1351         # This normalizes the path to the build dir.
1352         # FIXME: how this happens is not at all obvious; this is a dumb
1353         # interface and should be cleaned up.
1354         options.results_directory = port_obj.results_directory()
1355
1356     if not options.time_out_ms:
1357         if options.configuration == "Debug":
1358             options.time_out_ms = str(2 * TestRunner.DEFAULT_TEST_TIMEOUT_MS)
1359         else:
1360             options.time_out_ms = str(TestRunner.DEFAULT_TEST_TIMEOUT_MS)
1361
1362     options.slow_time_out_ms = str(5 * int(options.time_out_ms))
1363
1364
1365 def _gather_unexpected_results(options):
1366     """Returns the unexpected results from the previous run, if any."""
1367     last_unexpected_results = []
1368     if options.print_last_failures or options.retest_last_failures:
1369         unexpected_results_filename = os.path.join(
1370         options.results_directory, "unexpected_results.json")
1371         with codecs.open(unexpected_results_filename, "r", "utf-8") as file:
1372             results = simplejson.load(file)
1373         last_unexpected_results = results['tests'].keys()
1374     return last_unexpected_results
1375
1376
1377 def _compat_shim_callback(option, opt_str, value, parser):
1378     print "Ignoring unsupported option: %s" % opt_str
1379
1380
1381 def _compat_shim_option(option_name, **kwargs):
1382     return optparse.make_option(option_name, action="callback",
1383         callback=_compat_shim_callback,
1384         help="Ignored, for old-run-webkit-tests compat only.", **kwargs)
1385
1386
1387 def parse_args(args=None):
1388     """Provides a default set of command line args.
1389
1390     Returns a tuple of options, args from optparse"""
1391
1392     # FIXME: All of these options should be stored closer to the code which
1393     # FIXME: actually uses them. configuration_options should move
1394     # FIXME: to WebKitPort and be shared across all scripts.
1395     configuration_options = [
1396         optparse.make_option("-t", "--target", dest="configuration",
1397                              help="(DEPRECATED)"),
1398         # FIXME: --help should display which configuration is default.
1399         optparse.make_option('--debug', action='store_const', const='Debug',
1400                              dest="configuration",
1401                              help='Set the configuration to Debug'),
1402         optparse.make_option('--release', action='store_const',
1403                              const='Release', dest="configuration",
1404                              help='Set the configuration to Release'),
1405         # old-run-webkit-tests also accepts -c, --configuration CONFIGURATION.
1406     ]
1407
1408     print_options = printing.print_options()
1409
1410     # FIXME: These options should move onto the ChromiumPort.
1411     chromium_options = [
1412         optparse.make_option("--chromium", action="store_true", default=False,
1413             help="use the Chromium port"),
1414         optparse.make_option("--startup-dialog", action="store_true",
1415             default=False, help="create a dialog on DumpRenderTree startup"),
1416         optparse.make_option("--gp-fault-error-box", action="store_true",
1417             default=False, help="enable Windows GP fault error box"),
1418         optparse.make_option("--multiple-loads",
1419             type="int", help="turn on multiple loads of each test"),
1420         optparse.make_option("--js-flags",
1421             type="string", help="JavaScript flags to pass to tests"),
1422         optparse.make_option("--nocheck-sys-deps", action="store_true",
1423             default=False,
1424             help="Don't check the system dependencies (themes)"),
1425         optparse.make_option("--use-drt", action="store_true",
1426             default=None,
1427             help="Use DumpRenderTree instead of test_shell"),
1428         optparse.make_option("--accelerated-compositing",
1429             action="store_true",
1430             help="Use hardware-accelated compositing for rendering"),
1431         optparse.make_option("--no-accelerated-compositing",
1432             action="store_false",
1433             dest="accelerated_compositing",
1434             help="Don't use hardware-accelerated compositing for rendering"),
1435         optparse.make_option("--accelerated-2d-canvas",
1436             action="store_true",
1437             help="Use hardware-accelerated 2D Canvas calls"),
1438         optparse.make_option("--no-accelerated-2d-canvas",
1439             action="store_false",
1440             dest="accelerated_2d_canvas",
1441             help="Don't use hardware-accelerated 2D Canvas calls"),
1442     ]
1443
1444     # Missing Mac-specific old-run-webkit-tests options:
1445     # FIXME: Need: -g, --guard for guard malloc support on Mac.
1446     # FIXME: Need: -l --leaks    Enable leaks checking.
1447     # FIXME: Need: --sample-on-timeout Run sample on timeout
1448
1449     old_run_webkit_tests_compat = [
1450         # NRWT doesn't generate results by default anyway.
1451         _compat_shim_option("--no-new-test-results"),
1452         # NRWT doesn't sample on timeout yet anyway.
1453         _compat_shim_option("--no-sample-on-timeout"),
1454         # FIXME: NRWT needs to support remote links eventually.
1455         _compat_shim_option("--use-remote-links-to-tests"),
1456         # FIXME: NRWT doesn't need this option as much since failures are
1457         # designed to be cheap.  We eventually plan to add this support.
1458         _compat_shim_option("--exit-after-n-failures", nargs=1, type="int"),
1459     ]
1460
1461     results_options = [
1462         # NEED for bots: --use-remote-links-to-tests Link to test files
1463         # within the SVN repository in the results.
1464         optparse.make_option("-p", "--pixel-tests", action="store_true",
1465             dest="pixel_tests", help="Enable pixel-to-pixel PNG comparisons"),
1466         optparse.make_option("--no-pixel-tests", action="store_false",
1467             dest="pixel_tests", help="Disable pixel-to-pixel PNG comparisons"),
1468         optparse.make_option("--tolerance",
1469             help="Ignore image differences less than this percentage (some "
1470                 "ports may ignore this option)", type="float"),
1471         optparse.make_option("--results-directory",
1472             default="layout-test-results",
1473             help="Output results directory source dir, relative to Debug or "
1474                  "Release"),
1475         optparse.make_option("--new-baseline", action="store_true",
1476             default=False, help="Save all generated results as new baselines "
1477                  "into the platform directory, overwriting whatever's "
1478                  "already there."),
1479         optparse.make_option("--reset-results", action="store_true",
1480             default=False, help="Reset any existing baselines to the "
1481                  "generated results"),
1482         optparse.make_option("--no-show-results", action="store_false",
1483             default=True, dest="show_results",
1484             help="Don't launch a browser with results after the tests "
1485                  "are done"),
1486         # FIXME: We should have a helper function to do this sort of
1487         # deprectated mapping and automatically log, etc.
1488         optparse.make_option("--noshow-results", action="store_false",
1489             dest="show_results",
1490             help="Deprecated, same as --no-show-results."),
1491         optparse.make_option("--no-launch-safari", action="store_false",
1492             dest="show_results",
1493             help="old-run-webkit-tests compat, same as --noshow-results."),
1494         # old-run-webkit-tests:
1495         # --[no-]launch-safari    Launch (or do not launch) Safari to display
1496         #                         test results (default: launch)
1497         optparse.make_option("--full-results-html", action="store_true",
1498             default=False,
1499             help="Show all failures in results.html, rather than only "
1500                  "regressions"),
1501         optparse.make_option("--clobber-old-results", action="store_true",
1502             default=False, help="Clobbers test results from previous runs."),
1503         optparse.make_option("--platform",
1504             help="Override the platform for expected results"),
1505         optparse.make_option("--no-record-results", action="store_false",
1506             default=True, dest="record_results",
1507             help="Don't record the results."),
1508         # old-run-webkit-tests also has HTTP toggle options:
1509         # --[no-]http                     Run (or do not run) http tests
1510         #                                 (default: run)
1511     ]
1512
1513     test_options = [
1514         optparse.make_option("--build", dest="build",
1515             action="store_true", default=True,
1516             help="Check to ensure the DumpRenderTree build is up-to-date "
1517                  "(default)."),
1518         optparse.make_option("--no-build", dest="build",
1519             action="store_false", help="Don't check to see if the "
1520                                        "DumpRenderTree build is up-to-date."),
1521         # old-run-webkit-tests has --valgrind instead of wrapper.
1522         optparse.make_option("--wrapper",
1523             help="wrapper command to insert before invocations of "
1524                  "DumpRenderTree; option is split on whitespace before "
1525                  "running. (Example: --wrapper='valgrind --smc-check=all')"),
1526         # old-run-webkit-tests:
1527         # -i|--ignore-tests               Comma-separated list of directories
1528         #                                 or tests to ignore
1529         optparse.make_option("--test-list", action="append",
1530             help="read list of tests to run from file", metavar="FILE"),
1531         # old-run-webkit-tests uses --skipped==[default|ignore|only]
1532         # instead of --force:
1533         optparse.make_option("--force", action="store_true", default=False,
1534             help="Run all tests, even those marked SKIP in the test list"),
1535         optparse.make_option("--use-apache", action="store_true",
1536             default=False, help="Whether to use apache instead of lighttpd."),
1537         optparse.make_option("--time-out-ms",
1538             help="Set the timeout for each test"),
1539         # old-run-webkit-tests calls --randomize-order --random:
1540         optparse.make_option("--randomize-order", action="store_true",
1541             default=False, help=("Run tests in random order (useful "
1542                                 "for tracking down corruption)")),
1543         optparse.make_option("--run-chunk",
1544             help=("Run a specified chunk (n:l), the nth of len l, "
1545                  "of the layout tests")),
1546         optparse.make_option("--run-part", help=("Run a specified part (n:m), "
1547                   "the nth of m parts, of the layout tests")),
1548         # old-run-webkit-tests calls --batch-size: --nthly n
1549         #   Restart DumpRenderTree every n tests (default: 1000)
1550         optparse.make_option("--batch-size",
1551             help=("Run a the tests in batches (n), after every n tests, "
1552                   "DumpRenderTree is relaunched."), type="int", default=0),
1553         # old-run-webkit-tests calls --run-singly: -1|--singly
1554         # Isolate each test case run (implies --nthly 1 --verbose)
1555         optparse.make_option("--run-singly", action="store_true",
1556             default=False, help="run a separate DumpRenderTree for each test"),
1557         optparse.make_option("--child-processes",
1558             help="Number of DumpRenderTrees to run in parallel."),
1559         # FIXME: Display default number of child processes that will run.
1560         optparse.make_option("--worker-model", action="store",
1561             default="threads", help=("controls worker model. Valid values are "
1562             "'inline' and 'threads' (default).")),
1563         optparse.make_option("--experimental-fully-parallel",
1564             action="store_true", default=False,
1565             help="run all tests in parallel"),
1566         # FIXME: Need --exit-after-n-failures N
1567         #      Exit after the first N failures instead of running all tests
1568         # FIXME: Need --exit-after-n-crashes N
1569         #      Exit after the first N crashes instead of running all tests
1570         # FIXME: consider: --iterations n
1571         #      Number of times to run the set of tests (e.g. ABCABCABC)
1572         optparse.make_option("--print-last-failures", action="store_true",
1573             default=False, help="Print the tests in the last run that "
1574             "had unexpected failures (or passes)."),
1575         optparse.make_option("--retest-last-failures", action="store_true",
1576             default=False, help="re-test the tests in the last run that "
1577             "had unexpected failures (or passes)."),
1578         optparse.make_option("--retry-failures", action="store_true",
1579             default=True,
1580             help="Re-try any tests that produce unexpected results (default)"),
1581         optparse.make_option("--no-retry-failures", action="store_false",
1582             dest="retry_failures",
1583             help="Don't re-try any tests that produce unexpected results."),
1584     ]
1585
1586     misc_options = [
1587         optparse.make_option("--lint-test-files", action="store_true",
1588         default=False, help=("Makes sure the test files parse for all "
1589                             "configurations. Does not run any tests.")),
1590     ]
1591
1592     # FIXME: Move these into json_results_generator.py
1593     results_json_options = [
1594         optparse.make_option("--master-name", help="The name of the buildbot master."),
1595         optparse.make_option("--builder-name", default="DUMMY_BUILDER_NAME",
1596             help=("The name of the builder shown on the waterfall running "
1597                   "this script e.g. WebKit.")),
1598         optparse.make_option("--build-name", default="DUMMY_BUILD_NAME",
1599             help=("The name of the builder used in its path, e.g. "
1600                   "webkit-rel.")),
1601         optparse.make_option("--build-number", default="DUMMY_BUILD_NUMBER",
1602             help=("The build number of the builder running this script.")),
1603         optparse.make_option("--test-results-server", default="",
1604             help=("If specified, upload results json files to this appengine "
1605                   "server.")),
1606         optparse.make_option("--upload-full-results",
1607             action="store_true",
1608             default=False,
1609             help="If true, upload full json results to server."),
1610     ]
1611
1612     option_list = (configuration_options + print_options +
1613                    chromium_options + results_options + test_options +
1614                    misc_options + results_json_options +
1615                    old_run_webkit_tests_compat)
1616     option_parser = optparse.OptionParser(option_list=option_list)
1617
1618     return option_parser.parse_args(args)
1619
1620
1621 def main():
1622     options, args = parse_args()
1623     port_obj = port.get(options.platform, options)
1624     return run(port_obj, options, args)
1625
1626
1627 if '__main__' == __name__:
1628     try:
1629         sys.exit(main())
1630     except KeyboardInterrupt:
1631         # this mirrors what the shell normally does
1632         sys.exit(signal.SIGINT + 128)