9bb76047f72928b4f632acf4cca3be2c84ab458e
[WebKit.git] / WebKitTools / Scripts / webkitpy / layout_tests / run_webkit_tests.py
1 #!/usr/bin/env python
2 # Copyright (C) 2010 Google Inc. All rights reserved.
3 # Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Szeged
4 #
5 # Redistribution and use in source and binary forms, with or without
6 # modification, are permitted provided that the following conditions are
7 # met:
8 #
9 #     * Redistributions of source code must retain the above copyright
10 # notice, this list of conditions and the following disclaimer.
11 #     * Redistributions in binary form must reproduce the above
12 # copyright notice, this list of conditions and the following disclaimer
13 # in the documentation and/or other materials provided with the
14 # distribution.
15 #     * Neither the name of Google Inc. nor the names of its
16 # contributors may be used to endorse or promote products derived from
17 # this software without specific prior written permission.
18 #
19 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30
31 """Run layout tests.
32
33 This is a port of the existing webkit test script run-webkit-tests.
34
35 The TestRunner class runs a series of tests (TestType interface) against a set
36 of test files.  If a test file fails a TestType, it returns a list TestFailure
37 objects to the TestRunner.  The TestRunner then aggregates the TestFailures to
38 create a final report.
39
40 This script reads several files, if they exist in the test_lists subdirectory
41 next to this script itself.  Each should contain a list of paths to individual
42 tests or entire subdirectories of tests, relative to the outermost test
43 directory.  Entire lines starting with '//' (comments) will be ignored.
44
45 For details of the files' contents and purposes, see test_lists/README.
46 """
47
48 from __future__ import with_statement
49
50 import codecs
51 import errno
52 import glob
53 import logging
54 import math
55 import optparse
56 import os
57 import platform
58 import Queue
59 import random
60 import re
61 import shutil
62 import signal
63 import sys
64 import time
65 import traceback
66
67 from layout_package import dump_render_tree_thread
68 from layout_package import json_layout_results_generator
69 from layout_package import message_broker
70 from layout_package import printing
71 from layout_package import test_expectations
72 from layout_package import test_failures
73 from layout_package import test_results
74 from layout_package import test_results_uploader
75
76 from webkitpy.common.system import user
77 from webkitpy.thirdparty import simplejson
78
79 import port
80
81 _log = logging.getLogger("webkitpy.layout_tests.run_webkit_tests")
82
83 # Builder base URL where we have the archived test results.
84 BUILDER_BASE_URL = "http://build.chromium.org/buildbot/layout_test_results/"
85
86 LAYOUT_TESTS_DIRECTORY = "LayoutTests" + os.sep
87
88 TestExpectationsFile = test_expectations.TestExpectationsFile
89
90
91 class TestInput:
92     """Groups information about a test for easy passing of data."""
93
94     def __init__(self, filename, timeout):
95         """Holds the input parameters for a test.
96         Args:
97           filename: Full path to the test.
98           timeout: Timeout in msecs the driver should use while running the test
99           """
100         # FIXME: filename should really be test_name as a relative path.
101         self.filename = filename
102         self.timeout = timeout
103         # The image_hash is used to avoid doing an image dump if the
104         # checksums match. The image_hash is set later, and only if it is needed
105         # for the test.
106         self.image_hash = None
107
108
109 class ResultSummary(object):
110     """A class for partitioning the test results we get into buckets.
111
112     This class is basically a glorified struct and it's private to this file
113     so we don't bother with any information hiding."""
114
115     def __init__(self, expectations, test_files):
116         self.total = len(test_files)
117         self.remaining = self.total
118         self.expectations = expectations
119         self.expected = 0
120         self.unexpected = 0
121         self.tests_by_expectation = {}
122         self.tests_by_timeline = {}
123         self.results = {}
124         self.unexpected_results = {}
125         self.failures = {}
126         self.tests_by_expectation[test_expectations.SKIP] = set()
127         for expectation in TestExpectationsFile.EXPECTATIONS.values():
128             self.tests_by_expectation[expectation] = set()
129         for timeline in TestExpectationsFile.TIMELINES.values():
130             self.tests_by_timeline[timeline] = (
131                 expectations.get_tests_with_timeline(timeline))
132
133     def add(self, result, expected):
134         """Add a TestResult into the appropriate bin.
135
136         Args:
137           result: TestResult from dump_render_tree_thread.
138           expected: whether the result was what we expected it to be.
139         """
140
141         self.tests_by_expectation[result.type].add(result.filename)
142         self.results[result.filename] = result
143         self.remaining -= 1
144         if len(result.failures):
145             self.failures[result.filename] = result.failures
146         if expected:
147             self.expected += 1
148         else:
149             self.unexpected_results[result.filename] = result.type
150             self.unexpected += 1
151
152
153 def summarize_unexpected_results(port_obj, expectations, result_summary,
154                                  retry_summary):
155     """Summarize any unexpected results as a dict.
156
157     FIXME: split this data structure into a separate class?
158
159     Args:
160         port_obj: interface to port-specific hooks
161         expectations: test_expectations.TestExpectations object
162         result_summary: summary object from initial test runs
163         retry_summary: summary object from final test run of retried tests
164     Returns:
165         A dictionary containing a summary of the unexpected results from the
166         run, with the following fields:
167         'version': a version indicator (1 in this version)
168         'fixable': # of fixable tests (NOW - PASS)
169         'skipped': # of skipped tests (NOW & SKIPPED)
170         'num_regressions': # of non-flaky failures
171         'num_flaky': # of flaky failures
172         'num_passes': # of unexpected passes
173         'tests': a dict of tests -> {'expected': '...', 'actual': '...'}
174     """
175     results = {}
176     results['version'] = 1
177
178     tbe = result_summary.tests_by_expectation
179     tbt = result_summary.tests_by_timeline
180     results['fixable'] = len(tbt[test_expectations.NOW] -
181                                 tbe[test_expectations.PASS])
182     results['skipped'] = len(tbt[test_expectations.NOW] &
183                                 tbe[test_expectations.SKIP])
184
185     num_passes = 0
186     num_flaky = 0
187     num_regressions = 0
188     keywords = {}
189     for k, v in TestExpectationsFile.EXPECTATIONS.iteritems():
190         keywords[v] = k.upper()
191
192     tests = {}
193     for filename, result in result_summary.unexpected_results.iteritems():
194         # Note that if a test crashed in the original run, we ignore
195         # whether or not it crashed when we retried it (if we retried it),
196         # and always consider the result not flaky.
197         test = port_obj.relative_test_filename(filename)
198         expected = expectations.get_expectations_string(filename)
199         actual = [keywords[result]]
200
201         if result == test_expectations.PASS:
202             num_passes += 1
203         elif result == test_expectations.CRASH:
204             num_regressions += 1
205         else:
206             if filename not in retry_summary.unexpected_results:
207                 actual.extend(expectations.get_expectations_string(
208                     filename).split(" "))
209                 num_flaky += 1
210             else:
211                 retry_result = retry_summary.unexpected_results[filename]
212                 if result != retry_result:
213                     actual.append(keywords[retry_result])
214                     num_flaky += 1
215                 else:
216                     num_regressions += 1
217
218         tests[test] = {}
219         tests[test]['expected'] = expected
220         tests[test]['actual'] = " ".join(actual)
221
222     results['tests'] = tests
223     results['num_passes'] = num_passes
224     results['num_flaky'] = num_flaky
225     results['num_regressions'] = num_regressions
226
227     return results
228
229
230 class TestRunner:
231     """A class for managing running a series of tests on a series of layout
232     test files."""
233
234     HTTP_SUBDIR = os.sep.join(['', 'http', ''])
235     WEBSOCKET_SUBDIR = os.sep.join(['', 'websocket', ''])
236
237     # The per-test timeout in milliseconds, if no --time-out-ms option was
238     # given to run_webkit_tests. This should correspond to the default timeout
239     # in DumpRenderTree.
240     DEFAULT_TEST_TIMEOUT_MS = 6 * 1000
241
242     def __init__(self, port, options, printer):
243         """Initialize test runner data structures.
244
245         Args:
246           port: an object implementing port-specific
247           options: a dictionary of command line options
248           printer: a Printer object to record updates to.
249         """
250         self._port = port
251         self._options = options
252         self._printer = printer
253
254         # disable wss server. need to install pyOpenSSL on buildbots.
255         # self._websocket_secure_server = websocket_server.PyWebSocket(
256         #        options.results_directory, use_tls=True, port=9323)
257
258         # a set of test files, and the same tests as a list
259         self._test_files = set()
260         self._test_files_list = None
261         self._result_queue = Queue.Queue()
262         self._retrying = False
263
264     def collect_tests(self, args, last_unexpected_results):
265         """Find all the files to test.
266
267         Args:
268           args: list of test arguments from the command line
269           last_unexpected_results: list of unexpected results to retest, if any
270
271         """
272         paths = [self._strip_test_dir_prefix(arg) for arg in args if arg and arg != '']
273         paths += last_unexpected_results
274         if self._options.test_list:
275             paths += read_test_files(self._options.test_list)
276         self._test_files = self._port.tests(paths)
277
278     def _strip_test_dir_prefix(self, path):
279         if path.startswith(LAYOUT_TESTS_DIRECTORY):
280             return path[len(LAYOUT_TESTS_DIRECTORY):]
281         return path
282
283     def lint(self):
284         # Creating the expecations for each platform/configuration pair does
285         # all the test list parsing and ensures it's correct syntax (e.g. no
286         # dupes).
287         for platform_name in self._port.test_platform_names():
288             self.parse_expectations(platform_name, is_debug_mode=True)
289             self.parse_expectations(platform_name, is_debug_mode=False)
290         self._printer.write("")
291         _log.info("If there are no fail messages, errors or exceptions, "
292                   "then the lint succeeded.")
293         return 0
294
295     def parse_expectations(self, test_platform_name, is_debug_mode):
296         """Parse the expectations from the test_list files and return a data
297         structure holding them. Throws an error if the test_list files have
298         invalid syntax."""
299         if self._options.lint_test_files:
300             test_files = None
301         else:
302             test_files = self._test_files
303
304         try:
305             expectations_str = self._port.test_expectations()
306             overrides_str = self._port.test_expectations_overrides()
307             self._expectations = test_expectations.TestExpectations(
308                 self._port, test_files, expectations_str, test_platform_name,
309                 is_debug_mode, self._options.lint_test_files,
310                 overrides=overrides_str)
311             return self._expectations
312         except SyntaxError, err:
313             if self._options.lint_test_files:
314                 print str(err)
315             else:
316                 raise err
317
318     def prepare_lists_and_print_output(self):
319         """Create appropriate subsets of test lists and returns a
320         ResultSummary object. Also prints expected test counts.
321         """
322
323         # Remove skipped - both fixable and ignored - files from the
324         # top-level list of files to test.
325         num_all_test_files = len(self._test_files)
326         self._printer.print_expected("Found:  %d tests" %
327                                      (len(self._test_files)))
328         if not num_all_test_files:
329             _log.critical('No tests to run.')
330             return None
331
332         skipped = set()
333         if num_all_test_files > 1 and not self._options.force:
334             skipped = self._expectations.get_tests_with_result_type(
335                            test_expectations.SKIP)
336             self._test_files -= skipped
337
338         # Create a sorted list of test files so the subset chunk,
339         # if used, contains alphabetically consecutive tests.
340         self._test_files_list = list(self._test_files)
341         if self._options.randomize_order:
342             random.shuffle(self._test_files_list)
343         else:
344             self._test_files_list.sort()
345
346         # If the user specifies they just want to run a subset of the tests,
347         # just grab a subset of the non-skipped tests.
348         if self._options.run_chunk or self._options.run_part:
349             chunk_value = self._options.run_chunk or self._options.run_part
350             test_files = self._test_files_list
351             try:
352                 (chunk_num, chunk_len) = chunk_value.split(":")
353                 chunk_num = int(chunk_num)
354                 assert(chunk_num >= 0)
355                 test_size = int(chunk_len)
356                 assert(test_size > 0)
357             except:
358                 _log.critical("invalid chunk '%s'" % chunk_value)
359                 return None
360
361             # Get the number of tests
362             num_tests = len(test_files)
363
364             # Get the start offset of the slice.
365             if self._options.run_chunk:
366                 chunk_len = test_size
367                 # In this case chunk_num can be really large. We need
368                 # to make the slave fit in the current number of tests.
369                 slice_start = (chunk_num * chunk_len) % num_tests
370             else:
371                 # Validate the data.
372                 assert(test_size <= num_tests)
373                 assert(chunk_num <= test_size)
374
375                 # To count the chunk_len, and make sure we don't skip
376                 # some tests, we round to the next value that fits exactly
377                 # all the parts.
378                 rounded_tests = num_tests
379                 if rounded_tests % test_size != 0:
380                     rounded_tests = (num_tests + test_size -
381                                      (num_tests % test_size))
382
383                 chunk_len = rounded_tests / test_size
384                 slice_start = chunk_len * (chunk_num - 1)
385                 # It does not mind if we go over test_size.
386
387             # Get the end offset of the slice.
388             slice_end = min(num_tests, slice_start + chunk_len)
389
390             files = test_files[slice_start:slice_end]
391
392             tests_run_msg = 'Running: %d tests (chunk slice [%d:%d] of %d)' % (
393                 (slice_end - slice_start), slice_start, slice_end, num_tests)
394             self._printer.print_expected(tests_run_msg)
395
396             # If we reached the end and we don't have enough tests, we run some
397             # from the beginning.
398             if slice_end - slice_start < chunk_len:
399                 extra = chunk_len - (slice_end - slice_start)
400                 extra_msg = ('   last chunk is partial, appending [0:%d]' %
401                             extra)
402                 self._printer.print_expected(extra_msg)
403                 tests_run_msg += "\n" + extra_msg
404                 files.extend(test_files[0:extra])
405             tests_run_filename = os.path.join(self._options.results_directory,
406                                               "tests_run.txt")
407             with codecs.open(tests_run_filename, "w", "utf-8") as file:
408                 file.write(tests_run_msg + "\n")
409
410             len_skip_chunk = int(len(files) * len(skipped) /
411                                  float(len(self._test_files)))
412             skip_chunk_list = list(skipped)[0:len_skip_chunk]
413             skip_chunk = set(skip_chunk_list)
414
415             # Update expectations so that the stats are calculated correctly.
416             # We need to pass a list that includes the right # of skipped files
417             # to ParseExpectations so that ResultSummary() will get the correct
418             # stats. So, we add in the subset of skipped files, and then
419             # subtract them back out.
420             self._test_files_list = files + skip_chunk_list
421             self._test_files = set(self._test_files_list)
422
423             self._expectations = self.parse_expectations(
424                 self._port.test_platform_name(),
425                 self._options.configuration == 'Debug')
426
427             self._test_files = set(files)
428             self._test_files_list = files
429         else:
430             skip_chunk = skipped
431
432         result_summary = ResultSummary(self._expectations,
433             self._test_files | skip_chunk)
434         self._print_expected_results_of_type(result_summary,
435             test_expectations.PASS, "passes")
436         self._print_expected_results_of_type(result_summary,
437             test_expectations.FAIL, "failures")
438         self._print_expected_results_of_type(result_summary,
439             test_expectations.FLAKY, "flaky")
440         self._print_expected_results_of_type(result_summary,
441             test_expectations.SKIP, "skipped")
442
443         if self._options.force:
444             self._printer.print_expected('Running all tests, including '
445                                          'skips (--force)')
446         else:
447             # Note that we don't actually run the skipped tests (they were
448             # subtracted out of self._test_files, above), but we stub out the
449             # results here so the statistics can remain accurate.
450             for test in skip_chunk:
451                 result = test_results.TestResult(test,
452                     failures=[], test_run_time=0, total_time_for_all_diffs=0,
453                     time_for_diffs=0)
454                 result.type = test_expectations.SKIP
455                 result_summary.add(result, expected=True)
456         self._printer.print_expected('')
457
458         return result_summary
459
460     def _get_dir_for_test_file(self, test_file):
461         """Returns the highest-level directory by which to shard the given
462         test file."""
463         index = test_file.rfind(os.sep + LAYOUT_TESTS_DIRECTORY)
464
465         test_file = test_file[index + len(LAYOUT_TESTS_DIRECTORY):]
466         test_file_parts = test_file.split(os.sep, 1)
467         directory = test_file_parts[0]
468         test_file = test_file_parts[1]
469
470         # The http tests are very stable on mac/linux.
471         # TODO(ojan): Make the http server on Windows be apache so we can
472         # turn shard the http tests there as well. Switching to apache is
473         # what made them stable on linux/mac.
474         return_value = directory
475         while ((directory != 'http' or sys.platform in ('darwin', 'linux2'))
476                 and test_file.find(os.sep) >= 0):
477             test_file_parts = test_file.split(os.sep, 1)
478             directory = test_file_parts[0]
479             return_value = os.path.join(return_value, directory)
480             test_file = test_file_parts[1]
481
482         return return_value
483
484     def _get_test_input_for_file(self, test_file):
485         """Returns the appropriate TestInput object for the file. Mostly this
486         is used for looking up the timeout value (in ms) to use for the given
487         test."""
488         if self._expectations.has_modifier(test_file, test_expectations.SLOW):
489             return TestInput(test_file, self._options.slow_time_out_ms)
490         return TestInput(test_file, self._options.time_out_ms)
491
492     def _test_requires_lock(self, test_file):
493         """Return True if the test needs to be locked when
494         running multiple copies of NRWTs."""
495         split_path = test_file.split(os.sep)
496         return 'http' in split_path or 'websocket' in split_path
497
498     def _get_test_file_queue(self, test_files):
499         """Create the thread safe queue of lists of (test filenames, test URIs)
500         tuples. Each TestShellThread pulls a list from this queue and runs
501         those tests in order before grabbing the next available list.
502
503         Shard the lists by directory. This helps ensure that tests that depend
504         on each other (aka bad tests!) continue to run together as most
505         cross-tests dependencies tend to occur within the same directory.
506
507         Return:
508           The Queue of lists of TestInput objects.
509         """
510
511         test_lists = []
512         tests_to_http_lock = []
513         if (self._options.experimental_fully_parallel or
514             self._is_single_threaded()):
515             for test_file in test_files:
516                 test_input = self._get_test_input_for_file(test_file)
517                 if self._test_requires_lock(test_file):
518                     tests_to_http_lock.append(test_input)
519                 else:
520                     test_lists.append((".", [test_input]))
521         else:
522             tests_by_dir = {}
523             for test_file in test_files:
524                 directory = self._get_dir_for_test_file(test_file)
525                 test_input = self._get_test_input_for_file(test_file)
526                 if self._test_requires_lock(test_file):
527                     tests_to_http_lock.append(test_input)
528                 else:
529                     tests_by_dir.setdefault(directory, [])
530                     tests_by_dir[directory].append(test_input)
531             # Sort by the number of tests in the dir so that the ones with the
532             # most tests get run first in order to maximize parallelization.
533             # Number of tests is a good enough, but not perfect, approximation
534             # of how long that set of tests will take to run. We can't just use
535             # a PriorityQueue until we move to Python 2.6.
536             for directory in tests_by_dir:
537                 test_list = tests_by_dir[directory]
538                 # Keep the tests in alphabetical order.
539                 # FIXME: Remove once tests are fixed so they can be run in any
540                 # order.
541                 test_list.reverse()
542                 test_list_tuple = (directory, test_list)
543                 test_lists.append(test_list_tuple)
544             test_lists.sort(lambda a, b: cmp(len(b[1]), len(a[1])))
545
546         # Put the http tests first. There are only a couple hundred of them,
547         # but each http test takes a very long time to run, so sorting by the
548         # number of tests doesn't accurately capture how long they take to run.
549         if tests_to_http_lock:
550             tests_to_http_lock.reverse()
551             test_lists.insert(0, ("tests_to_http_lock", tests_to_http_lock))
552
553         filename_queue = Queue.Queue()
554         for item in test_lists:
555             filename_queue.put(item)
556         return filename_queue
557
558     def _contains_tests(self, subdir):
559         for test_file in self._test_files:
560             if test_file.find(subdir) >= 0:
561                 return True
562         return False
563
564     def _instantiate_dump_render_tree_threads(self, test_files,
565                                               result_summary):
566         """Instantitates and starts the TestShellThread(s).
567
568         Return:
569           The list of threads.
570         """
571         filename_queue = self._get_test_file_queue(test_files)
572
573         # Instantiate TestShellThreads and start them.
574         threads = []
575         for worker_number in xrange(int(self._options.child_processes)):
576             thread = dump_render_tree_thread.TestShellThread(self._port,
577                 self._options, worker_number,
578                 filename_queue, self._result_queue)
579             if self._is_single_threaded():
580                 thread.run_in_main_thread(self, result_summary)
581             else:
582                 thread.start()
583             threads.append(thread)
584
585         return threads
586
587     def _is_single_threaded(self):
588         """Returns whether we should run all the tests in the main thread."""
589         return int(self._options.child_processes) == 1
590
591     def _run_tests(self, file_list, result_summary):
592         """Runs the tests in the file_list.
593
594         Return: A tuple (keyboard_interrupted, thread_timings, test_timings,
595             individual_test_timings)
596             keyboard_interrupted is whether someone typed Ctrl^C
597             thread_timings is a list of dicts with the total runtime
598               of each thread with 'name', 'num_tests', 'total_time' properties
599             test_timings is a list of timings for each sharded subdirectory
600               of the form [time, directory_name, num_tests]
601             individual_test_timings is a list of run times for each test
602               in the form {filename:filename, test_run_time:test_run_time}
603             result_summary: summary object to populate with the results
604         """
605         # FIXME: We should use webkitpy.tool.grammar.pluralize here.
606         plural = ""
607         if not self._is_single_threaded():
608             plural = "s"
609         self._printer.print_update('Starting %s%s ...' %
610                                    (self._port.driver_name(), plural))
611         threads = self._instantiate_dump_render_tree_threads(file_list,
612                                                              result_summary)
613         self._printer.print_update("Starting testing ...")
614
615         keyboard_interrupted = self._wait_for_threads_to_finish(threads,
616                                                                 result_summary)
617         (thread_timings, test_timings, individual_test_timings) = \
618             self._collect_timing_info(threads)
619
620         return (keyboard_interrupted, thread_timings, test_timings,
621                 individual_test_timings)
622
623     def _wait_for_threads_to_finish(self, threads, result_summary):
624         keyboard_interrupted = False
625         try:
626             # Loop through all the threads waiting for them to finish.
627             some_thread_is_alive = True
628             while some_thread_is_alive:
629                 some_thread_is_alive = False
630                 t = time.time()
631                 for thread in threads:
632                     exception_info = thread.exception_info()
633                     if exception_info is not None:
634                         # Re-raise the thread's exception here to make it
635                         # clear that testing was aborted. Otherwise,
636                         # the tests that did not run would be assumed
637                         # to have passed.
638                         raise exception_info[0], exception_info[1], exception_info[2]
639
640                     if thread.isAlive():
641                         some_thread_is_alive = True
642                         next_timeout = thread.next_timeout()
643                         if (next_timeout and t > next_timeout):
644                             message_broker.log_wedged_thread(thread.id())
645                             thread.clear_next_timeout()
646
647                 self.update_summary(result_summary)
648
649                 if some_thread_is_alive:
650                     time.sleep(0.01)
651
652         except KeyboardInterrupt:
653             keyboard_interrupted = True
654             for thread in threads:
655                 thread.cancel()
656
657         return keyboard_interrupted
658
659     def _collect_timing_info(self, threads):
660         test_timings = {}
661         individual_test_timings = []
662         thread_timings = []
663
664         for thread in threads:
665             thread_timings.append({'name': thread.getName(),
666                                    'num_tests': thread.get_num_tests(),
667                                    'total_time': thread.get_total_time()})
668             test_timings.update(thread.get_test_group_timing_stats())
669             individual_test_timings.extend(thread.get_test_results())
670
671         return (thread_timings, test_timings, individual_test_timings)
672
673     def needs_http(self):
674         """Returns whether the test runner needs an HTTP server."""
675         return self._contains_tests(self.HTTP_SUBDIR)
676
677     def needs_websocket(self):
678         """Returns whether the test runner needs a WEBSOCKET server."""
679         return self._contains_tests(self.WEBSOCKET_SUBDIR)
680
681     def set_up_run(self):
682         """Configures the system to be ready to run tests.
683
684         Returns a ResultSummary object if we should continue to run tests,
685         or None if we should abort.
686
687         """
688         # This must be started before we check the system dependencies,
689         # since the helper may do things to make the setup correct.
690         self._printer.print_update("Starting helper ...")
691         self._port.start_helper()
692
693         # Check that the system dependencies (themes, fonts, ...) are correct.
694         if not self._options.nocheck_sys_deps:
695             self._printer.print_update("Checking system dependencies ...")
696             if not self._port.check_sys_deps(self.needs_http()):
697                 self._port.stop_helper()
698                 return None
699
700         if self._options.clobber_old_results:
701             self._clobber_old_results()
702
703         # Create the output directory if it doesn't already exist.
704         self._port.maybe_make_directory(self._options.results_directory)
705
706         self._port.setup_test_run()
707
708         self._printer.print_update("Preparing tests ...")
709         result_summary = self.prepare_lists_and_print_output()
710         if not result_summary:
711             return None
712
713         return result_summary
714
715     def run(self, result_summary):
716         """Run all our tests on all our test files.
717
718         For each test file, we run each test type. If there are any failures,
719         we collect them for reporting.
720
721         Args:
722           result_summary: a summary object tracking the test results.
723
724         Return:
725           The number of unexpected results (0 == success)
726         """
727         # gather_test_files() must have been called first to initialize us.
728         # If we didn't find any files to test, we've errored out already in
729         # prepare_lists_and_print_output().
730         assert(len(self._test_files))
731
732         start_time = time.time()
733
734         keyboard_interrupted, thread_timings, test_timings, \
735             individual_test_timings = (
736             self._run_tests(self._test_files_list, result_summary))
737
738         # We exclude the crashes from the list of results to retry, because
739         # we want to treat even a potentially flaky crash as an error.
740         failures = self._get_failures(result_summary, include_crashes=False)
741         retry_summary = result_summary
742         while (len(failures) and self._options.retry_failures and
743             not self._retrying and not keyboard_interrupted):
744             _log.info('')
745             _log.info("Retrying %d unexpected failure(s) ..." % len(failures))
746             _log.info('')
747             self._retrying = True
748             retry_summary = ResultSummary(self._expectations, failures.keys())
749             # Note that we intentionally ignore the return value here.
750             self._run_tests(failures.keys(), retry_summary)
751             failures = self._get_failures(retry_summary, include_crashes=True)
752
753         end_time = time.time()
754
755         self._print_timing_statistics(end_time - start_time,
756                                       thread_timings, test_timings,
757                                       individual_test_timings,
758                                       result_summary)
759
760         self._print_result_summary(result_summary)
761
762         sys.stdout.flush()
763         sys.stderr.flush()
764
765         self._printer.print_one_line_summary(result_summary.total,
766                                              result_summary.expected,
767                                              result_summary.unexpected)
768
769         unexpected_results = summarize_unexpected_results(self._port,
770             self._expectations, result_summary, retry_summary)
771         self._printer.print_unexpected_results(unexpected_results)
772
773         if self._options.record_results:
774             # Write the same data to log files and upload generated JSON files
775             # to appengine server.
776             self._upload_json_files(unexpected_results, result_summary,
777                                     individual_test_timings)
778
779         # Write the summary to disk (results.html) and display it if requested.
780         wrote_results = self._write_results_html_file(result_summary)
781         if self._options.show_results and wrote_results:
782             self._show_results_html_file()
783
784         # Now that we've completed all the processing we can, we re-raise
785         # a KeyboardInterrupt if necessary so the caller can handle it.
786         if keyboard_interrupted:
787             raise KeyboardInterrupt
788
789         # Ignore flaky failures and unexpected passes so we don't turn the
790         # bot red for those.
791         return unexpected_results['num_regressions']
792
793     def clean_up_run(self):
794         """Restores the system after we're done running tests."""
795
796         _log.debug("flushing stdout")
797         sys.stdout.flush()
798         _log.debug("flushing stderr")
799         sys.stderr.flush()
800         _log.debug("stopping helper")
801         self._port.stop_helper()
802
803     def update_summary(self, result_summary):
804         """Update the summary and print results with any completed tests."""
805         while True:
806             try:
807                 result = test_results.TestResult.loads(self._result_queue.get_nowait())
808             except Queue.Empty:
809                 return
810
811             expected = self._expectations.matches_an_expected_result(
812                 result.filename, result.type, self._options.pixel_tests)
813             result_summary.add(result, expected)
814             exp_str = self._expectations.get_expectations_string(
815                 result.filename)
816             got_str = self._expectations.expectation_to_string(result.type)
817             self._printer.print_test_result(result, expected, exp_str, got_str)
818             self._printer.print_progress(result_summary, self._retrying,
819                                          self._test_files_list)
820
821     def _clobber_old_results(self):
822         # Just clobber the actual test results directories since the other
823         # files in the results directory are explicitly used for cross-run
824         # tracking.
825         self._printer.print_update("Clobbering old results in %s" %
826                                    self._options.results_directory)
827         layout_tests_dir = self._port.layout_tests_dir()
828         possible_dirs = self._port.test_dirs()
829         for dirname in possible_dirs:
830             if os.path.isdir(os.path.join(layout_tests_dir, dirname)):
831                 shutil.rmtree(os.path.join(self._options.results_directory,
832                                            dirname),
833                               ignore_errors=True)
834
835     def _get_failures(self, result_summary, include_crashes):
836         """Filters a dict of results and returns only the failures.
837
838         Args:
839           result_summary: the results of the test run
840           include_crashes: whether crashes are included in the output.
841             We use False when finding the list of failures to retry
842             to see if the results were flaky. Although the crashes may also be
843             flaky, we treat them as if they aren't so that they're not ignored.
844         Returns:
845           a dict of files -> results
846         """
847         failed_results = {}
848         for test, result in result_summary.unexpected_results.iteritems():
849             if (result == test_expectations.PASS or
850                 result == test_expectations.CRASH and not include_crashes):
851                 continue
852             failed_results[test] = result
853
854         return failed_results
855
856     def _upload_json_files(self, unexpected_results, result_summary,
857                         individual_test_timings):
858         """Writes the results of the test run as JSON files into the results
859         dir and upload the files to the appengine server.
860
861         There are three different files written into the results dir:
862           unexpected_results.json: A short list of any unexpected results.
863             This is used by the buildbots to display results.
864           expectations.json: This is used by the flakiness dashboard.
865           results.json: A full list of the results - used by the flakiness
866             dashboard and the aggregate results dashboard.
867
868         Args:
869           unexpected_results: dict of unexpected results
870           result_summary: full summary object
871           individual_test_timings: list of test times (used by the flakiness
872             dashboard).
873         """
874         results_directory = self._options.results_directory
875         _log.debug("Writing JSON files in %s." % results_directory)
876         unexpected_json_path = os.path.join(results_directory, "unexpected_results.json")
877         with codecs.open(unexpected_json_path, "w", "utf-8") as file:
878             simplejson.dump(unexpected_results, file, sort_keys=True, indent=2)
879
880         # Write a json file of the test_expectations.txt file for the layout
881         # tests dashboard.
882         expectations_path = os.path.join(results_directory, "expectations.json")
883         expectations_json = \
884             self._expectations.get_expectations_json_for_all_platforms()
885         with codecs.open(expectations_path, "w", "utf-8") as file:
886             file.write(u"ADD_EXPECTATIONS(%s);" % expectations_json)
887
888         generator = json_layout_results_generator.JSONLayoutResultsGenerator(
889             self._port, self._options.builder_name, self._options.build_name,
890             self._options.build_number, self._options.results_directory,
891             BUILDER_BASE_URL, individual_test_timings,
892             self._expectations, result_summary, self._test_files_list,
893             not self._options.upload_full_results,
894             self._options.test_results_server,
895             "layout-tests",
896             self._options.master_name)
897
898         _log.debug("Finished writing JSON files.")
899
900         json_files = ["expectations.json"]
901         if self._options.upload_full_results:
902             json_files.append("results.json")
903         else:
904             json_files.append("incremental_results.json")
905
906         generator.upload_json_files(json_files)
907
908     def _print_config(self):
909         """Prints the configuration for the test run."""
910         p = self._printer
911         p.print_config("Using port '%s'" % self._port.name())
912         p.print_config("Placing test results in %s" %
913                        self._options.results_directory)
914         if self._options.new_baseline:
915             p.print_config("Placing new baselines in %s" %
916                            self._port.baseline_path())
917         p.print_config("Using %s build" % self._options.configuration)
918         if self._options.pixel_tests:
919             p.print_config("Pixel tests enabled")
920         else:
921             p.print_config("Pixel tests disabled")
922
923         p.print_config("Regular timeout: %s, slow test timeout: %s" %
924                        (self._options.time_out_ms,
925                         self._options.slow_time_out_ms))
926
927         if self._is_single_threaded():
928             p.print_config("Running one %s" % self._port.driver_name())
929         else:
930             p.print_config("Running %s %ss in parallel" %
931                            (self._options.child_processes,
932                             self._port.driver_name()))
933         p.print_config("")
934
935     def _print_expected_results_of_type(self, result_summary,
936                                         result_type, result_type_str):
937         """Print the number of the tests in a given result class.
938
939         Args:
940           result_summary - the object containing all the results to report on
941           result_type - the particular result type to report in the summary.
942           result_type_str - a string description of the result_type.
943         """
944         tests = self._expectations.get_tests_with_result_type(result_type)
945         now = result_summary.tests_by_timeline[test_expectations.NOW]
946         wontfix = result_summary.tests_by_timeline[test_expectations.WONTFIX]
947
948         # We use a fancy format string in order to print the data out in a
949         # nicely-aligned table.
950         fmtstr = ("Expect: %%5d %%-8s (%%%dd now, %%%dd wontfix)"
951                   % (self._num_digits(now), self._num_digits(wontfix)))
952         self._printer.print_expected(fmtstr %
953             (len(tests), result_type_str, len(tests & now), len(tests & wontfix)))
954
955     def _num_digits(self, num):
956         """Returns the number of digits needed to represent the length of a
957         sequence."""
958         ndigits = 1
959         if len(num):
960             ndigits = int(math.log10(len(num))) + 1
961         return ndigits
962
963     def _print_timing_statistics(self, total_time, thread_timings,
964                                directory_test_timings, individual_test_timings,
965                                result_summary):
966         """Record timing-specific information for the test run.
967
968         Args:
969           total_time: total elapsed time (in seconds) for the test run
970           thread_timings: wall clock time each thread ran for
971           directory_test_timings: timing by directory
972           individual_test_timings: timing by file
973           result_summary: summary object for the test run
974         """
975         self._printer.print_timing("Test timing:")
976         self._printer.print_timing("  %6.2f total testing time" % total_time)
977         self._printer.print_timing("")
978         self._printer.print_timing("Thread timing:")
979         cuml_time = 0
980         for t in thread_timings:
981             self._printer.print_timing("    %10s: %5d tests, %6.2f secs" %
982                   (t['name'], t['num_tests'], t['total_time']))
983             cuml_time += t['total_time']
984         self._printer.print_timing("   %6.2f cumulative, %6.2f optimal" %
985               (cuml_time, cuml_time / int(self._options.child_processes)))
986         self._printer.print_timing("")
987
988         self._print_aggregate_test_statistics(individual_test_timings)
989         self._print_individual_test_times(individual_test_timings,
990                                           result_summary)
991         self._print_directory_timings(directory_test_timings)
992
993     def _print_aggregate_test_statistics(self, individual_test_timings):
994         """Prints aggregate statistics (e.g. median, mean, etc.) for all tests.
995         Args:
996           individual_test_timings: List of dump_render_tree_thread.TestStats
997               for all tests.
998         """
999         test_types = []  # Unit tests don't actually produce any timings.
1000         if individual_test_timings:
1001             test_types = individual_test_timings[0].time_for_diffs.keys()
1002         times_for_dump_render_tree = []
1003         times_for_diff_processing = []
1004         times_per_test_type = {}
1005         for test_type in test_types:
1006             times_per_test_type[test_type] = []
1007
1008         for test_stats in individual_test_timings:
1009             times_for_dump_render_tree.append(test_stats.test_run_time)
1010             times_for_diff_processing.append(
1011                 test_stats.total_time_for_all_diffs)
1012             time_for_diffs = test_stats.time_for_diffs
1013             for test_type in test_types:
1014                 times_per_test_type[test_type].append(
1015                     time_for_diffs[test_type])
1016
1017         self._print_statistics_for_test_timings(
1018             "PER TEST TIME IN TESTSHELL (seconds):",
1019             times_for_dump_render_tree)
1020         self._print_statistics_for_test_timings(
1021             "PER TEST DIFF PROCESSING TIMES (seconds):",
1022             times_for_diff_processing)
1023         for test_type in test_types:
1024             self._print_statistics_for_test_timings(
1025                 "PER TEST TIMES BY TEST TYPE: %s" % test_type,
1026                 times_per_test_type[test_type])
1027
1028     def _print_individual_test_times(self, individual_test_timings,
1029                                   result_summary):
1030         """Prints the run times for slow, timeout and crash tests.
1031         Args:
1032           individual_test_timings: List of dump_render_tree_thread.TestStats
1033               for all tests.
1034           result_summary: summary object for test run
1035         """
1036         # Reverse-sort by the time spent in DumpRenderTree.
1037         individual_test_timings.sort(lambda a, b:
1038             cmp(b.test_run_time, a.test_run_time))
1039
1040         num_printed = 0
1041         slow_tests = []
1042         timeout_or_crash_tests = []
1043         unexpected_slow_tests = []
1044         for test_tuple in individual_test_timings:
1045             filename = test_tuple.filename
1046             is_timeout_crash_or_slow = False
1047             if self._expectations.has_modifier(filename,
1048                                                test_expectations.SLOW):
1049                 is_timeout_crash_or_slow = True
1050                 slow_tests.append(test_tuple)
1051
1052             if filename in result_summary.failures:
1053                 result = result_summary.results[filename].type
1054                 if (result == test_expectations.TIMEOUT or
1055                     result == test_expectations.CRASH):
1056                     is_timeout_crash_or_slow = True
1057                     timeout_or_crash_tests.append(test_tuple)
1058
1059             if (not is_timeout_crash_or_slow and
1060                 num_printed < printing.NUM_SLOW_TESTS_TO_LOG):
1061                 num_printed = num_printed + 1
1062                 unexpected_slow_tests.append(test_tuple)
1063
1064         self._printer.print_timing("")
1065         self._print_test_list_timing("%s slowest tests that are not "
1066             "marked as SLOW and did not timeout/crash:" %
1067             printing.NUM_SLOW_TESTS_TO_LOG, unexpected_slow_tests)
1068         self._printer.print_timing("")
1069         self._print_test_list_timing("Tests marked as SLOW:", slow_tests)
1070         self._printer.print_timing("")
1071         self._print_test_list_timing("Tests that timed out or crashed:",
1072                                      timeout_or_crash_tests)
1073         self._printer.print_timing("")
1074
1075     def _print_test_list_timing(self, title, test_list):
1076         """Print timing info for each test.
1077
1078         Args:
1079           title: section heading
1080           test_list: tests that fall in this section
1081         """
1082         if self._printer.disabled('slowest'):
1083             return
1084
1085         self._printer.print_timing(title)
1086         for test_tuple in test_list:
1087             filename = test_tuple.filename[len(
1088                 self._port.layout_tests_dir()) + 1:]
1089             filename = filename.replace('\\', '/')
1090             test_run_time = round(test_tuple.test_run_time, 1)
1091             self._printer.print_timing("  %s took %s seconds" %
1092                                        (filename, test_run_time))
1093
1094     def _print_directory_timings(self, directory_test_timings):
1095         """Print timing info by directory for any directories that
1096         take > 10 seconds to run.
1097
1098         Args:
1099           directory_test_timing: time info for each directory
1100         """
1101         timings = []
1102         for directory in directory_test_timings:
1103             num_tests, time_for_directory = directory_test_timings[directory]
1104             timings.append((round(time_for_directory, 1), directory,
1105                             num_tests))
1106         timings.sort()
1107
1108         self._printer.print_timing("Time to process slowest subdirectories:")
1109         min_seconds_to_print = 10
1110         for timing in timings:
1111             if timing[0] > min_seconds_to_print:
1112                 self._printer.print_timing(
1113                     "  %s took %s seconds to run %s tests." % (timing[1],
1114                     timing[0], timing[2]))
1115         self._printer.print_timing("")
1116
1117     def _print_statistics_for_test_timings(self, title, timings):
1118         """Prints the median, mean and standard deviation of the values in
1119         timings.
1120
1121         Args:
1122           title: Title for these timings.
1123           timings: A list of floats representing times.
1124         """
1125         self._printer.print_timing(title)
1126         timings.sort()
1127
1128         num_tests = len(timings)
1129         if not num_tests:
1130             return
1131         percentile90 = timings[int(.9 * num_tests)]
1132         percentile99 = timings[int(.99 * num_tests)]
1133
1134         if num_tests % 2 == 1:
1135             median = timings[((num_tests - 1) / 2) - 1]
1136         else:
1137             lower = timings[num_tests / 2 - 1]
1138             upper = timings[num_tests / 2]
1139             median = (float(lower + upper)) / 2
1140
1141         mean = sum(timings) / num_tests
1142
1143         for time in timings:
1144             sum_of_deviations = math.pow(time - mean, 2)
1145
1146         std_deviation = math.sqrt(sum_of_deviations / num_tests)
1147         self._printer.print_timing("  Median:          %6.3f" % median)
1148         self._printer.print_timing("  Mean:            %6.3f" % mean)
1149         self._printer.print_timing("  90th percentile: %6.3f" % percentile90)
1150         self._printer.print_timing("  99th percentile: %6.3f" % percentile99)
1151         self._printer.print_timing("  Standard dev:    %6.3f" % std_deviation)
1152         self._printer.print_timing("")
1153
1154     def _print_result_summary(self, result_summary):
1155         """Print a short summary about how many tests passed.
1156
1157         Args:
1158           result_summary: information to log
1159         """
1160         failed = len(result_summary.failures)
1161         skipped = len(
1162             result_summary.tests_by_expectation[test_expectations.SKIP])
1163         total = result_summary.total
1164         passed = total - failed - skipped
1165         pct_passed = 0.0
1166         if total > 0:
1167             pct_passed = float(passed) * 100 / total
1168
1169         self._printer.print_actual("")
1170         self._printer.print_actual("=> Results: %d/%d tests passed (%.1f%%)" %
1171                      (passed, total, pct_passed))
1172         self._printer.print_actual("")
1173         self._print_result_summary_entry(result_summary,
1174             test_expectations.NOW, "Tests to be fixed")
1175
1176         self._printer.print_actual("")
1177         self._print_result_summary_entry(result_summary,
1178             test_expectations.WONTFIX,
1179             "Tests that will only be fixed if they crash (WONTFIX)")
1180         self._printer.print_actual("")
1181
1182     def _print_result_summary_entry(self, result_summary, timeline,
1183                                     heading):
1184         """Print a summary block of results for a particular timeline of test.
1185
1186         Args:
1187           result_summary: summary to print results for
1188           timeline: the timeline to print results for (NOT, WONTFIX, etc.)
1189           heading: a textual description of the timeline
1190         """
1191         total = len(result_summary.tests_by_timeline[timeline])
1192         not_passing = (total -
1193            len(result_summary.tests_by_expectation[test_expectations.PASS] &
1194                result_summary.tests_by_timeline[timeline]))
1195         self._printer.print_actual("=> %s (%d):" % (heading, not_passing))
1196
1197         for result in TestExpectationsFile.EXPECTATION_ORDER:
1198             if result == test_expectations.PASS:
1199                 continue
1200             results = (result_summary.tests_by_expectation[result] &
1201                        result_summary.tests_by_timeline[timeline])
1202             desc = TestExpectationsFile.EXPECTATION_DESCRIPTIONS[result]
1203             if not_passing and len(results):
1204                 pct = len(results) * 100.0 / not_passing
1205                 self._printer.print_actual("  %5d %-24s (%4.1f%%)" %
1206                     (len(results), desc[len(results) != 1], pct))
1207
1208     def _results_html(self, test_files, failures, title="Test Failures", override_time=None):
1209         """
1210         test_files = a list of file paths
1211         failures = dictionary mapping test paths to failure objects
1212         title = title printed at top of test
1213         override_time = current time (used by unit tests)
1214         """
1215         page = """<html>
1216   <head>
1217     <title>Layout Test Results (%(time)s)</title>
1218   </head>
1219   <body>
1220     <h2>%(title)s (%(time)s)</h2>
1221         """ % {'title': title, 'time': override_time or time.asctime()}
1222
1223         for test_file in sorted(test_files):
1224             test_name = self._port.relative_test_filename(test_file)
1225             test_url = self._port.filename_to_uri(test_file)
1226             page += u"<p><a href='%s'>%s</a><br />\n" % (test_url, test_name)
1227             test_failures = failures.get(test_file, [])
1228             for failure in test_failures:
1229                 page += (u"&nbsp;&nbsp;%s<br/>" %
1230                          failure.result_html_output(test_name))
1231             page += "</p>\n"
1232         page += "</body></html>\n"
1233         return page
1234
1235     def _write_results_html_file(self, result_summary):
1236         """Write results.html which is a summary of tests that failed.
1237
1238         Args:
1239           result_summary: a summary of the results :)
1240
1241         Returns:
1242           True if any results were written (since expected failures may be
1243           omitted)
1244         """
1245         # test failures
1246         if self._options.full_results_html:
1247             results_title = "Test Failures"
1248             test_files = result_summary.failures.keys()
1249         else:
1250             results_title = "Unexpected Test Failures"
1251             unexpected_failures = self._get_failures(result_summary,
1252                 include_crashes=True)
1253             test_files = unexpected_failures.keys()
1254         if not len(test_files):
1255             return False
1256
1257         out_filename = os.path.join(self._options.results_directory,
1258                                     "results.html")
1259         with codecs.open(out_filename, "w", "utf-8") as results_file:
1260             html = self._results_html(test_files, result_summary.failures, results_title)
1261             results_file.write(html)
1262
1263         return True
1264
1265     def _show_results_html_file(self):
1266         """Shows the results.html page."""
1267         results_filename = os.path.join(self._options.results_directory,
1268                                         "results.html")
1269         self._port.show_results_html_file(results_filename)
1270
1271
1272 def read_test_files(files):
1273     tests = []
1274     for file in files:
1275         try:
1276             with codecs.open(file, 'r', 'utf-8') as file_contents:
1277                 # FIXME: This could be cleaner using a list comprehension.
1278                 for line in file_contents:
1279                     line = test_expectations.strip_comments(line)
1280                     if line:
1281                         tests.append(line)
1282         except IOError, e:
1283             if e.errno == errno.ENOENT:
1284                 _log.critical('')
1285                 _log.critical('--test-list file "%s" not found' % file)
1286             raise
1287     return tests
1288
1289
1290 def run(port, options, args, regular_output=sys.stderr,
1291         buildbot_output=sys.stdout):
1292     """Run the tests.
1293
1294     Args:
1295       port: Port object for port-specific behavior
1296       options: a dictionary of command line options
1297       args: a list of sub directories or files to test
1298       regular_output: a stream-like object that we can send logging/debug
1299           output to
1300       buildbot_output: a stream-like object that we can write all output that
1301           is intended to be parsed by the buildbot to
1302     Returns:
1303       the number of unexpected results that occurred, or -1 if there is an
1304           error.
1305
1306     """
1307     _set_up_derived_options(port, options)
1308
1309     printer = printing.Printer(port, options, regular_output, buildbot_output,
1310         int(options.child_processes), options.experimental_fully_parallel)
1311     if options.help_printing:
1312         printer.help_printing()
1313         printer.cleanup()
1314         return 0
1315
1316     last_unexpected_results = _gather_unexpected_results(options)
1317     if options.print_last_failures:
1318         printer.write("\n".join(last_unexpected_results) + "\n")
1319         printer.cleanup()
1320         return 0
1321
1322     # We wrap any parts of the run that are slow or likely to raise exceptions
1323     # in a try/finally to ensure that we clean up the logging configuration.
1324     num_unexpected_results = -1
1325     try:
1326         test_runner = TestRunner(port, options, printer)
1327         test_runner._print_config()
1328
1329         printer.print_update("Collecting tests ...")
1330         try:
1331             test_runner.collect_tests(args, last_unexpected_results)
1332         except IOError, e:
1333             if e.errno == errno.ENOENT:
1334                 return -1
1335             raise
1336
1337         printer.print_update("Parsing expectations ...")
1338         if options.lint_test_files:
1339             return test_runner.lint()
1340         test_runner.parse_expectations(port.test_platform_name(),
1341                                        options.configuration == 'Debug')
1342
1343         printer.print_update("Checking build ...")
1344         if not port.check_build(test_runner.needs_http()):
1345             _log.error("Build check failed")
1346             return -1
1347
1348         result_summary = test_runner.set_up_run()
1349         if result_summary:
1350             num_unexpected_results = test_runner.run(result_summary)
1351             test_runner.clean_up_run()
1352             _log.debug("Testing completed, Exit status: %d" %
1353                        num_unexpected_results)
1354     finally:
1355         printer.cleanup()
1356
1357     return num_unexpected_results
1358
1359
1360 def _set_up_derived_options(port_obj, options):
1361     """Sets the options values that depend on other options values."""
1362
1363     if not options.child_processes:
1364         # FIXME: Investigate perf/flakiness impact of using cpu_count + 1.
1365         options.child_processes = os.environ.get("WEBKIT_TEST_CHILD_PROCESSES",
1366                                                  str(port_obj.default_child_processes()))
1367
1368     if not options.configuration:
1369         options.configuration = port_obj.default_configuration()
1370
1371     if options.pixel_tests is None:
1372         options.pixel_tests = True
1373
1374     if not options.use_apache:
1375         options.use_apache = sys.platform in ('darwin', 'linux2')
1376
1377     if not os.path.isabs(options.results_directory):
1378         # This normalizes the path to the build dir.
1379         # FIXME: how this happens is not at all obvious; this is a dumb
1380         # interface and should be cleaned up.
1381         options.results_directory = port_obj.results_directory()
1382
1383     if not options.time_out_ms:
1384         if options.configuration == "Debug":
1385             options.time_out_ms = str(2 * TestRunner.DEFAULT_TEST_TIMEOUT_MS)
1386         else:
1387             options.time_out_ms = str(TestRunner.DEFAULT_TEST_TIMEOUT_MS)
1388
1389     options.slow_time_out_ms = str(5 * int(options.time_out_ms))
1390
1391
1392 def _gather_unexpected_results(options):
1393     """Returns the unexpected results from the previous run, if any."""
1394     last_unexpected_results = []
1395     if options.print_last_failures or options.retest_last_failures:
1396         unexpected_results_filename = os.path.join(
1397         options.results_directory, "unexpected_results.json")
1398         with codecs.open(unexpected_results_filename, "r", "utf-8") as file:
1399             results = simplejson.load(file)
1400         last_unexpected_results = results['tests'].keys()
1401     return last_unexpected_results
1402
1403
1404 def _compat_shim_callback(option, opt_str, value, parser):
1405     print "Ignoring unsupported option: %s" % opt_str
1406
1407
1408 def _compat_shim_option(option_name, **kwargs):
1409     return optparse.make_option(option_name, action="callback",
1410         callback=_compat_shim_callback,
1411         help="Ignored, for old-run-webkit-tests compat only.", **kwargs)
1412
1413
1414 def parse_args(args=None):
1415     """Provides a default set of command line args.
1416
1417     Returns a tuple of options, args from optparse"""
1418
1419     # FIXME: All of these options should be stored closer to the code which
1420     # FIXME: actually uses them. configuration_options should move
1421     # FIXME: to WebKitPort and be shared across all scripts.
1422     configuration_options = [
1423         optparse.make_option("-t", "--target", dest="configuration",
1424                              help="(DEPRECATED)"),
1425         # FIXME: --help should display which configuration is default.
1426         optparse.make_option('--debug', action='store_const', const='Debug',
1427                              dest="configuration",
1428                              help='Set the configuration to Debug'),
1429         optparse.make_option('--release', action='store_const',
1430                              const='Release', dest="configuration",
1431                              help='Set the configuration to Release'),
1432         # old-run-webkit-tests also accepts -c, --configuration CONFIGURATION.
1433     ]
1434
1435     print_options = printing.print_options()
1436
1437     # FIXME: These options should move onto the ChromiumPort.
1438     chromium_options = [
1439         optparse.make_option("--chromium", action="store_true", default=False,
1440             help="use the Chromium port"),
1441         optparse.make_option("--startup-dialog", action="store_true",
1442             default=False, help="create a dialog on DumpRenderTree startup"),
1443         optparse.make_option("--gp-fault-error-box", action="store_true",
1444             default=False, help="enable Windows GP fault error box"),
1445         optparse.make_option("--multiple-loads",
1446             type="int", help="turn on multiple loads of each test"),
1447         optparse.make_option("--js-flags",
1448             type="string", help="JavaScript flags to pass to tests"),
1449         optparse.make_option("--nocheck-sys-deps", action="store_true",
1450             default=False,
1451             help="Don't check the system dependencies (themes)"),
1452         optparse.make_option("--use-drt", action="store_true",
1453             default=None,
1454             help="Use DumpRenderTree instead of test_shell"),
1455         optparse.make_option("--accelerated-compositing",
1456             action="store_true",
1457             help="Use hardware-accelated compositing for rendering"),
1458         optparse.make_option("--no-accelerated-compositing",
1459             action="store_false",
1460             dest="accelerated_compositing",
1461             help="Don't use hardware-accelerated compositing for rendering"),
1462         optparse.make_option("--accelerated-2d-canvas",
1463             action="store_true",
1464             help="Use hardware-accelerated 2D Canvas calls"),
1465         optparse.make_option("--no-accelerated-2d-canvas",
1466             action="store_false",
1467             dest="accelerated_2d_canvas",
1468             help="Don't use hardware-accelerated 2D Canvas calls"),
1469     ]
1470
1471     # Missing Mac-specific old-run-webkit-tests options:
1472     # FIXME: Need: -g, --guard for guard malloc support on Mac.
1473     # FIXME: Need: -l --leaks    Enable leaks checking.
1474     # FIXME: Need: --sample-on-timeout Run sample on timeout
1475
1476     old_run_webkit_tests_compat = [
1477         # NRWT doesn't generate results by default anyway.
1478         _compat_shim_option("--no-new-test-results"),
1479         # NRWT doesn't sample on timeout yet anyway.
1480         _compat_shim_option("--no-sample-on-timeout"),
1481         # FIXME: NRWT needs to support remote links eventually.
1482         _compat_shim_option("--use-remote-links-to-tests"),
1483         # FIXME: NRWT doesn't need this option as much since failures are
1484         # designed to be cheap.  We eventually plan to add this support.
1485         _compat_shim_option("--exit-after-n-failures", nargs=1, type="int"),
1486     ]
1487
1488     results_options = [
1489         # NEED for bots: --use-remote-links-to-tests Link to test files
1490         # within the SVN repository in the results.
1491         optparse.make_option("-p", "--pixel-tests", action="store_true",
1492             dest="pixel_tests", help="Enable pixel-to-pixel PNG comparisons"),
1493         optparse.make_option("--no-pixel-tests", action="store_false",
1494             dest="pixel_tests", help="Disable pixel-to-pixel PNG comparisons"),
1495         optparse.make_option("--tolerance",
1496             help="Ignore image differences less than this percentage (some "
1497                 "ports may ignore this option)", type="float"),
1498         optparse.make_option("--results-directory",
1499             default="layout-test-results",
1500             help="Output results directory source dir, relative to Debug or "
1501                  "Release"),
1502         optparse.make_option("--new-baseline", action="store_true",
1503             default=False, help="Save all generated results as new baselines "
1504                  "into the platform directory, overwriting whatever's "
1505                  "already there."),
1506         optparse.make_option("--reset-results", action="store_true",
1507             default=False, help="Reset any existing baselines to the "
1508                  "generated results"),
1509         optparse.make_option("--no-show-results", action="store_false",
1510             default=True, dest="show_results",
1511             help="Don't launch a browser with results after the tests "
1512                  "are done"),
1513         # FIXME: We should have a helper function to do this sort of
1514         # deprectated mapping and automatically log, etc.
1515         optparse.make_option("--noshow-results", action="store_false",
1516             dest="show_results",
1517             help="Deprecated, same as --no-show-results."),
1518         optparse.make_option("--no-launch-safari", action="store_false",
1519             dest="show_results",
1520             help="old-run-webkit-tests compat, same as --noshow-results."),
1521         # old-run-webkit-tests:
1522         # --[no-]launch-safari    Launch (or do not launch) Safari to display
1523         #                         test results (default: launch)
1524         optparse.make_option("--full-results-html", action="store_true",
1525             default=False,
1526             help="Show all failures in results.html, rather than only "
1527                  "regressions"),
1528         optparse.make_option("--clobber-old-results", action="store_true",
1529             default=False, help="Clobbers test results from previous runs."),
1530         optparse.make_option("--platform",
1531             help="Override the platform for expected results"),
1532         optparse.make_option("--no-record-results", action="store_false",
1533             default=True, dest="record_results",
1534             help="Don't record the results."),
1535         # old-run-webkit-tests also has HTTP toggle options:
1536         # --[no-]http                     Run (or do not run) http tests
1537         #                                 (default: run)
1538     ]
1539
1540     test_options = [
1541         optparse.make_option("--build", dest="build",
1542             action="store_true", default=True,
1543             help="Check to ensure the DumpRenderTree build is up-to-date "
1544                  "(default)."),
1545         optparse.make_option("--no-build", dest="build",
1546             action="store_false", help="Don't check to see if the "
1547                                        "DumpRenderTree build is up-to-date."),
1548         # old-run-webkit-tests has --valgrind instead of wrapper.
1549         optparse.make_option("--wrapper",
1550             help="wrapper command to insert before invocations of "
1551                  "DumpRenderTree; option is split on whitespace before "
1552                  "running. (Example: --wrapper='valgrind --smc-check=all')"),
1553         # old-run-webkit-tests:
1554         # -i|--ignore-tests               Comma-separated list of directories
1555         #                                 or tests to ignore
1556         optparse.make_option("--test-list", action="append",
1557             help="read list of tests to run from file", metavar="FILE"),
1558         # old-run-webkit-tests uses --skipped==[default|ignore|only]
1559         # instead of --force:
1560         optparse.make_option("--force", action="store_true", default=False,
1561             help="Run all tests, even those marked SKIP in the test list"),
1562         optparse.make_option("--use-apache", action="store_true",
1563             default=False, help="Whether to use apache instead of lighttpd."),
1564         optparse.make_option("--time-out-ms",
1565             help="Set the timeout for each test"),
1566         # old-run-webkit-tests calls --randomize-order --random:
1567         optparse.make_option("--randomize-order", action="store_true",
1568             default=False, help=("Run tests in random order (useful "
1569                                 "for tracking down corruption)")),
1570         optparse.make_option("--run-chunk",
1571             help=("Run a specified chunk (n:l), the nth of len l, "
1572                  "of the layout tests")),
1573         optparse.make_option("--run-part", help=("Run a specified part (n:m), "
1574                   "the nth of m parts, of the layout tests")),
1575         # old-run-webkit-tests calls --batch-size: --nthly n
1576         #   Restart DumpRenderTree every n tests (default: 1000)
1577         optparse.make_option("--batch-size",
1578             help=("Run a the tests in batches (n), after every n tests, "
1579                   "DumpRenderTree is relaunched."), type="int", default=0),
1580         # old-run-webkit-tests calls --run-singly: -1|--singly
1581         # Isolate each test case run (implies --nthly 1 --verbose)
1582         optparse.make_option("--run-singly", action="store_true",
1583             default=False, help="run a separate DumpRenderTree for each test"),
1584         optparse.make_option("--child-processes",
1585             help="Number of DumpRenderTrees to run in parallel."),
1586         # FIXME: Display default number of child processes that will run.
1587         optparse.make_option("--experimental-fully-parallel",
1588             action="store_true", default=False,
1589             help="run all tests in parallel"),
1590         # FIXME: Need --exit-after-n-failures N
1591         #      Exit after the first N failures instead of running all tests
1592         # FIXME: Need --exit-after-n-crashes N
1593         #      Exit after the first N crashes instead of running all tests
1594         # FIXME: consider: --iterations n
1595         #      Number of times to run the set of tests (e.g. ABCABCABC)
1596         optparse.make_option("--print-last-failures", action="store_true",
1597             default=False, help="Print the tests in the last run that "
1598             "had unexpected failures (or passes)."),
1599         optparse.make_option("--retest-last-failures", action="store_true",
1600             default=False, help="re-test the tests in the last run that "
1601             "had unexpected failures (or passes)."),
1602         optparse.make_option("--retry-failures", action="store_true",
1603             default=True,
1604             help="Re-try any tests that produce unexpected results (default)"),
1605         optparse.make_option("--no-retry-failures", action="store_false",
1606             dest="retry_failures",
1607             help="Don't re-try any tests that produce unexpected results."),
1608     ]
1609
1610     misc_options = [
1611         optparse.make_option("--lint-test-files", action="store_true",
1612         default=False, help=("Makes sure the test files parse for all "
1613                             "configurations. Does not run any tests.")),
1614     ]
1615
1616     # FIXME: Move these into json_results_generator.py
1617     results_json_options = [
1618         optparse.make_option("--master-name", help="The name of the buildbot master."),
1619         optparse.make_option("--builder-name", default="DUMMY_BUILDER_NAME",
1620             help=("The name of the builder shown on the waterfall running "
1621                   "this script e.g. WebKit.")),
1622         optparse.make_option("--build-name", default="DUMMY_BUILD_NAME",
1623             help=("The name of the builder used in its path, e.g. "
1624                   "webkit-rel.")),
1625         optparse.make_option("--build-number", default="DUMMY_BUILD_NUMBER",
1626             help=("The build number of the builder running this script.")),
1627         optparse.make_option("--test-results-server", default="",
1628             help=("If specified, upload results json files to this appengine "
1629                   "server.")),
1630         optparse.make_option("--upload-full-results",
1631             action="store_true",
1632             default=False,
1633             help="If true, upload full json results to server."),
1634     ]
1635
1636     option_list = (configuration_options + print_options +
1637                    chromium_options + results_options + test_options +
1638                    misc_options + results_json_options +
1639                    old_run_webkit_tests_compat)
1640     option_parser = optparse.OptionParser(option_list=option_list)
1641
1642     options, args = option_parser.parse_args(args)
1643
1644     return options, args
1645
1646
1647
1648 def main():
1649     options, args = parse_args()
1650     port_obj = port.get(options.platform, options)
1651     return run(port_obj, options, args)
1652
1653 if '__main__' == __name__:
1654     try:
1655         sys.exit(main())
1656     except KeyboardInterrupt:
1657         # this mirrors what the shell normally does
1658         sys.exit(signal.SIGINT + 128)