2010-12-01 Dirk Pranke <dpranke@chromium.org>
[WebKit.git] / WebKitTools / Scripts / webkitpy / layout_tests / run_webkit_tests.py
1 #!/usr/bin/env python
2 # Copyright (C) 2010 Google Inc. All rights reserved.
3 # Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Szeged
4 #
5 # Redistribution and use in source and binary forms, with or without
6 # modification, are permitted provided that the following conditions are
7 # met:
8 #
9 #     * Redistributions of source code must retain the above copyright
10 # notice, this list of conditions and the following disclaimer.
11 #     * Redistributions in binary form must reproduce the above
12 # copyright notice, this list of conditions and the following disclaimer
13 # in the documentation and/or other materials provided with the
14 # distribution.
15 #     * Neither the name of Google Inc. nor the names of its
16 # contributors may be used to endorse or promote products derived from
17 # this software without specific prior written permission.
18 #
19 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30
31 """Run layout tests.
32
33 This is a port of the existing webkit test script run-webkit-tests.
34
35 The TestRunner class runs a series of tests (TestType interface) against a set
36 of test files.  If a test file fails a TestType, it returns a list TestFailure
37 objects to the TestRunner.  The TestRunner then aggregates the TestFailures to
38 create a final report.
39
40 This script reads several files, if they exist in the test_lists subdirectory
41 next to this script itself.  Each should contain a list of paths to individual
42 tests or entire subdirectories of tests, relative to the outermost test
43 directory.  Entire lines starting with '//' (comments) will be ignored.
44
45 For details of the files' contents and purposes, see test_lists/README.
46 """
47
48 from __future__ import with_statement
49
50 import codecs
51 import errno
52 import glob
53 import logging
54 import math
55 import optparse
56 import os
57 import platform
58 import Queue
59 import random
60 import re
61 import shutil
62 import signal
63 import sys
64 import time
65 import traceback
66
67 from layout_package import dump_render_tree_thread
68 from layout_package import json_layout_results_generator
69 from layout_package import message_broker
70 from layout_package import printing
71 from layout_package import test_expectations
72 from layout_package import test_failures
73 from layout_package import test_results
74 from layout_package import test_results_uploader
75
76 from webkitpy.common.system import user
77 from webkitpy.thirdparty import simplejson
78 from webkitpy.tool import grammar
79
80 import port
81
82 _log = logging.getLogger("webkitpy.layout_tests.run_webkit_tests")
83
84 # Builder base URL where we have the archived test results.
85 BUILDER_BASE_URL = "http://build.chromium.org/buildbot/layout_test_results/"
86
87 LAYOUT_TESTS_DIRECTORY = "LayoutTests" + os.sep
88
89 TestExpectationsFile = test_expectations.TestExpectationsFile
90
91
92 class TestInput:
93     """Groups information about a test for easy passing of data."""
94
95     def __init__(self, filename, timeout):
96         """Holds the input parameters for a test.
97         Args:
98           filename: Full path to the test.
99           timeout: Timeout in msecs the driver should use while running the test
100           """
101         # FIXME: filename should really be test_name as a relative path.
102         self.filename = filename
103         self.timeout = timeout
104         # The image_hash is used to avoid doing an image dump if the
105         # checksums match. The image_hash is set later, and only if it is needed
106         # for the test.
107         self.image_hash = None
108
109
110 class ResultSummary(object):
111     """A class for partitioning the test results we get into buckets.
112
113     This class is basically a glorified struct and it's private to this file
114     so we don't bother with any information hiding."""
115
116     def __init__(self, expectations, test_files):
117         self.total = len(test_files)
118         self.remaining = self.total
119         self.expectations = expectations
120         self.expected = 0
121         self.unexpected = 0
122         self.tests_by_expectation = {}
123         self.tests_by_timeline = {}
124         self.results = {}
125         self.unexpected_results = {}
126         self.failures = {}
127         self.tests_by_expectation[test_expectations.SKIP] = set()
128         for expectation in TestExpectationsFile.EXPECTATIONS.values():
129             self.tests_by_expectation[expectation] = set()
130         for timeline in TestExpectationsFile.TIMELINES.values():
131             self.tests_by_timeline[timeline] = (
132                 expectations.get_tests_with_timeline(timeline))
133
134     def add(self, result, expected):
135         """Add a TestResult into the appropriate bin.
136
137         Args:
138           result: TestResult from dump_render_tree_thread.
139           expected: whether the result was what we expected it to be.
140         """
141
142         self.tests_by_expectation[result.type].add(result.filename)
143         self.results[result.filename] = result
144         self.remaining -= 1
145         if len(result.failures):
146             self.failures[result.filename] = result.failures
147         if expected:
148             self.expected += 1
149         else:
150             self.unexpected_results[result.filename] = result.type
151             self.unexpected += 1
152
153
154 def summarize_unexpected_results(port_obj, expectations, result_summary,
155                                  retry_summary):
156     """Summarize any unexpected results as a dict.
157
158     FIXME: split this data structure into a separate class?
159
160     Args:
161         port_obj: interface to port-specific hooks
162         expectations: test_expectations.TestExpectations object
163         result_summary: summary object from initial test runs
164         retry_summary: summary object from final test run of retried tests
165     Returns:
166         A dictionary containing a summary of the unexpected results from the
167         run, with the following fields:
168         'version': a version indicator (1 in this version)
169         'fixable': # of fixable tests (NOW - PASS)
170         'skipped': # of skipped tests (NOW & SKIPPED)
171         'num_regressions': # of non-flaky failures
172         'num_flaky': # of flaky failures
173         'num_passes': # of unexpected passes
174         'tests': a dict of tests -> {'expected': '...', 'actual': '...'}
175     """
176     results = {}
177     results['version'] = 1
178
179     tbe = result_summary.tests_by_expectation
180     tbt = result_summary.tests_by_timeline
181     results['fixable'] = len(tbt[test_expectations.NOW] -
182                                 tbe[test_expectations.PASS])
183     results['skipped'] = len(tbt[test_expectations.NOW] &
184                                 tbe[test_expectations.SKIP])
185
186     num_passes = 0
187     num_flaky = 0
188     num_regressions = 0
189     keywords = {}
190     for k, v in TestExpectationsFile.EXPECTATIONS.iteritems():
191         keywords[v] = k.upper()
192
193     tests = {}
194     for filename, result in result_summary.unexpected_results.iteritems():
195         # Note that if a test crashed in the original run, we ignore
196         # whether or not it crashed when we retried it (if we retried it),
197         # and always consider the result not flaky.
198         test = port_obj.relative_test_filename(filename)
199         expected = expectations.get_expectations_string(filename)
200         actual = [keywords[result]]
201
202         if result == test_expectations.PASS:
203             num_passes += 1
204         elif result == test_expectations.CRASH:
205             num_regressions += 1
206         else:
207             if filename not in retry_summary.unexpected_results:
208                 actual.extend(expectations.get_expectations_string(
209                     filename).split(" "))
210                 num_flaky += 1
211             else:
212                 retry_result = retry_summary.unexpected_results[filename]
213                 if result != retry_result:
214                     actual.append(keywords[retry_result])
215                     num_flaky += 1
216                 else:
217                     num_regressions += 1
218
219         tests[test] = {}
220         tests[test]['expected'] = expected
221         tests[test]['actual'] = " ".join(actual)
222
223     results['tests'] = tests
224     results['num_passes'] = num_passes
225     results['num_flaky'] = num_flaky
226     results['num_regressions'] = num_regressions
227
228     return results
229
230
231 class TestRunner:
232     """A class for managing running a series of tests on a series of layout
233     test files."""
234
235     HTTP_SUBDIR = os.sep.join(['', 'http', ''])
236     WEBSOCKET_SUBDIR = os.sep.join(['', 'websocket', ''])
237
238     # The per-test timeout in milliseconds, if no --time-out-ms option was
239     # given to run_webkit_tests. This should correspond to the default timeout
240     # in DumpRenderTree.
241     DEFAULT_TEST_TIMEOUT_MS = 6 * 1000
242
243     def __init__(self, port, options, printer, message_broker):
244         """Initialize test runner data structures.
245
246         Args:
247           port: an object implementing port-specific
248           options: a dictionary of command line options
249           printer: a Printer object to record updates to.
250           message_broker: object used to communicate with workers.
251         """
252         self._port = port
253         self._options = options
254         self._printer = printer
255         self._message_broker = message_broker
256
257         # disable wss server. need to install pyOpenSSL on buildbots.
258         # self._websocket_secure_server = websocket_server.PyWebSocket(
259         #        options.results_directory, use_tls=True, port=9323)
260
261         # a set of test files, and the same tests as a list
262         self._test_files = set()
263         self._test_files_list = None
264         self._result_queue = Queue.Queue()
265         self._retrying = False
266
267     def collect_tests(self, args, last_unexpected_results):
268         """Find all the files to test.
269
270         Args:
271           args: list of test arguments from the command line
272           last_unexpected_results: list of unexpected results to retest, if any
273
274         """
275         paths = [self._strip_test_dir_prefix(arg) for arg in args if arg and arg != '']
276         paths += last_unexpected_results
277         if self._options.test_list:
278             paths += read_test_files(self._options.test_list)
279         self._test_files = self._port.tests(paths)
280
281     def _strip_test_dir_prefix(self, path):
282         if path.startswith(LAYOUT_TESTS_DIRECTORY):
283             return path[len(LAYOUT_TESTS_DIRECTORY):]
284         return path
285
286     def lint(self):
287         # Creating the expecations for each platform/configuration pair does
288         # all the test list parsing and ensures it's correct syntax (e.g. no
289         # dupes).
290         for platform_name in self._port.test_platform_names():
291             self.parse_expectations(platform_name, is_debug_mode=True)
292             self.parse_expectations(platform_name, is_debug_mode=False)
293         self._printer.write("")
294         _log.info("If there are no fail messages, errors or exceptions, "
295                   "then the lint succeeded.")
296         return 0
297
298     def parse_expectations(self, test_platform_name, is_debug_mode):
299         """Parse the expectations from the test_list files and return a data
300         structure holding them. Throws an error if the test_list files have
301         invalid syntax."""
302         if self._options.lint_test_files:
303             test_files = None
304         else:
305             test_files = self._test_files
306
307         try:
308             expectations_str = self._port.test_expectations()
309             overrides_str = self._port.test_expectations_overrides()
310             self._expectations = test_expectations.TestExpectations(
311                 self._port, test_files, expectations_str, test_platform_name,
312                 is_debug_mode, self._options.lint_test_files,
313                 overrides=overrides_str)
314             return self._expectations
315         except SyntaxError, err:
316             if self._options.lint_test_files:
317                 print str(err)
318             else:
319                 raise err
320
321     def prepare_lists_and_print_output(self):
322         """Create appropriate subsets of test lists and returns a
323         ResultSummary object. Also prints expected test counts.
324         """
325
326         # Remove skipped - both fixable and ignored - files from the
327         # top-level list of files to test.
328         num_all_test_files = len(self._test_files)
329         self._printer.print_expected("Found:  %d tests" %
330                                      (len(self._test_files)))
331         if not num_all_test_files:
332             _log.critical('No tests to run.')
333             return None
334
335         skipped = set()
336         if num_all_test_files > 1 and not self._options.force:
337             skipped = self._expectations.get_tests_with_result_type(
338                            test_expectations.SKIP)
339             self._test_files -= skipped
340
341         # Create a sorted list of test files so the subset chunk,
342         # if used, contains alphabetically consecutive tests.
343         self._test_files_list = list(self._test_files)
344         if self._options.randomize_order:
345             random.shuffle(self._test_files_list)
346         else:
347             self._test_files_list.sort()
348
349         # If the user specifies they just want to run a subset of the tests,
350         # just grab a subset of the non-skipped tests.
351         if self._options.run_chunk or self._options.run_part:
352             chunk_value = self._options.run_chunk or self._options.run_part
353             test_files = self._test_files_list
354             try:
355                 (chunk_num, chunk_len) = chunk_value.split(":")
356                 chunk_num = int(chunk_num)
357                 assert(chunk_num >= 0)
358                 test_size = int(chunk_len)
359                 assert(test_size > 0)
360             except:
361                 _log.critical("invalid chunk '%s'" % chunk_value)
362                 return None
363
364             # Get the number of tests
365             num_tests = len(test_files)
366
367             # Get the start offset of the slice.
368             if self._options.run_chunk:
369                 chunk_len = test_size
370                 # In this case chunk_num can be really large. We need
371                 # to make the slave fit in the current number of tests.
372                 slice_start = (chunk_num * chunk_len) % num_tests
373             else:
374                 # Validate the data.
375                 assert(test_size <= num_tests)
376                 assert(chunk_num <= test_size)
377
378                 # To count the chunk_len, and make sure we don't skip
379                 # some tests, we round to the next value that fits exactly
380                 # all the parts.
381                 rounded_tests = num_tests
382                 if rounded_tests % test_size != 0:
383                     rounded_tests = (num_tests + test_size -
384                                      (num_tests % test_size))
385
386                 chunk_len = rounded_tests / test_size
387                 slice_start = chunk_len * (chunk_num - 1)
388                 # It does not mind if we go over test_size.
389
390             # Get the end offset of the slice.
391             slice_end = min(num_tests, slice_start + chunk_len)
392
393             files = test_files[slice_start:slice_end]
394
395             tests_run_msg = 'Running: %d tests (chunk slice [%d:%d] of %d)' % (
396                 (slice_end - slice_start), slice_start, slice_end, num_tests)
397             self._printer.print_expected(tests_run_msg)
398
399             # If we reached the end and we don't have enough tests, we run some
400             # from the beginning.
401             if slice_end - slice_start < chunk_len:
402                 extra = chunk_len - (slice_end - slice_start)
403                 extra_msg = ('   last chunk is partial, appending [0:%d]' %
404                             extra)
405                 self._printer.print_expected(extra_msg)
406                 tests_run_msg += "\n" + extra_msg
407                 files.extend(test_files[0:extra])
408             tests_run_filename = os.path.join(self._options.results_directory,
409                                               "tests_run.txt")
410             with codecs.open(tests_run_filename, "w", "utf-8") as file:
411                 file.write(tests_run_msg + "\n")
412
413             len_skip_chunk = int(len(files) * len(skipped) /
414                                  float(len(self._test_files)))
415             skip_chunk_list = list(skipped)[0:len_skip_chunk]
416             skip_chunk = set(skip_chunk_list)
417
418             # Update expectations so that the stats are calculated correctly.
419             # We need to pass a list that includes the right # of skipped files
420             # to ParseExpectations so that ResultSummary() will get the correct
421             # stats. So, we add in the subset of skipped files, and then
422             # subtract them back out.
423             self._test_files_list = files + skip_chunk_list
424             self._test_files = set(self._test_files_list)
425
426             self._expectations = self.parse_expectations(
427                 self._port.test_platform_name(),
428                 self._options.configuration == 'Debug')
429
430             self._test_files = set(files)
431             self._test_files_list = files
432         else:
433             skip_chunk = skipped
434
435         result_summary = ResultSummary(self._expectations,
436             self._test_files | skip_chunk)
437         self._print_expected_results_of_type(result_summary,
438             test_expectations.PASS, "passes")
439         self._print_expected_results_of_type(result_summary,
440             test_expectations.FAIL, "failures")
441         self._print_expected_results_of_type(result_summary,
442             test_expectations.FLAKY, "flaky")
443         self._print_expected_results_of_type(result_summary,
444             test_expectations.SKIP, "skipped")
445
446         if self._options.force:
447             self._printer.print_expected('Running all tests, including '
448                                          'skips (--force)')
449         else:
450             # Note that we don't actually run the skipped tests (they were
451             # subtracted out of self._test_files, above), but we stub out the
452             # results here so the statistics can remain accurate.
453             for test in skip_chunk:
454                 result = test_results.TestResult(test,
455                     failures=[], test_run_time=0, total_time_for_all_diffs=0,
456                     time_for_diffs=0)
457                 result.type = test_expectations.SKIP
458                 result_summary.add(result, expected=True)
459         self._printer.print_expected('')
460
461         return result_summary
462
463     def _get_dir_for_test_file(self, test_file):
464         """Returns the highest-level directory by which to shard the given
465         test file."""
466         index = test_file.rfind(os.sep + LAYOUT_TESTS_DIRECTORY)
467
468         test_file = test_file[index + len(LAYOUT_TESTS_DIRECTORY):]
469         test_file_parts = test_file.split(os.sep, 1)
470         directory = test_file_parts[0]
471         test_file = test_file_parts[1]
472
473         # The http tests are very stable on mac/linux.
474         # TODO(ojan): Make the http server on Windows be apache so we can
475         # turn shard the http tests there as well. Switching to apache is
476         # what made them stable on linux/mac.
477         return_value = directory
478         while ((directory != 'http' or sys.platform in ('darwin', 'linux2'))
479                 and test_file.find(os.sep) >= 0):
480             test_file_parts = test_file.split(os.sep, 1)
481             directory = test_file_parts[0]
482             return_value = os.path.join(return_value, directory)
483             test_file = test_file_parts[1]
484
485         return return_value
486
487     def _get_test_input_for_file(self, test_file):
488         """Returns the appropriate TestInput object for the file. Mostly this
489         is used for looking up the timeout value (in ms) to use for the given
490         test."""
491         if self._test_is_slow(test_file):
492             return TestInput(test_file, self._options.slow_time_out_ms)
493         return TestInput(test_file, self._options.time_out_ms)
494
495     def _test_requires_lock(self, test_file):
496         """Return True if the test needs to be locked when
497         running multiple copies of NRWTs."""
498         split_path = test_file.split(os.sep)
499         return 'http' in split_path or 'websocket' in split_path
500
501     def _test_is_slow(self, test_file):
502         return self._expectations.has_modifier(test_file,
503                                                test_expectations.SLOW)
504
505     def _shard_tests(self, test_files, use_real_shards):
506         """Groups tests into batches.
507         This helps ensure that tests that depend on each other (aka bad tests!)
508         continue to run together as most cross-tests dependencies tend to
509         occur within the same directory. If use_real_shards is false, we
510         put each (non-HTTP/websocket) test into its own shard for maximum
511         concurrency instead of trying to do any sort of real sharding.
512
513         Return:
514             A list of lists of TestInput objects.
515         """
516         # FIXME: when we added http locking, we changed how this works such
517         # that we always lump all of the HTTP threads into a single shard.
518         # That will slow down experimental-fully-parallel, but it's unclear
519         # what the best alternative is completely revamping how we track
520         # when to grab the lock.
521
522         test_lists = []
523         tests_to_http_lock = []
524         if not use_real_shards:
525             for test_file in test_files:
526                 test_input = self._get_test_input_for_file(test_file)
527                 if self._test_requires_lock(test_file):
528                     tests_to_http_lock.append(test_input)
529                 else:
530                     test_lists.append((".", [test_input]))
531         else:
532             tests_by_dir = {}
533             for test_file in test_files:
534                 directory = self._get_dir_for_test_file(test_file)
535                 test_input = self._get_test_input_for_file(test_file)
536                 if self._test_requires_lock(test_file):
537                     tests_to_http_lock.append(test_input)
538                 else:
539                     tests_by_dir.setdefault(directory, [])
540                     tests_by_dir[directory].append(test_input)
541             # Sort by the number of tests in the dir so that the ones with the
542             # most tests get run first in order to maximize parallelization.
543             # Number of tests is a good enough, but not perfect, approximation
544             # of how long that set of tests will take to run. We can't just use
545             # a PriorityQueue until we move to Python 2.6.
546             for directory in tests_by_dir:
547                 test_list = tests_by_dir[directory]
548                 # Keep the tests in alphabetical order.
549                 # FIXME: Remove once tests are fixed so they can be run in any
550                 # order.
551                 test_list.reverse()
552                 test_list_tuple = (directory, test_list)
553                 test_lists.append(test_list_tuple)
554             test_lists.sort(lambda a, b: cmp(len(b[1]), len(a[1])))
555
556         # Put the http tests first. There are only a couple hundred of them,
557         # but each http test takes a very long time to run, so sorting by the
558         # number of tests doesn't accurately capture how long they take to run.
559         if tests_to_http_lock:
560             tests_to_http_lock.reverse()
561             test_lists.insert(0, ("tests_to_http_lock", tests_to_http_lock))
562
563         return test_lists
564
565     def _contains_tests(self, subdir):
566         for test_file in self._test_files:
567             if test_file.find(subdir) >= 0:
568                 return True
569         return False
570
571     def _num_workers(self):
572         return int(self._options.child_processes)
573
574     def _run_tests(self, file_list, result_summary):
575         """Runs the tests in the file_list.
576
577         Return: A tuple (keyboard_interrupted, thread_timings, test_timings,
578             individual_test_timings)
579             keyboard_interrupted is whether someone typed Ctrl^C
580             thread_timings is a list of dicts with the total runtime
581               of each thread with 'name', 'num_tests', 'total_time' properties
582             test_timings is a list of timings for each sharded subdirectory
583               of the form [time, directory_name, num_tests]
584             individual_test_timings is a list of run times for each test
585               in the form {filename:filename, test_run_time:test_run_time}
586             result_summary: summary object to populate with the results
587         """
588
589         self._printer.print_update('Sharding tests ...')
590         num_workers = self._num_workers()
591         test_lists = self._shard_tests(file_list,
592             num_workers > 1 and not self._options.experimental_fully_parallel)
593         filename_queue = Queue.Queue()
594         for item in test_lists:
595             filename_queue.put(item)
596
597         self._printer.print_update('Starting %s ...' %
598                                    grammar.pluralize('worker', num_workers))
599         message_broker = self._message_broker
600         self._current_filename_queue = filename_queue
601         self._current_result_summary = result_summary
602
603         if not self._options.dry_run:
604             threads = message_broker.start_workers(self)
605         else:
606             threads = {}
607
608         self._printer.print_update("Starting testing ...")
609         keyboard_interrupted = False
610         if not self._options.dry_run:
611             try:
612                 message_broker.run_message_loop()
613             except KeyboardInterrupt:
614                 _log.info("Interrupted, exiting")
615                 message_broker.cancel_workers()
616                 keyboard_interrupted = True
617             except:
618                 # Unexpected exception; don't try to clean up workers.
619                 _log.info("Exception raised, exiting")
620                 raise
621
622         thread_timings, test_timings, individual_test_timings = \
623             self._collect_timing_info(threads)
624
625         return (keyboard_interrupted, thread_timings, test_timings,
626                 individual_test_timings)
627
628     def update(self):
629         self.update_summary(self._current_result_summary)
630
631     def _collect_timing_info(self, threads):
632         test_timings = {}
633         individual_test_timings = []
634         thread_timings = []
635
636         for thread in threads:
637             thread_timings.append({'name': thread.getName(),
638                                    'num_tests': thread.get_num_tests(),
639                                    'total_time': thread.get_total_time()})
640             test_timings.update(thread.get_test_group_timing_stats())
641             individual_test_timings.extend(thread.get_test_results())
642
643         return (thread_timings, test_timings, individual_test_timings)
644
645     def needs_http(self):
646         """Returns whether the test runner needs an HTTP server."""
647         return self._contains_tests(self.HTTP_SUBDIR)
648
649     def needs_websocket(self):
650         """Returns whether the test runner needs a WEBSOCKET server."""
651         return self._contains_tests(self.WEBSOCKET_SUBDIR)
652
653     def set_up_run(self):
654         """Configures the system to be ready to run tests.
655
656         Returns a ResultSummary object if we should continue to run tests,
657         or None if we should abort.
658
659         """
660         # This must be started before we check the system dependencies,
661         # since the helper may do things to make the setup correct.
662         self._printer.print_update("Starting helper ...")
663         self._port.start_helper()
664
665         # Check that the system dependencies (themes, fonts, ...) are correct.
666         if not self._options.nocheck_sys_deps:
667             self._printer.print_update("Checking system dependencies ...")
668             if not self._port.check_sys_deps(self.needs_http()):
669                 self._port.stop_helper()
670                 return None
671
672         if self._options.clobber_old_results:
673             self._clobber_old_results()
674
675         # Create the output directory if it doesn't already exist.
676         self._port.maybe_make_directory(self._options.results_directory)
677
678         self._port.setup_test_run()
679
680         self._printer.print_update("Preparing tests ...")
681         result_summary = self.prepare_lists_and_print_output()
682         if not result_summary:
683             return None
684
685         return result_summary
686
687     def run(self, result_summary):
688         """Run all our tests on all our test files.
689
690         For each test file, we run each test type. If there are any failures,
691         we collect them for reporting.
692
693         Args:
694           result_summary: a summary object tracking the test results.
695
696         Return:
697           The number of unexpected results (0 == success)
698         """
699         # gather_test_files() must have been called first to initialize us.
700         # If we didn't find any files to test, we've errored out already in
701         # prepare_lists_and_print_output().
702         assert(len(self._test_files))
703
704         start_time = time.time()
705
706         keyboard_interrupted, thread_timings, test_timings, \
707             individual_test_timings = (
708             self._run_tests(self._test_files_list, result_summary))
709
710         # We exclude the crashes from the list of results to retry, because
711         # we want to treat even a potentially flaky crash as an error.
712         failures = self._get_failures(result_summary, include_crashes=False)
713         retry_summary = result_summary
714         while (len(failures) and self._options.retry_failures and
715             not self._retrying and not keyboard_interrupted):
716             _log.info('')
717             _log.info("Retrying %d unexpected failure(s) ..." % len(failures))
718             _log.info('')
719             self._retrying = True
720             retry_summary = ResultSummary(self._expectations, failures.keys())
721             # Note that we intentionally ignore the return value here.
722             self._run_tests(failures.keys(), retry_summary)
723             failures = self._get_failures(retry_summary, include_crashes=True)
724
725         end_time = time.time()
726
727         self._print_timing_statistics(end_time - start_time,
728                                       thread_timings, test_timings,
729                                       individual_test_timings,
730                                       result_summary)
731
732         self._print_result_summary(result_summary)
733
734         sys.stdout.flush()
735         sys.stderr.flush()
736
737         self._printer.print_one_line_summary(result_summary.total,
738                                              result_summary.expected,
739                                              result_summary.unexpected)
740
741         unexpected_results = summarize_unexpected_results(self._port,
742             self._expectations, result_summary, retry_summary)
743         self._printer.print_unexpected_results(unexpected_results)
744
745         if (self._options.record_results and not self._options.dry_run and
746             not keyboard_interrupted):
747             # Write the same data to log files and upload generated JSON files
748             # to appengine server.
749             self._upload_json_files(unexpected_results, result_summary,
750                                     individual_test_timings)
751
752         # Write the summary to disk (results.html) and display it if requested.
753         if not self._options.dry_run:
754             wrote_results = self._write_results_html_file(result_summary)
755             if self._options.show_results and wrote_results:
756                 self._show_results_html_file()
757
758         # Now that we've completed all the processing we can, we re-raise
759         # a KeyboardInterrupt if necessary so the caller can handle it.
760         if keyboard_interrupted:
761             raise KeyboardInterrupt
762
763         # Ignore flaky failures and unexpected passes so we don't turn the
764         # bot red for those.
765         return unexpected_results['num_regressions']
766
767     def clean_up_run(self):
768         """Restores the system after we're done running tests."""
769
770         _log.debug("flushing stdout")
771         sys.stdout.flush()
772         _log.debug("flushing stderr")
773         sys.stderr.flush()
774         _log.debug("stopping helper")
775         self._port.stop_helper()
776
777     def update_summary(self, result_summary):
778         """Update the summary and print results with any completed tests."""
779         while True:
780             try:
781                 result = test_results.TestResult.loads(self._result_queue.get_nowait())
782             except Queue.Empty:
783                 return
784
785             expected = self._expectations.matches_an_expected_result(
786                 result.filename, result.type, self._options.pixel_tests)
787             result_summary.add(result, expected)
788             exp_str = self._expectations.get_expectations_string(
789                 result.filename)
790             got_str = self._expectations.expectation_to_string(result.type)
791             self._printer.print_test_result(result, expected, exp_str, got_str)
792             self._printer.print_progress(result_summary, self._retrying,
793                                          self._test_files_list)
794
795     def _clobber_old_results(self):
796         # Just clobber the actual test results directories since the other
797         # files in the results directory are explicitly used for cross-run
798         # tracking.
799         self._printer.print_update("Clobbering old results in %s" %
800                                    self._options.results_directory)
801         layout_tests_dir = self._port.layout_tests_dir()
802         possible_dirs = self._port.test_dirs()
803         for dirname in possible_dirs:
804             if os.path.isdir(os.path.join(layout_tests_dir, dirname)):
805                 shutil.rmtree(os.path.join(self._options.results_directory,
806                                            dirname),
807                               ignore_errors=True)
808
809     def _get_failures(self, result_summary, include_crashes):
810         """Filters a dict of results and returns only the failures.
811
812         Args:
813           result_summary: the results of the test run
814           include_crashes: whether crashes are included in the output.
815             We use False when finding the list of failures to retry
816             to see if the results were flaky. Although the crashes may also be
817             flaky, we treat them as if they aren't so that they're not ignored.
818         Returns:
819           a dict of files -> results
820         """
821         failed_results = {}
822         for test, result in result_summary.unexpected_results.iteritems():
823             if (result == test_expectations.PASS or
824                 result == test_expectations.CRASH and not include_crashes):
825                 continue
826             failed_results[test] = result
827
828         return failed_results
829
830     def _upload_json_files(self, unexpected_results, result_summary,
831                         individual_test_timings):
832         """Writes the results of the test run as JSON files into the results
833         dir and upload the files to the appengine server.
834
835         There are three different files written into the results dir:
836           unexpected_results.json: A short list of any unexpected results.
837             This is used by the buildbots to display results.
838           expectations.json: This is used by the flakiness dashboard.
839           results.json: A full list of the results - used by the flakiness
840             dashboard and the aggregate results dashboard.
841
842         Args:
843           unexpected_results: dict of unexpected results
844           result_summary: full summary object
845           individual_test_timings: list of test times (used by the flakiness
846             dashboard).
847         """
848         results_directory = self._options.results_directory
849         _log.debug("Writing JSON files in %s." % results_directory)
850         unexpected_json_path = os.path.join(results_directory, "unexpected_results.json")
851         with codecs.open(unexpected_json_path, "w", "utf-8") as file:
852             simplejson.dump(unexpected_results, file, sort_keys=True, indent=2)
853
854         # Write a json file of the test_expectations.txt file for the layout
855         # tests dashboard.
856         expectations_path = os.path.join(results_directory, "expectations.json")
857         expectations_json = \
858             self._expectations.get_expectations_json_for_all_platforms()
859         with codecs.open(expectations_path, "w", "utf-8") as file:
860             file.write(u"ADD_EXPECTATIONS(%s);" % expectations_json)
861
862         generator = json_layout_results_generator.JSONLayoutResultsGenerator(
863             self._port, self._options.builder_name, self._options.build_name,
864             self._options.build_number, self._options.results_directory,
865             BUILDER_BASE_URL, individual_test_timings,
866             self._expectations, result_summary, self._test_files_list,
867             not self._options.upload_full_results,
868             self._options.test_results_server,
869             "layout-tests",
870             self._options.master_name)
871
872         _log.debug("Finished writing JSON files.")
873
874         json_files = ["expectations.json"]
875         if self._options.upload_full_results:
876             json_files.append("results.json")
877         else:
878             json_files.append("incremental_results.json")
879
880         generator.upload_json_files(json_files)
881
882     def _print_config(self):
883         """Prints the configuration for the test run."""
884         p = self._printer
885         p.print_config("Using port '%s'" % self._port.name())
886         p.print_config("Placing test results in %s" %
887                        self._options.results_directory)
888         if self._options.new_baseline:
889             p.print_config("Placing new baselines in %s" %
890                            self._port.baseline_path())
891         p.print_config("Using %s build" % self._options.configuration)
892         if self._options.pixel_tests:
893             p.print_config("Pixel tests enabled")
894         else:
895             p.print_config("Pixel tests disabled")
896
897         p.print_config("Regular timeout: %s, slow test timeout: %s" %
898                        (self._options.time_out_ms,
899                         self._options.slow_time_out_ms))
900
901         if self._num_workers() == 1:
902             p.print_config("Running one %s" % self._port.driver_name())
903         else:
904             p.print_config("Running %s %ss in parallel" %
905                            (self._options.child_processes,
906                             self._port.driver_name()))
907         p.print_config('Command line: ' +
908                        ' '.join(self._port.driver_cmd_line()))
909         p.print_config("Worker model: %s" % self._options.worker_model)
910         p.print_config("")
911
912     def _print_expected_results_of_type(self, result_summary,
913                                         result_type, result_type_str):
914         """Print the number of the tests in a given result class.
915
916         Args:
917           result_summary - the object containing all the results to report on
918           result_type - the particular result type to report in the summary.
919           result_type_str - a string description of the result_type.
920         """
921         tests = self._expectations.get_tests_with_result_type(result_type)
922         now = result_summary.tests_by_timeline[test_expectations.NOW]
923         wontfix = result_summary.tests_by_timeline[test_expectations.WONTFIX]
924
925         # We use a fancy format string in order to print the data out in a
926         # nicely-aligned table.
927         fmtstr = ("Expect: %%5d %%-8s (%%%dd now, %%%dd wontfix)"
928                   % (self._num_digits(now), self._num_digits(wontfix)))
929         self._printer.print_expected(fmtstr %
930             (len(tests), result_type_str, len(tests & now), len(tests & wontfix)))
931
932     def _num_digits(self, num):
933         """Returns the number of digits needed to represent the length of a
934         sequence."""
935         ndigits = 1
936         if len(num):
937             ndigits = int(math.log10(len(num))) + 1
938         return ndigits
939
940     def _print_timing_statistics(self, total_time, thread_timings,
941                                directory_test_timings, individual_test_timings,
942                                result_summary):
943         """Record timing-specific information for the test run.
944
945         Args:
946           total_time: total elapsed time (in seconds) for the test run
947           thread_timings: wall clock time each thread ran for
948           directory_test_timings: timing by directory
949           individual_test_timings: timing by file
950           result_summary: summary object for the test run
951         """
952         self._printer.print_timing("Test timing:")
953         self._printer.print_timing("  %6.2f total testing time" % total_time)
954         self._printer.print_timing("")
955         self._printer.print_timing("Thread timing:")
956         cuml_time = 0
957         for t in thread_timings:
958             self._printer.print_timing("    %10s: %5d tests, %6.2f secs" %
959                   (t['name'], t['num_tests'], t['total_time']))
960             cuml_time += t['total_time']
961         self._printer.print_timing("   %6.2f cumulative, %6.2f optimal" %
962               (cuml_time, cuml_time / int(self._options.child_processes)))
963         self._printer.print_timing("")
964
965         self._print_aggregate_test_statistics(individual_test_timings)
966         self._print_individual_test_times(individual_test_timings,
967                                           result_summary)
968         self._print_directory_timings(directory_test_timings)
969
970     def _print_aggregate_test_statistics(self, individual_test_timings):
971         """Prints aggregate statistics (e.g. median, mean, etc.) for all tests.
972         Args:
973           individual_test_timings: List of dump_render_tree_thread.TestStats
974               for all tests.
975         """
976         test_types = []  # Unit tests don't actually produce any timings.
977         if individual_test_timings:
978             test_types = individual_test_timings[0].time_for_diffs.keys()
979         times_for_dump_render_tree = []
980         times_for_diff_processing = []
981         times_per_test_type = {}
982         for test_type in test_types:
983             times_per_test_type[test_type] = []
984
985         for test_stats in individual_test_timings:
986             times_for_dump_render_tree.append(test_stats.test_run_time)
987             times_for_diff_processing.append(
988                 test_stats.total_time_for_all_diffs)
989             time_for_diffs = test_stats.time_for_diffs
990             for test_type in test_types:
991                 times_per_test_type[test_type].append(
992                     time_for_diffs[test_type])
993
994         self._print_statistics_for_test_timings(
995             "PER TEST TIME IN TESTSHELL (seconds):",
996             times_for_dump_render_tree)
997         self._print_statistics_for_test_timings(
998             "PER TEST DIFF PROCESSING TIMES (seconds):",
999             times_for_diff_processing)
1000         for test_type in test_types:
1001             self._print_statistics_for_test_timings(
1002                 "PER TEST TIMES BY TEST TYPE: %s" % test_type,
1003                 times_per_test_type[test_type])
1004
1005     def _print_individual_test_times(self, individual_test_timings,
1006                                   result_summary):
1007         """Prints the run times for slow, timeout and crash tests.
1008         Args:
1009           individual_test_timings: List of dump_render_tree_thread.TestStats
1010               for all tests.
1011           result_summary: summary object for test run
1012         """
1013         # Reverse-sort by the time spent in DumpRenderTree.
1014         individual_test_timings.sort(lambda a, b:
1015             cmp(b.test_run_time, a.test_run_time))
1016
1017         num_printed = 0
1018         slow_tests = []
1019         timeout_or_crash_tests = []
1020         unexpected_slow_tests = []
1021         for test_tuple in individual_test_timings:
1022             filename = test_tuple.filename
1023             is_timeout_crash_or_slow = False
1024             if self._test_is_slow(filename):
1025                 is_timeout_crash_or_slow = True
1026                 slow_tests.append(test_tuple)
1027
1028             if filename in result_summary.failures:
1029                 result = result_summary.results[filename].type
1030                 if (result == test_expectations.TIMEOUT or
1031                     result == test_expectations.CRASH):
1032                     is_timeout_crash_or_slow = True
1033                     timeout_or_crash_tests.append(test_tuple)
1034
1035             if (not is_timeout_crash_or_slow and
1036                 num_printed < printing.NUM_SLOW_TESTS_TO_LOG):
1037                 num_printed = num_printed + 1
1038                 unexpected_slow_tests.append(test_tuple)
1039
1040         self._printer.print_timing("")
1041         self._print_test_list_timing("%s slowest tests that are not "
1042             "marked as SLOW and did not timeout/crash:" %
1043             printing.NUM_SLOW_TESTS_TO_LOG, unexpected_slow_tests)
1044         self._printer.print_timing("")
1045         self._print_test_list_timing("Tests marked as SLOW:", slow_tests)
1046         self._printer.print_timing("")
1047         self._print_test_list_timing("Tests that timed out or crashed:",
1048                                      timeout_or_crash_tests)
1049         self._printer.print_timing("")
1050
1051     def _print_test_list_timing(self, title, test_list):
1052         """Print timing info for each test.
1053
1054         Args:
1055           title: section heading
1056           test_list: tests that fall in this section
1057         """
1058         if self._printer.disabled('slowest'):
1059             return
1060
1061         self._printer.print_timing(title)
1062         for test_tuple in test_list:
1063             filename = test_tuple.filename[len(
1064                 self._port.layout_tests_dir()) + 1:]
1065             filename = filename.replace('\\', '/')
1066             test_run_time = round(test_tuple.test_run_time, 1)
1067             self._printer.print_timing("  %s took %s seconds" %
1068                                        (filename, test_run_time))
1069
1070     def _print_directory_timings(self, directory_test_timings):
1071         """Print timing info by directory for any directories that
1072         take > 10 seconds to run.
1073
1074         Args:
1075           directory_test_timing: time info for each directory
1076         """
1077         timings = []
1078         for directory in directory_test_timings:
1079             num_tests, time_for_directory = directory_test_timings[directory]
1080             timings.append((round(time_for_directory, 1), directory,
1081                             num_tests))
1082         timings.sort()
1083
1084         self._printer.print_timing("Time to process slowest subdirectories:")
1085         min_seconds_to_print = 10
1086         for timing in timings:
1087             if timing[0] > min_seconds_to_print:
1088                 self._printer.print_timing(
1089                     "  %s took %s seconds to run %s tests." % (timing[1],
1090                     timing[0], timing[2]))
1091         self._printer.print_timing("")
1092
1093     def _print_statistics_for_test_timings(self, title, timings):
1094         """Prints the median, mean and standard deviation of the values in
1095         timings.
1096
1097         Args:
1098           title: Title for these timings.
1099           timings: A list of floats representing times.
1100         """
1101         self._printer.print_timing(title)
1102         timings.sort()
1103
1104         num_tests = len(timings)
1105         if not num_tests:
1106             return
1107         percentile90 = timings[int(.9 * num_tests)]
1108         percentile99 = timings[int(.99 * num_tests)]
1109
1110         if num_tests % 2 == 1:
1111             median = timings[((num_tests - 1) / 2) - 1]
1112         else:
1113             lower = timings[num_tests / 2 - 1]
1114             upper = timings[num_tests / 2]
1115             median = (float(lower + upper)) / 2
1116
1117         mean = sum(timings) / num_tests
1118
1119         for time in timings:
1120             sum_of_deviations = math.pow(time - mean, 2)
1121
1122         std_deviation = math.sqrt(sum_of_deviations / num_tests)
1123         self._printer.print_timing("  Median:          %6.3f" % median)
1124         self._printer.print_timing("  Mean:            %6.3f" % mean)
1125         self._printer.print_timing("  90th percentile: %6.3f" % percentile90)
1126         self._printer.print_timing("  99th percentile: %6.3f" % percentile99)
1127         self._printer.print_timing("  Standard dev:    %6.3f" % std_deviation)
1128         self._printer.print_timing("")
1129
1130     def _print_result_summary(self, result_summary):
1131         """Print a short summary about how many tests passed.
1132
1133         Args:
1134           result_summary: information to log
1135         """
1136         failed = len(result_summary.failures)
1137         skipped = len(
1138             result_summary.tests_by_expectation[test_expectations.SKIP])
1139         total = result_summary.total
1140         passed = total - failed - skipped
1141         pct_passed = 0.0
1142         if total > 0:
1143             pct_passed = float(passed) * 100 / total
1144
1145         self._printer.print_actual("")
1146         self._printer.print_actual("=> Results: %d/%d tests passed (%.1f%%)" %
1147                      (passed, total, pct_passed))
1148         self._printer.print_actual("")
1149         self._print_result_summary_entry(result_summary,
1150             test_expectations.NOW, "Tests to be fixed")
1151
1152         self._printer.print_actual("")
1153         self._print_result_summary_entry(result_summary,
1154             test_expectations.WONTFIX,
1155             "Tests that will only be fixed if they crash (WONTFIX)")
1156         self._printer.print_actual("")
1157
1158     def _print_result_summary_entry(self, result_summary, timeline,
1159                                     heading):
1160         """Print a summary block of results for a particular timeline of test.
1161
1162         Args:
1163           result_summary: summary to print results for
1164           timeline: the timeline to print results for (NOT, WONTFIX, etc.)
1165           heading: a textual description of the timeline
1166         """
1167         total = len(result_summary.tests_by_timeline[timeline])
1168         not_passing = (total -
1169            len(result_summary.tests_by_expectation[test_expectations.PASS] &
1170                result_summary.tests_by_timeline[timeline]))
1171         self._printer.print_actual("=> %s (%d):" % (heading, not_passing))
1172
1173         for result in TestExpectationsFile.EXPECTATION_ORDER:
1174             if result == test_expectations.PASS:
1175                 continue
1176             results = (result_summary.tests_by_expectation[result] &
1177                        result_summary.tests_by_timeline[timeline])
1178             desc = TestExpectationsFile.EXPECTATION_DESCRIPTIONS[result]
1179             if not_passing and len(results):
1180                 pct = len(results) * 100.0 / not_passing
1181                 self._printer.print_actual("  %5d %-24s (%4.1f%%)" %
1182                     (len(results), desc[len(results) != 1], pct))
1183
1184     def _results_html(self, test_files, failures, title="Test Failures", override_time=None):
1185         """
1186         test_files = a list of file paths
1187         failures = dictionary mapping test paths to failure objects
1188         title = title printed at top of test
1189         override_time = current time (used by unit tests)
1190         """
1191         page = """<html>
1192   <head>
1193     <title>Layout Test Results (%(time)s)</title>
1194   </head>
1195   <body>
1196     <h2>%(title)s (%(time)s)</h2>
1197         """ % {'title': title, 'time': override_time or time.asctime()}
1198
1199         for test_file in sorted(test_files):
1200             test_name = self._port.relative_test_filename(test_file)
1201             test_url = self._port.filename_to_uri(test_file)
1202             page += u"<p><a href='%s'>%s</a><br />\n" % (test_url, test_name)
1203             test_failures = failures.get(test_file, [])
1204             for failure in test_failures:
1205                 page += (u"&nbsp;&nbsp;%s<br/>" %
1206                          failure.result_html_output(test_name))
1207             page += "</p>\n"
1208         page += "</body></html>\n"
1209         return page
1210
1211     def _write_results_html_file(self, result_summary):
1212         """Write results.html which is a summary of tests that failed.
1213
1214         Args:
1215           result_summary: a summary of the results :)
1216
1217         Returns:
1218           True if any results were written (since expected failures may be
1219           omitted)
1220         """
1221         # test failures
1222         if self._options.full_results_html:
1223             results_title = "Test Failures"
1224             test_files = result_summary.failures.keys()
1225         else:
1226             results_title = "Unexpected Test Failures"
1227             unexpected_failures = self._get_failures(result_summary,
1228                 include_crashes=True)
1229             test_files = unexpected_failures.keys()
1230         if not len(test_files):
1231             return False
1232
1233         out_filename = os.path.join(self._options.results_directory,
1234                                     "results.html")
1235         with codecs.open(out_filename, "w", "utf-8") as results_file:
1236             html = self._results_html(test_files, result_summary.failures, results_title)
1237             results_file.write(html)
1238
1239         return True
1240
1241     def _show_results_html_file(self):
1242         """Shows the results.html page."""
1243         results_filename = os.path.join(self._options.results_directory,
1244                                         "results.html")
1245         self._port.show_results_html_file(results_filename)
1246
1247
1248 def read_test_files(files):
1249     tests = []
1250     for file in files:
1251         try:
1252             with codecs.open(file, 'r', 'utf-8') as file_contents:
1253                 # FIXME: This could be cleaner using a list comprehension.
1254                 for line in file_contents:
1255                     line = test_expectations.strip_comments(line)
1256                     if line:
1257                         tests.append(line)
1258         except IOError, e:
1259             if e.errno == errno.ENOENT:
1260                 _log.critical('')
1261                 _log.critical('--test-list file "%s" not found' % file)
1262             raise
1263     return tests
1264
1265
1266 def run(port, options, args, regular_output=sys.stderr,
1267         buildbot_output=sys.stdout):
1268     """Run the tests.
1269
1270     Args:
1271       port: Port object for port-specific behavior
1272       options: a dictionary of command line options
1273       args: a list of sub directories or files to test
1274       regular_output: a stream-like object that we can send logging/debug
1275           output to
1276       buildbot_output: a stream-like object that we can write all output that
1277           is intended to be parsed by the buildbot to
1278     Returns:
1279       the number of unexpected results that occurred, or -1 if there is an
1280           error.
1281
1282     """
1283     _set_up_derived_options(port, options)
1284
1285     printer = printing.Printer(port, options, regular_output, buildbot_output,
1286         int(options.child_processes), options.experimental_fully_parallel)
1287     if options.help_printing:
1288         printer.help_printing()
1289         printer.cleanup()
1290         return 0
1291
1292     last_unexpected_results = _gather_unexpected_results(options)
1293     if options.print_last_failures:
1294         printer.write("\n".join(last_unexpected_results) + "\n")
1295         printer.cleanup()
1296         return 0
1297
1298     broker = message_broker.get(port, options)
1299
1300     # We wrap any parts of the run that are slow or likely to raise exceptions
1301     # in a try/finally to ensure that we clean up the logging configuration.
1302     num_unexpected_results = -1
1303     try:
1304         test_runner = TestRunner(port, options, printer, broker)
1305         test_runner._print_config()
1306
1307         printer.print_update("Collecting tests ...")
1308         try:
1309             test_runner.collect_tests(args, last_unexpected_results)
1310         except IOError, e:
1311             if e.errno == errno.ENOENT:
1312                 return -1
1313             raise
1314
1315         printer.print_update("Parsing expectations ...")
1316         if options.lint_test_files:
1317             return test_runner.lint()
1318         test_runner.parse_expectations(port.test_platform_name(),
1319                                        options.configuration == 'Debug')
1320
1321         printer.print_update("Checking build ...")
1322         if not port.check_build(test_runner.needs_http()):
1323             _log.error("Build check failed")
1324             return -1
1325
1326         result_summary = test_runner.set_up_run()
1327         if result_summary:
1328             num_unexpected_results = test_runner.run(result_summary)
1329             test_runner.clean_up_run()
1330             _log.debug("Testing completed, Exit status: %d" %
1331                        num_unexpected_results)
1332     finally:
1333         broker.cleanup()
1334         printer.cleanup()
1335
1336     return num_unexpected_results
1337
1338
1339 def _set_up_derived_options(port_obj, options):
1340     """Sets the options values that depend on other options values."""
1341
1342     if options.worker_model == 'inline':
1343         if options.child_processes and int(options.child_processes) > 1:
1344             _log.warning("--worker-model=inline overrides --child-processes")
1345         options.child_processes = "1"
1346     if not options.child_processes:
1347         options.child_processes = os.environ.get("WEBKIT_TEST_CHILD_PROCESSES",
1348                                                  str(port_obj.default_child_processes()))
1349
1350     if not options.configuration:
1351         options.configuration = port_obj.default_configuration()
1352
1353     if options.pixel_tests is None:
1354         options.pixel_tests = True
1355
1356     if not options.use_apache:
1357         options.use_apache = sys.platform in ('darwin', 'linux2')
1358
1359     if not os.path.isabs(options.results_directory):
1360         # This normalizes the path to the build dir.
1361         # FIXME: how this happens is not at all obvious; this is a dumb
1362         # interface and should be cleaned up.
1363         options.results_directory = port_obj.results_directory()
1364
1365     if not options.time_out_ms:
1366         if options.configuration == "Debug":
1367             options.time_out_ms = str(2 * TestRunner.DEFAULT_TEST_TIMEOUT_MS)
1368         else:
1369             options.time_out_ms = str(TestRunner.DEFAULT_TEST_TIMEOUT_MS)
1370
1371     options.slow_time_out_ms = str(5 * int(options.time_out_ms))
1372
1373
1374 def _gather_unexpected_results(options):
1375     """Returns the unexpected results from the previous run, if any."""
1376     last_unexpected_results = []
1377     if options.print_last_failures or options.retest_last_failures:
1378         unexpected_results_filename = os.path.join(
1379         options.results_directory, "unexpected_results.json")
1380         with codecs.open(unexpected_results_filename, "r", "utf-8") as file:
1381             results = simplejson.load(file)
1382         last_unexpected_results = results['tests'].keys()
1383     return last_unexpected_results
1384
1385
1386 def _compat_shim_callback(option, opt_str, value, parser):
1387     print "Ignoring unsupported option: %s" % opt_str
1388
1389
1390 def _compat_shim_option(option_name, **kwargs):
1391     return optparse.make_option(option_name, action="callback",
1392         callback=_compat_shim_callback,
1393         help="Ignored, for old-run-webkit-tests compat only.", **kwargs)
1394
1395
1396 def parse_args(args=None):
1397     """Provides a default set of command line args.
1398
1399     Returns a tuple of options, args from optparse"""
1400
1401     # FIXME: All of these options should be stored closer to the code which
1402     # FIXME: actually uses them. configuration_options should move
1403     # FIXME: to WebKitPort and be shared across all scripts.
1404     configuration_options = [
1405         optparse.make_option("-t", "--target", dest="configuration",
1406                              help="(DEPRECATED)"),
1407         # FIXME: --help should display which configuration is default.
1408         optparse.make_option('--debug', action='store_const', const='Debug',
1409                              dest="configuration",
1410                              help='Set the configuration to Debug'),
1411         optparse.make_option('--release', action='store_const',
1412                              const='Release', dest="configuration",
1413                              help='Set the configuration to Release'),
1414         # old-run-webkit-tests also accepts -c, --configuration CONFIGURATION.
1415     ]
1416
1417     print_options = printing.print_options()
1418
1419     # FIXME: These options should move onto the ChromiumPort.
1420     chromium_options = [
1421         optparse.make_option("--chromium", action="store_true", default=False,
1422             help="use the Chromium port"),
1423         optparse.make_option("--startup-dialog", action="store_true",
1424             default=False, help="create a dialog on DumpRenderTree startup"),
1425         optparse.make_option("--gp-fault-error-box", action="store_true",
1426             default=False, help="enable Windows GP fault error box"),
1427         optparse.make_option("--multiple-loads",
1428             type="int", help="turn on multiple loads of each test"),
1429         optparse.make_option("--js-flags",
1430             type="string", help="JavaScript flags to pass to tests"),
1431         optparse.make_option("--nocheck-sys-deps", action="store_true",
1432             default=False,
1433             help="Don't check the system dependencies (themes)"),
1434         optparse.make_option("--use-drt", action="store_true",
1435             default=None,
1436             help="Use DumpRenderTree instead of test_shell"),
1437         optparse.make_option("--accelerated-compositing",
1438             action="store_true",
1439             help="Use hardware-accelated compositing for rendering"),
1440         optparse.make_option("--no-accelerated-compositing",
1441             action="store_false",
1442             dest="accelerated_compositing",
1443             help="Don't use hardware-accelerated compositing for rendering"),
1444         optparse.make_option("--accelerated-2d-canvas",
1445             action="store_true",
1446             help="Use hardware-accelerated 2D Canvas calls"),
1447         optparse.make_option("--no-accelerated-2d-canvas",
1448             action="store_false",
1449             dest="accelerated_2d_canvas",
1450             help="Don't use hardware-accelerated 2D Canvas calls"),
1451     ]
1452
1453     # Missing Mac-specific old-run-webkit-tests options:
1454     # FIXME: Need: -g, --guard for guard malloc support on Mac.
1455     # FIXME: Need: -l --leaks    Enable leaks checking.
1456     # FIXME: Need: --sample-on-timeout Run sample on timeout
1457
1458     old_run_webkit_tests_compat = [
1459         # NRWT doesn't generate results by default anyway.
1460         _compat_shim_option("--no-new-test-results"),
1461         # NRWT doesn't sample on timeout yet anyway.
1462         _compat_shim_option("--no-sample-on-timeout"),
1463         # FIXME: NRWT needs to support remote links eventually.
1464         _compat_shim_option("--use-remote-links-to-tests"),
1465         # FIXME: NRWT doesn't need this option as much since failures are
1466         # designed to be cheap.  We eventually plan to add this support.
1467         _compat_shim_option("--exit-after-n-failures", nargs=1, type="int"),
1468     ]
1469
1470     results_options = [
1471         # NEED for bots: --use-remote-links-to-tests Link to test files
1472         # within the SVN repository in the results.
1473         optparse.make_option("-p", "--pixel-tests", action="store_true",
1474             dest="pixel_tests", help="Enable pixel-to-pixel PNG comparisons"),
1475         optparse.make_option("--no-pixel-tests", action="store_false",
1476             dest="pixel_tests", help="Disable pixel-to-pixel PNG comparisons"),
1477         optparse.make_option("--tolerance",
1478             help="Ignore image differences less than this percentage (some "
1479                 "ports may ignore this option)", type="float"),
1480         optparse.make_option("--results-directory",
1481             default="layout-test-results",
1482             help="Output results directory source dir, relative to Debug or "
1483                  "Release"),
1484         optparse.make_option("--new-baseline", action="store_true",
1485             default=False, help="Save all generated results as new baselines "
1486                  "into the platform directory, overwriting whatever's "
1487                  "already there."),
1488         optparse.make_option("--reset-results", action="store_true",
1489             default=False, help="Reset any existing baselines to the "
1490                  "generated results"),
1491         optparse.make_option("--no-show-results", action="store_false",
1492             default=True, dest="show_results",
1493             help="Don't launch a browser with results after the tests "
1494                  "are done"),
1495         # FIXME: We should have a helper function to do this sort of
1496         # deprectated mapping and automatically log, etc.
1497         optparse.make_option("--noshow-results", action="store_false",
1498             dest="show_results",
1499             help="Deprecated, same as --no-show-results."),
1500         optparse.make_option("--no-launch-safari", action="store_false",
1501             dest="show_results",
1502             help="old-run-webkit-tests compat, same as --noshow-results."),
1503         # old-run-webkit-tests:
1504         # --[no-]launch-safari    Launch (or do not launch) Safari to display
1505         #                         test results (default: launch)
1506         optparse.make_option("--full-results-html", action="store_true",
1507             default=False,
1508             help="Show all failures in results.html, rather than only "
1509                  "regressions"),
1510         optparse.make_option("--clobber-old-results", action="store_true",
1511             default=False, help="Clobbers test results from previous runs."),
1512         optparse.make_option("--platform",
1513             help="Override the platform for expected results"),
1514         optparse.make_option("--no-record-results", action="store_false",
1515             default=True, dest="record_results",
1516             help="Don't record the results."),
1517         # old-run-webkit-tests also has HTTP toggle options:
1518         # --[no-]http                     Run (or do not run) http tests
1519         #                                 (default: run)
1520     ]
1521
1522     test_options = [
1523         optparse.make_option("--build", dest="build",
1524             action="store_true", default=True,
1525             help="Check to ensure the DumpRenderTree build is up-to-date "
1526                  "(default)."),
1527         optparse.make_option("--no-build", dest="build",
1528             action="store_false", help="Don't check to see if the "
1529                                        "DumpRenderTree build is up-to-date."),
1530         optparse.make_option("-n", "--dry-run", action="store_true",
1531             default=False,
1532             help="Do everything but actually run the tests or upload results."),
1533         # old-run-webkit-tests has --valgrind instead of wrapper.
1534         optparse.make_option("--wrapper",
1535             help="wrapper command to insert before invocations of "
1536                  "DumpRenderTree; option is split on whitespace before "
1537                  "running. (Example: --wrapper='valgrind --smc-check=all')"),
1538         # old-run-webkit-tests:
1539         # -i|--ignore-tests               Comma-separated list of directories
1540         #                                 or tests to ignore
1541         optparse.make_option("--test-list", action="append",
1542             help="read list of tests to run from file", metavar="FILE"),
1543         # old-run-webkit-tests uses --skipped==[default|ignore|only]
1544         # instead of --force:
1545         optparse.make_option("--force", action="store_true", default=False,
1546             help="Run all tests, even those marked SKIP in the test list"),
1547         optparse.make_option("--use-apache", action="store_true",
1548             default=False, help="Whether to use apache instead of lighttpd."),
1549         optparse.make_option("--time-out-ms",
1550             help="Set the timeout for each test"),
1551         # old-run-webkit-tests calls --randomize-order --random:
1552         optparse.make_option("--randomize-order", action="store_true",
1553             default=False, help=("Run tests in random order (useful "
1554                                 "for tracking down corruption)")),
1555         optparse.make_option("--run-chunk",
1556             help=("Run a specified chunk (n:l), the nth of len l, "
1557                  "of the layout tests")),
1558         optparse.make_option("--run-part", help=("Run a specified part (n:m), "
1559                   "the nth of m parts, of the layout tests")),
1560         # old-run-webkit-tests calls --batch-size: --nthly n
1561         #   Restart DumpRenderTree every n tests (default: 1000)
1562         optparse.make_option("--batch-size",
1563             help=("Run a the tests in batches (n), after every n tests, "
1564                   "DumpRenderTree is relaunched."), type="int", default=0),
1565         # old-run-webkit-tests calls --run-singly: -1|--singly
1566         # Isolate each test case run (implies --nthly 1 --verbose)
1567         optparse.make_option("--run-singly", action="store_true",
1568             default=False, help="run a separate DumpRenderTree for each test"),
1569         optparse.make_option("--child-processes",
1570             help="Number of DumpRenderTrees to run in parallel."),
1571         # FIXME: Display default number of child processes that will run.
1572         optparse.make_option("--worker-model", action="store",
1573             default="threads", help=("controls worker model. Valid values are "
1574             "'inline' and 'threads' (default).")),
1575         optparse.make_option("--experimental-fully-parallel",
1576             action="store_true", default=False,
1577             help="run all tests in parallel"),
1578         # FIXME: Need --exit-after-n-failures N
1579         #      Exit after the first N failures instead of running all tests
1580         # FIXME: Need --exit-after-n-crashes N
1581         #      Exit after the first N crashes instead of running all tests
1582         # FIXME: consider: --iterations n
1583         #      Number of times to run the set of tests (e.g. ABCABCABC)
1584         optparse.make_option("--print-last-failures", action="store_true",
1585             default=False, help="Print the tests in the last run that "
1586             "had unexpected failures (or passes) and then exit."),
1587         optparse.make_option("--retest-last-failures", action="store_true",
1588             default=False, help="re-test the tests in the last run that "
1589             "had unexpected failures (or passes)."),
1590         optparse.make_option("--retry-failures", action="store_true",
1591             default=True,
1592             help="Re-try any tests that produce unexpected results (default)"),
1593         optparse.make_option("--no-retry-failures", action="store_false",
1594             dest="retry_failures",
1595             help="Don't re-try any tests that produce unexpected results."),
1596     ]
1597
1598     misc_options = [
1599         optparse.make_option("--lint-test-files", action="store_true",
1600         default=False, help=("Makes sure the test files parse for all "
1601                             "configurations. Does not run any tests.")),
1602     ]
1603
1604     # FIXME: Move these into json_results_generator.py
1605     results_json_options = [
1606         optparse.make_option("--master-name", help="The name of the buildbot master."),
1607         optparse.make_option("--builder-name", default="DUMMY_BUILDER_NAME",
1608             help=("The name of the builder shown on the waterfall running "
1609                   "this script e.g. WebKit.")),
1610         optparse.make_option("--build-name", default="DUMMY_BUILD_NAME",
1611             help=("The name of the builder used in its path, e.g. "
1612                   "webkit-rel.")),
1613         optparse.make_option("--build-number", default="DUMMY_BUILD_NUMBER",
1614             help=("The build number of the builder running this script.")),
1615         optparse.make_option("--test-results-server", default="",
1616             help=("If specified, upload results json files to this appengine "
1617                   "server.")),
1618         optparse.make_option("--upload-full-results",
1619             action="store_true",
1620             default=False,
1621             help="If true, upload full json results to server."),
1622     ]
1623
1624     option_list = (configuration_options + print_options +
1625                    chromium_options + results_options + test_options +
1626                    misc_options + results_json_options +
1627                    old_run_webkit_tests_compat)
1628     option_parser = optparse.OptionParser(option_list=option_list)
1629
1630     return option_parser.parse_args(args)
1631
1632
1633 def main():
1634     options, args = parse_args()
1635     port_obj = port.get(options.platform, options)
1636     return run(port_obj, options, args)
1637
1638
1639 if '__main__' == __name__:
1640     try:
1641         sys.exit(main())
1642     except KeyboardInterrupt:
1643         # this mirrors what the shell normally does
1644         sys.exit(signal.SIGINT + 128)