Clean up ChunkedUpdateDrawingAreaProxy
[WebKit-https.git] / WebKitTools / Scripts / webkitpy / layout_tests / run_webkit_tests.py
1 #!/usr/bin/env python
2 # Copyright (C) 2010 Google Inc. All rights reserved.
3 # Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Szeged
4 #
5 # Redistribution and use in source and binary forms, with or without
6 # modification, are permitted provided that the following conditions are
7 # met:
8 #
9 #     * Redistributions of source code must retain the above copyright
10 # notice, this list of conditions and the following disclaimer.
11 #     * Redistributions in binary form must reproduce the above
12 # copyright notice, this list of conditions and the following disclaimer
13 # in the documentation and/or other materials provided with the
14 # distribution.
15 #     * Neither the name of Google Inc. nor the names of its
16 # contributors may be used to endorse or promote products derived from
17 # this software without specific prior written permission.
18 #
19 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30
31 """Run layout tests.
32
33 This is a port of the existing webkit test script run-webkit-tests.
34
35 The TestRunner class runs a series of tests (TestType interface) against a set
36 of test files.  If a test file fails a TestType, it returns a list TestFailure
37 objects to the TestRunner.  The TestRunner then aggregates the TestFailures to
38 create a final report.
39
40 This script reads several files, if they exist in the test_lists subdirectory
41 next to this script itself.  Each should contain a list of paths to individual
42 tests or entire subdirectories of tests, relative to the outermost test
43 directory.  Entire lines starting with '//' (comments) will be ignored.
44
45 For details of the files' contents and purposes, see test_lists/README.
46 """
47
48 from __future__ import with_statement
49
50 import codecs
51 import errno
52 import glob
53 import logging
54 import math
55 import optparse
56 import os
57 import platform
58 import Queue
59 import random
60 import re
61 import shutil
62 import signal
63 import sys
64 import time
65 import traceback
66
67 from layout_package import dump_render_tree_thread
68 from layout_package import json_layout_results_generator
69 from layout_package import message_broker
70 from layout_package import printing
71 from layout_package import test_expectations
72 from layout_package import test_failures
73 from layout_package import test_results
74 from layout_package import test_results_uploader
75
76 from webkitpy.common.system import user
77 from webkitpy.thirdparty import simplejson
78 from webkitpy.tool import grammar
79
80 import port
81
82 _log = logging.getLogger("webkitpy.layout_tests.run_webkit_tests")
83
84 # Builder base URL where we have the archived test results.
85 BUILDER_BASE_URL = "http://build.chromium.org/buildbot/layout_test_results/"
86
87 LAYOUT_TESTS_DIRECTORY = "LayoutTests" + os.sep
88
89 TestExpectationsFile = test_expectations.TestExpectationsFile
90
91
92 class TestInput:
93     """Groups information about a test for easy passing of data."""
94
95     def __init__(self, filename, timeout):
96         """Holds the input parameters for a test.
97         Args:
98           filename: Full path to the test.
99           timeout: Timeout in msecs the driver should use while running the test
100           """
101         # FIXME: filename should really be test_name as a relative path.
102         self.filename = filename
103         self.timeout = timeout
104         # The image_hash is used to avoid doing an image dump if the
105         # checksums match. The image_hash is set later, and only if it is needed
106         # for the test.
107         self.image_hash = None
108
109
110 class TestRunInterruptedException(Exception):
111     """Raised when a test run should be stopped immediately."""
112     def __init__(self, reason):
113         self.reason = reason
114
115
116 class ResultSummary(object):
117     """A class for partitioning the test results we get into buckets.
118
119     This class is basically a glorified struct and it's private to this file
120     so we don't bother with any information hiding."""
121
122     def __init__(self, expectations, test_files):
123         self.total = len(test_files)
124         self.remaining = self.total
125         self.expectations = expectations
126         self.expected = 0
127         self.unexpected = 0
128         self.unexpected_failures = 0
129         self.unexpected_crashes_or_timeouts = 0
130         self.tests_by_expectation = {}
131         self.tests_by_timeline = {}
132         self.results = {}
133         self.unexpected_results = {}
134         self.failures = {}
135         self.tests_by_expectation[test_expectations.SKIP] = set()
136         for expectation in TestExpectationsFile.EXPECTATIONS.values():
137             self.tests_by_expectation[expectation] = set()
138         for timeline in TestExpectationsFile.TIMELINES.values():
139             self.tests_by_timeline[timeline] = (
140                 expectations.get_tests_with_timeline(timeline))
141
142     def add(self, result, expected):
143         """Add a TestResult into the appropriate bin.
144
145         Args:
146           result: TestResult from dump_render_tree_thread.
147           expected: whether the result was what we expected it to be.
148         """
149
150         self.tests_by_expectation[result.type].add(result.filename)
151         self.results[result.filename] = result
152         self.remaining -= 1
153         if len(result.failures):
154             self.failures[result.filename] = result.failures
155         if expected:
156             self.expected += 1
157         else:
158             self.unexpected_results[result.filename] = result.type
159             self.unexpected += 1
160             if len(result.failures):
161                 self.unexpected_failures += 1
162             if result.type == test_expectations.CRASH or result.type == test_expectations.TIMEOUT:
163                 self.unexpected_crashes_or_timeouts += 1
164
165
166 def summarize_unexpected_results(port_obj, expectations, result_summary,
167                                  retry_summary):
168     """Summarize any unexpected results as a dict.
169
170     FIXME: split this data structure into a separate class?
171
172     Args:
173         port_obj: interface to port-specific hooks
174         expectations: test_expectations.TestExpectations object
175         result_summary: summary object from initial test runs
176         retry_summary: summary object from final test run of retried tests
177     Returns:
178         A dictionary containing a summary of the unexpected results from the
179         run, with the following fields:
180         'version': a version indicator (1 in this version)
181         'fixable': # of fixable tests (NOW - PASS)
182         'skipped': # of skipped tests (NOW & SKIPPED)
183         'num_regressions': # of non-flaky failures
184         'num_flaky': # of flaky failures
185         'num_passes': # of unexpected passes
186         'tests': a dict of tests -> {'expected': '...', 'actual': '...'}
187     """
188     results = {}
189     results['version'] = 1
190
191     tbe = result_summary.tests_by_expectation
192     tbt = result_summary.tests_by_timeline
193     results['fixable'] = len(tbt[test_expectations.NOW] -
194                                 tbe[test_expectations.PASS])
195     results['skipped'] = len(tbt[test_expectations.NOW] &
196                                 tbe[test_expectations.SKIP])
197
198     num_passes = 0
199     num_flaky = 0
200     num_regressions = 0
201     keywords = {}
202     for k, v in TestExpectationsFile.EXPECTATIONS.iteritems():
203         keywords[v] = k.upper()
204
205     tests = {}
206     for filename, result in result_summary.unexpected_results.iteritems():
207         # Note that if a test crashed in the original run, we ignore
208         # whether or not it crashed when we retried it (if we retried it),
209         # and always consider the result not flaky.
210         test = port_obj.relative_test_filename(filename)
211         expected = expectations.get_expectations_string(filename)
212         actual = [keywords[result]]
213
214         if result == test_expectations.PASS:
215             num_passes += 1
216         elif result == test_expectations.CRASH:
217             num_regressions += 1
218         else:
219             if filename not in retry_summary.unexpected_results:
220                 actual.extend(expectations.get_expectations_string(
221                     filename).split(" "))
222                 num_flaky += 1
223             else:
224                 retry_result = retry_summary.unexpected_results[filename]
225                 if result != retry_result:
226                     actual.append(keywords[retry_result])
227                     num_flaky += 1
228                 else:
229                     num_regressions += 1
230
231         tests[test] = {}
232         tests[test]['expected'] = expected
233         tests[test]['actual'] = " ".join(actual)
234
235     results['tests'] = tests
236     results['num_passes'] = num_passes
237     results['num_flaky'] = num_flaky
238     results['num_regressions'] = num_regressions
239
240     return results
241
242
243 class TestRunner:
244     """A class for managing running a series of tests on a series of layout
245     test files."""
246
247     HTTP_SUBDIR = os.sep.join(['', 'http', ''])
248     WEBSOCKET_SUBDIR = os.sep.join(['', 'websocket', ''])
249
250     # The per-test timeout in milliseconds, if no --time-out-ms option was
251     # given to run_webkit_tests. This should correspond to the default timeout
252     # in DumpRenderTree.
253     DEFAULT_TEST_TIMEOUT_MS = 6 * 1000
254
255     def __init__(self, port, options, printer, message_broker):
256         """Initialize test runner data structures.
257
258         Args:
259           port: an object implementing port-specific
260           options: a dictionary of command line options
261           printer: a Printer object to record updates to.
262           message_broker: object used to communicate with workers.
263         """
264         self._port = port
265         self._options = options
266         self._printer = printer
267         self._message_broker = message_broker
268
269         # disable wss server. need to install pyOpenSSL on buildbots.
270         # self._websocket_secure_server = websocket_server.PyWebSocket(
271         #        options.results_directory, use_tls=True, port=9323)
272
273         # a set of test files, and the same tests as a list
274         self._test_files = set()
275         self._test_files_list = None
276         self._result_queue = Queue.Queue()
277         self._retrying = False
278
279     def collect_tests(self, args, last_unexpected_results):
280         """Find all the files to test.
281
282         Args:
283           args: list of test arguments from the command line
284           last_unexpected_results: list of unexpected results to retest, if any
285
286         """
287         paths = [self._strip_test_dir_prefix(arg) for arg in args if arg and arg != '']
288         paths += last_unexpected_results
289         if self._options.test_list:
290             paths += read_test_files(self._options.test_list)
291         self._test_files = self._port.tests(paths)
292
293     def _strip_test_dir_prefix(self, path):
294         if path.startswith(LAYOUT_TESTS_DIRECTORY):
295             return path[len(LAYOUT_TESTS_DIRECTORY):]
296         return path
297
298     def lint(self):
299         lint_failed = False
300
301         # Creating the expecations for each platform/configuration pair does
302         # all the test list parsing and ensures it's correct syntax (e.g. no
303         # dupes).
304         for platform_name in self._port.test_platform_names():
305             try:
306                 self.parse_expectations(platform_name, is_debug_mode=True)
307             except test_expectations.ParseError:
308                 lint_failed = True
309             try:
310                 self.parse_expectations(platform_name, is_debug_mode=False)
311             except test_expectations.ParseError:
312                 lint_failed = True
313
314         self._printer.write("")
315         if lint_failed:
316             _log.error("Lint failed.")
317             return -1
318
319         _log.info("Lint succeeded.")
320         return 0
321
322     def parse_expectations(self, test_platform_name, is_debug_mode):
323         """Parse the expectations from the test_list files and return a data
324         structure holding them. Throws an error if the test_list files have
325         invalid syntax."""
326         if self._options.lint_test_files:
327             test_files = None
328         else:
329             test_files = self._test_files
330
331         expectations_str = self._port.test_expectations()
332         overrides_str = self._port.test_expectations_overrides()
333         self._expectations = test_expectations.TestExpectations(
334             self._port, test_files, expectations_str, test_platform_name,
335             is_debug_mode, self._options.lint_test_files,
336             overrides=overrides_str)
337         return self._expectations
338
339
340     def prepare_lists_and_print_output(self):
341         """Create appropriate subsets of test lists and returns a
342         ResultSummary object. Also prints expected test counts.
343         """
344
345         # Remove skipped - both fixable and ignored - files from the
346         # top-level list of files to test.
347         num_all_test_files = len(self._test_files)
348         self._printer.print_expected("Found:  %d tests" %
349                                      (len(self._test_files)))
350         if not num_all_test_files:
351             _log.critical('No tests to run.')
352             return None
353
354         skipped = set()
355         if num_all_test_files > 1 and not self._options.force:
356             skipped = self._expectations.get_tests_with_result_type(
357                            test_expectations.SKIP)
358             self._test_files -= skipped
359
360         # Create a sorted list of test files so the subset chunk,
361         # if used, contains alphabetically consecutive tests.
362         self._test_files_list = list(self._test_files)
363         if self._options.randomize_order:
364             random.shuffle(self._test_files_list)
365         else:
366             self._test_files_list.sort()
367
368         # If the user specifies they just want to run a subset of the tests,
369         # just grab a subset of the non-skipped tests.
370         if self._options.run_chunk or self._options.run_part:
371             chunk_value = self._options.run_chunk or self._options.run_part
372             test_files = self._test_files_list
373             try:
374                 (chunk_num, chunk_len) = chunk_value.split(":")
375                 chunk_num = int(chunk_num)
376                 assert(chunk_num >= 0)
377                 test_size = int(chunk_len)
378                 assert(test_size > 0)
379             except:
380                 _log.critical("invalid chunk '%s'" % chunk_value)
381                 return None
382
383             # Get the number of tests
384             num_tests = len(test_files)
385
386             # Get the start offset of the slice.
387             if self._options.run_chunk:
388                 chunk_len = test_size
389                 # In this case chunk_num can be really large. We need
390                 # to make the slave fit in the current number of tests.
391                 slice_start = (chunk_num * chunk_len) % num_tests
392             else:
393                 # Validate the data.
394                 assert(test_size <= num_tests)
395                 assert(chunk_num <= test_size)
396
397                 # To count the chunk_len, and make sure we don't skip
398                 # some tests, we round to the next value that fits exactly
399                 # all the parts.
400                 rounded_tests = num_tests
401                 if rounded_tests % test_size != 0:
402                     rounded_tests = (num_tests + test_size -
403                                      (num_tests % test_size))
404
405                 chunk_len = rounded_tests / test_size
406                 slice_start = chunk_len * (chunk_num - 1)
407                 # It does not mind if we go over test_size.
408
409             # Get the end offset of the slice.
410             slice_end = min(num_tests, slice_start + chunk_len)
411
412             files = test_files[slice_start:slice_end]
413
414             tests_run_msg = 'Running: %d tests (chunk slice [%d:%d] of %d)' % (
415                 (slice_end - slice_start), slice_start, slice_end, num_tests)
416             self._printer.print_expected(tests_run_msg)
417
418             # If we reached the end and we don't have enough tests, we run some
419             # from the beginning.
420             if slice_end - slice_start < chunk_len:
421                 extra = chunk_len - (slice_end - slice_start)
422                 extra_msg = ('   last chunk is partial, appending [0:%d]' %
423                             extra)
424                 self._printer.print_expected(extra_msg)
425                 tests_run_msg += "\n" + extra_msg
426                 files.extend(test_files[0:extra])
427             tests_run_filename = os.path.join(self._options.results_directory,
428                                               "tests_run.txt")
429             with codecs.open(tests_run_filename, "w", "utf-8") as file:
430                 file.write(tests_run_msg + "\n")
431
432             len_skip_chunk = int(len(files) * len(skipped) /
433                                  float(len(self._test_files)))
434             skip_chunk_list = list(skipped)[0:len_skip_chunk]
435             skip_chunk = set(skip_chunk_list)
436
437             # Update expectations so that the stats are calculated correctly.
438             # We need to pass a list that includes the right # of skipped files
439             # to ParseExpectations so that ResultSummary() will get the correct
440             # stats. So, we add in the subset of skipped files, and then
441             # subtract them back out.
442             self._test_files_list = files + skip_chunk_list
443             self._test_files = set(self._test_files_list)
444
445             self._expectations = self.parse_expectations(
446                 self._port.test_platform_name(),
447                 self._options.configuration == 'Debug')
448
449             self._test_files = set(files)
450             self._test_files_list = files
451         else:
452             skip_chunk = skipped
453
454         result_summary = ResultSummary(self._expectations,
455             self._test_files | skip_chunk)
456         self._print_expected_results_of_type(result_summary,
457             test_expectations.PASS, "passes")
458         self._print_expected_results_of_type(result_summary,
459             test_expectations.FAIL, "failures")
460         self._print_expected_results_of_type(result_summary,
461             test_expectations.FLAKY, "flaky")
462         self._print_expected_results_of_type(result_summary,
463             test_expectations.SKIP, "skipped")
464
465         if self._options.force:
466             self._printer.print_expected('Running all tests, including '
467                                          'skips (--force)')
468         else:
469             # Note that we don't actually run the skipped tests (they were
470             # subtracted out of self._test_files, above), but we stub out the
471             # results here so the statistics can remain accurate.
472             for test in skip_chunk:
473                 result = test_results.TestResult(test,
474                     failures=[], test_run_time=0, total_time_for_all_diffs=0,
475                     time_for_diffs=0)
476                 result.type = test_expectations.SKIP
477                 result_summary.add(result, expected=True)
478         self._printer.print_expected('')
479
480         return result_summary
481
482     def _get_dir_for_test_file(self, test_file):
483         """Returns the highest-level directory by which to shard the given
484         test file."""
485         index = test_file.rfind(os.sep + LAYOUT_TESTS_DIRECTORY)
486
487         test_file = test_file[index + len(LAYOUT_TESTS_DIRECTORY):]
488         test_file_parts = test_file.split(os.sep, 1)
489         directory = test_file_parts[0]
490         test_file = test_file_parts[1]
491
492         # The http tests are very stable on mac/linux.
493         # TODO(ojan): Make the http server on Windows be apache so we can
494         # turn shard the http tests there as well. Switching to apache is
495         # what made them stable on linux/mac.
496         return_value = directory
497         while ((directory != 'http' or sys.platform in ('darwin', 'linux2'))
498                 and test_file.find(os.sep) >= 0):
499             test_file_parts = test_file.split(os.sep, 1)
500             directory = test_file_parts[0]
501             return_value = os.path.join(return_value, directory)
502             test_file = test_file_parts[1]
503
504         return return_value
505
506     def _get_test_input_for_file(self, test_file):
507         """Returns the appropriate TestInput object for the file. Mostly this
508         is used for looking up the timeout value (in ms) to use for the given
509         test."""
510         if self._test_is_slow(test_file):
511             return TestInput(test_file, self._options.slow_time_out_ms)
512         return TestInput(test_file, self._options.time_out_ms)
513
514     def _test_requires_lock(self, test_file):
515         """Return True if the test needs to be locked when
516         running multiple copies of NRWTs."""
517         split_path = test_file.split(os.sep)
518         return 'http' in split_path or 'websocket' in split_path
519
520     def _test_is_slow(self, test_file):
521         return self._expectations.has_modifier(test_file,
522                                                test_expectations.SLOW)
523
524     def _shard_tests(self, test_files, use_real_shards):
525         """Groups tests into batches.
526         This helps ensure that tests that depend on each other (aka bad tests!)
527         continue to run together as most cross-tests dependencies tend to
528         occur within the same directory. If use_real_shards is false, we
529         put each (non-HTTP/websocket) test into its own shard for maximum
530         concurrency instead of trying to do any sort of real sharding.
531
532         Return:
533             A list of lists of TestInput objects.
534         """
535         # FIXME: when we added http locking, we changed how this works such
536         # that we always lump all of the HTTP threads into a single shard.
537         # That will slow down experimental-fully-parallel, but it's unclear
538         # what the best alternative is completely revamping how we track
539         # when to grab the lock.
540
541         test_lists = []
542         tests_to_http_lock = []
543         if not use_real_shards:
544             for test_file in test_files:
545                 test_input = self._get_test_input_for_file(test_file)
546                 if self._test_requires_lock(test_file):
547                     tests_to_http_lock.append(test_input)
548                 else:
549                     test_lists.append((".", [test_input]))
550         else:
551             tests_by_dir = {}
552             for test_file in test_files:
553                 directory = self._get_dir_for_test_file(test_file)
554                 test_input = self._get_test_input_for_file(test_file)
555                 if self._test_requires_lock(test_file):
556                     tests_to_http_lock.append(test_input)
557                 else:
558                     tests_by_dir.setdefault(directory, [])
559                     tests_by_dir[directory].append(test_input)
560             # Sort by the number of tests in the dir so that the ones with the
561             # most tests get run first in order to maximize parallelization.
562             # Number of tests is a good enough, but not perfect, approximation
563             # of how long that set of tests will take to run. We can't just use
564             # a PriorityQueue until we move to Python 2.6.
565             for directory in tests_by_dir:
566                 test_list = tests_by_dir[directory]
567                 # Keep the tests in alphabetical order.
568                 # FIXME: Remove once tests are fixed so they can be run in any
569                 # order.
570                 test_list.reverse()
571                 test_list_tuple = (directory, test_list)
572                 test_lists.append(test_list_tuple)
573             test_lists.sort(lambda a, b: cmp(len(b[1]), len(a[1])))
574
575         # Put the http tests first. There are only a couple hundred of them,
576         # but each http test takes a very long time to run, so sorting by the
577         # number of tests doesn't accurately capture how long they take to run.
578         if tests_to_http_lock:
579             tests_to_http_lock.reverse()
580             test_lists.insert(0, ("tests_to_http_lock", tests_to_http_lock))
581
582         return test_lists
583
584     def _contains_tests(self, subdir):
585         for test_file in self._test_files:
586             if test_file.find(subdir) >= 0:
587                 return True
588         return False
589
590     def _num_workers(self):
591         return int(self._options.child_processes)
592
593     def _run_tests(self, file_list, result_summary):
594         """Runs the tests in the file_list.
595
596         Return: A tuple (interrupted, keyboard_interrupted, thread_timings,
597             test_timings, individual_test_timings)
598             interrupted is whether the run was interrupted
599             keyboard_interrupted is whether the interruption was because someone
600               typed Ctrl^C
601             thread_timings is a list of dicts with the total runtime
602               of each thread with 'name', 'num_tests', 'total_time' properties
603             test_timings is a list of timings for each sharded subdirectory
604               of the form [time, directory_name, num_tests]
605             individual_test_timings is a list of run times for each test
606               in the form {filename:filename, test_run_time:test_run_time}
607             result_summary: summary object to populate with the results
608         """
609
610         self._printer.print_update('Sharding tests ...')
611         num_workers = self._num_workers()
612         test_lists = self._shard_tests(file_list,
613             num_workers > 1 and not self._options.experimental_fully_parallel)
614         filename_queue = Queue.Queue()
615         for item in test_lists:
616             filename_queue.put(item)
617
618         self._printer.print_update('Starting %s ...' %
619                                    grammar.pluralize('worker', num_workers))
620         message_broker = self._message_broker
621         self._current_filename_queue = filename_queue
622         self._current_result_summary = result_summary
623
624         if not self._options.dry_run:
625             threads = message_broker.start_workers(self)
626         else:
627             threads = {}
628
629         self._printer.print_update("Starting testing ...")
630         keyboard_interrupted = False
631         interrupted = False
632         if not self._options.dry_run:
633             try:
634                 message_broker.run_message_loop()
635             except KeyboardInterrupt:
636                 _log.info("Interrupted, exiting")
637                 message_broker.cancel_workers()
638                 keyboard_interrupted = True
639                 interrupted = True
640             except TestRunInterruptedException, e:
641                 _log.info(e.reason)
642                 message_broker.cancel_workers()
643                 interrupted = True
644             except:
645                 # Unexpected exception; don't try to clean up workers.
646                 _log.info("Exception raised, exiting")
647                 raise
648
649         thread_timings, test_timings, individual_test_timings = \
650             self._collect_timing_info(threads)
651
652         return (interrupted, keyboard_interrupted, thread_timings, test_timings,
653                 individual_test_timings)
654
655     def update(self):
656         self.update_summary(self._current_result_summary)
657
658     def _collect_timing_info(self, threads):
659         test_timings = {}
660         individual_test_timings = []
661         thread_timings = []
662
663         for thread in threads:
664             thread_timings.append({'name': thread.getName(),
665                                    'num_tests': thread.get_num_tests(),
666                                    'total_time': thread.get_total_time()})
667             test_timings.update(thread.get_test_group_timing_stats())
668             individual_test_timings.extend(thread.get_test_results())
669
670         return (thread_timings, test_timings, individual_test_timings)
671
672     def needs_http(self):
673         """Returns whether the test runner needs an HTTP server."""
674         return self._contains_tests(self.HTTP_SUBDIR)
675
676     def needs_websocket(self):
677         """Returns whether the test runner needs a WEBSOCKET server."""
678         return self._contains_tests(self.WEBSOCKET_SUBDIR)
679
680     def set_up_run(self):
681         """Configures the system to be ready to run tests.
682
683         Returns a ResultSummary object if we should continue to run tests,
684         or None if we should abort.
685
686         """
687         # This must be started before we check the system dependencies,
688         # since the helper may do things to make the setup correct.
689         self._printer.print_update("Starting helper ...")
690         self._port.start_helper()
691
692         # Check that the system dependencies (themes, fonts, ...) are correct.
693         if not self._options.nocheck_sys_deps:
694             self._printer.print_update("Checking system dependencies ...")
695             if not self._port.check_sys_deps(self.needs_http()):
696                 self._port.stop_helper()
697                 return None
698
699         if self._options.clobber_old_results:
700             self._clobber_old_results()
701
702         # Create the output directory if it doesn't already exist.
703         self._port.maybe_make_directory(self._options.results_directory)
704
705         self._port.setup_test_run()
706
707         self._printer.print_update("Preparing tests ...")
708         result_summary = self.prepare_lists_and_print_output()
709         if not result_summary:
710             return None
711
712         return result_summary
713
714     def run(self, result_summary):
715         """Run all our tests on all our test files.
716
717         For each test file, we run each test type. If there are any failures,
718         we collect them for reporting.
719
720         Args:
721           result_summary: a summary object tracking the test results.
722
723         Return:
724           The number of unexpected results (0 == success)
725         """
726         # gather_test_files() must have been called first to initialize us.
727         # If we didn't find any files to test, we've errored out already in
728         # prepare_lists_and_print_output().
729         assert(len(self._test_files))
730
731         start_time = time.time()
732
733         interrupted, keyboard_interrupted, thread_timings, test_timings, \
734             individual_test_timings = (
735             self._run_tests(self._test_files_list, result_summary))
736
737         # We exclude the crashes from the list of results to retry, because
738         # we want to treat even a potentially flaky crash as an error.
739         failures = self._get_failures(result_summary, include_crashes=False)
740         retry_summary = result_summary
741         while (len(failures) and self._options.retry_failures and
742             not self._retrying and not interrupted):
743             _log.info('')
744             _log.info("Retrying %d unexpected failure(s) ..." % len(failures))
745             _log.info('')
746             self._retrying = True
747             retry_summary = ResultSummary(self._expectations, failures.keys())
748             # Note that we intentionally ignore the return value here.
749             self._run_tests(failures.keys(), retry_summary)
750             failures = self._get_failures(retry_summary, include_crashes=True)
751
752         end_time = time.time()
753
754         self._print_timing_statistics(end_time - start_time,
755                                       thread_timings, test_timings,
756                                       individual_test_timings,
757                                       result_summary)
758
759         self._print_result_summary(result_summary)
760
761         sys.stdout.flush()
762         sys.stderr.flush()
763
764         self._printer.print_one_line_summary(result_summary.total,
765                                              result_summary.expected,
766                                              result_summary.unexpected)
767
768         unexpected_results = summarize_unexpected_results(self._port,
769             self._expectations, result_summary, retry_summary)
770         self._printer.print_unexpected_results(unexpected_results)
771
772         if (self._options.record_results and not self._options.dry_run and
773             not interrupted):
774             # Write the same data to log files and upload generated JSON files
775             # to appengine server.
776             self._upload_json_files(unexpected_results, result_summary,
777                                     individual_test_timings)
778
779         # Write the summary to disk (results.html) and display it if requested.
780         if not self._options.dry_run:
781             wrote_results = self._write_results_html_file(result_summary)
782             if self._options.show_results and wrote_results:
783                 self._show_results_html_file()
784
785         # Now that we've completed all the processing we can, we re-raise
786         # a KeyboardInterrupt if necessary so the caller can handle it.
787         if keyboard_interrupted:
788             raise KeyboardInterrupt
789
790         # Ignore flaky failures and unexpected passes so we don't turn the
791         # bot red for those.
792         return unexpected_results['num_regressions']
793
794     def clean_up_run(self):
795         """Restores the system after we're done running tests."""
796
797         _log.debug("flushing stdout")
798         sys.stdout.flush()
799         _log.debug("flushing stderr")
800         sys.stderr.flush()
801         _log.debug("stopping helper")
802         self._port.stop_helper()
803
804     def update_summary(self, result_summary):
805         """Update the summary and print results with any completed tests."""
806         while True:
807             try:
808                 result = test_results.TestResult.loads(self._result_queue.get_nowait())
809             except Queue.Empty:
810                 return
811
812             expected = self._expectations.matches_an_expected_result(
813                 result.filename, result.type, self._options.pixel_tests)
814             result_summary.add(result, expected)
815             exp_str = self._expectations.get_expectations_string(
816                 result.filename)
817             got_str = self._expectations.expectation_to_string(result.type)
818             self._printer.print_test_result(result, expected, exp_str, got_str)
819             self._printer.print_progress(result_summary, self._retrying,
820                                          self._test_files_list)
821
822             def interrupt_if_at_failure_limit(limit, count, message):
823                 if limit and count >= limit:
824                     raise TestRunInterruptedException(message % count)
825
826             interrupt_if_at_failure_limit(
827                 self._options.exit_after_n_failures,
828                 result_summary.unexpected_failures,
829                 "Aborting run since %d failures were reached")
830             interrupt_if_at_failure_limit(
831                 self._options.exit_after_n_crashes_or_timeouts,
832                 result_summary.unexpected_crashes_or_timeouts,
833                 "Aborting run since %d crashes or timeouts were reached")
834
835
836     def _clobber_old_results(self):
837         # Just clobber the actual test results directories since the other
838         # files in the results directory are explicitly used for cross-run
839         # tracking.
840         self._printer.print_update("Clobbering old results in %s" %
841                                    self._options.results_directory)
842         layout_tests_dir = self._port.layout_tests_dir()
843         possible_dirs = self._port.test_dirs()
844         for dirname in possible_dirs:
845             if os.path.isdir(os.path.join(layout_tests_dir, dirname)):
846                 shutil.rmtree(os.path.join(self._options.results_directory,
847                                            dirname),
848                               ignore_errors=True)
849
850     def _get_failures(self, result_summary, include_crashes):
851         """Filters a dict of results and returns only the failures.
852
853         Args:
854           result_summary: the results of the test run
855           include_crashes: whether crashes are included in the output.
856             We use False when finding the list of failures to retry
857             to see if the results were flaky. Although the crashes may also be
858             flaky, we treat them as if they aren't so that they're not ignored.
859         Returns:
860           a dict of files -> results
861         """
862         failed_results = {}
863         for test, result in result_summary.unexpected_results.iteritems():
864             if (result == test_expectations.PASS or
865                 result == test_expectations.CRASH and not include_crashes):
866                 continue
867             failed_results[test] = result
868
869         return failed_results
870
871     def _upload_json_files(self, unexpected_results, result_summary,
872                         individual_test_timings):
873         """Writes the results of the test run as JSON files into the results
874         dir and upload the files to the appengine server.
875
876         There are three different files written into the results dir:
877           unexpected_results.json: A short list of any unexpected results.
878             This is used by the buildbots to display results.
879           expectations.json: This is used by the flakiness dashboard.
880           results.json: A full list of the results - used by the flakiness
881             dashboard and the aggregate results dashboard.
882
883         Args:
884           unexpected_results: dict of unexpected results
885           result_summary: full summary object
886           individual_test_timings: list of test times (used by the flakiness
887             dashboard).
888         """
889         results_directory = self._options.results_directory
890         _log.debug("Writing JSON files in %s." % results_directory)
891         unexpected_json_path = os.path.join(results_directory, "unexpected_results.json")
892         with codecs.open(unexpected_json_path, "w", "utf-8") as file:
893             simplejson.dump(unexpected_results, file, sort_keys=True, indent=2)
894
895         # Write a json file of the test_expectations.txt file for the layout
896         # tests dashboard.
897         expectations_path = os.path.join(results_directory, "expectations.json")
898         expectations_json = \
899             self._expectations.get_expectations_json_for_all_platforms()
900         with codecs.open(expectations_path, "w", "utf-8") as file:
901             file.write(u"ADD_EXPECTATIONS(%s);" % expectations_json)
902
903         generator = json_layout_results_generator.JSONLayoutResultsGenerator(
904             self._port, self._options.builder_name, self._options.build_name,
905             self._options.build_number, self._options.results_directory,
906             BUILDER_BASE_URL, individual_test_timings,
907             self._expectations, result_summary, self._test_files_list,
908             not self._options.upload_full_results,
909             self._options.test_results_server,
910             "layout-tests",
911             self._options.master_name)
912
913         _log.debug("Finished writing JSON files.")
914
915         json_files = ["expectations.json"]
916         if self._options.upload_full_results:
917             json_files.append("results.json")
918         else:
919             json_files.append("incremental_results.json")
920
921         generator.upload_json_files(json_files)
922
923     def _print_config(self):
924         """Prints the configuration for the test run."""
925         p = self._printer
926         p.print_config("Using port '%s'" % self._port.name())
927         p.print_config("Placing test results in %s" %
928                        self._options.results_directory)
929         if self._options.new_baseline:
930             p.print_config("Placing new baselines in %s" %
931                            self._port.baseline_path())
932         p.print_config("Using %s build" % self._options.configuration)
933         if self._options.pixel_tests:
934             p.print_config("Pixel tests enabled")
935         else:
936             p.print_config("Pixel tests disabled")
937
938         p.print_config("Regular timeout: %s, slow test timeout: %s" %
939                        (self._options.time_out_ms,
940                         self._options.slow_time_out_ms))
941
942         if self._num_workers() == 1:
943             p.print_config("Running one %s" % self._port.driver_name())
944         else:
945             p.print_config("Running %s %ss in parallel" %
946                            (self._options.child_processes,
947                             self._port.driver_name()))
948         p.print_config('Command line: ' +
949                        ' '.join(self._port.driver_cmd_line()))
950         p.print_config("Worker model: %s" % self._options.worker_model)
951         p.print_config("")
952
953     def _print_expected_results_of_type(self, result_summary,
954                                         result_type, result_type_str):
955         """Print the number of the tests in a given result class.
956
957         Args:
958           result_summary - the object containing all the results to report on
959           result_type - the particular result type to report in the summary.
960           result_type_str - a string description of the result_type.
961         """
962         tests = self._expectations.get_tests_with_result_type(result_type)
963         now = result_summary.tests_by_timeline[test_expectations.NOW]
964         wontfix = result_summary.tests_by_timeline[test_expectations.WONTFIX]
965
966         # We use a fancy format string in order to print the data out in a
967         # nicely-aligned table.
968         fmtstr = ("Expect: %%5d %%-8s (%%%dd now, %%%dd wontfix)"
969                   % (self._num_digits(now), self._num_digits(wontfix)))
970         self._printer.print_expected(fmtstr %
971             (len(tests), result_type_str, len(tests & now), len(tests & wontfix)))
972
973     def _num_digits(self, num):
974         """Returns the number of digits needed to represent the length of a
975         sequence."""
976         ndigits = 1
977         if len(num):
978             ndigits = int(math.log10(len(num))) + 1
979         return ndigits
980
981     def _print_timing_statistics(self, total_time, thread_timings,
982                                directory_test_timings, individual_test_timings,
983                                result_summary):
984         """Record timing-specific information for the test run.
985
986         Args:
987           total_time: total elapsed time (in seconds) for the test run
988           thread_timings: wall clock time each thread ran for
989           directory_test_timings: timing by directory
990           individual_test_timings: timing by file
991           result_summary: summary object for the test run
992         """
993         self._printer.print_timing("Test timing:")
994         self._printer.print_timing("  %6.2f total testing time" % total_time)
995         self._printer.print_timing("")
996         self._printer.print_timing("Thread timing:")
997         cuml_time = 0
998         for t in thread_timings:
999             self._printer.print_timing("    %10s: %5d tests, %6.2f secs" %
1000                   (t['name'], t['num_tests'], t['total_time']))
1001             cuml_time += t['total_time']
1002         self._printer.print_timing("   %6.2f cumulative, %6.2f optimal" %
1003               (cuml_time, cuml_time / int(self._options.child_processes)))
1004         self._printer.print_timing("")
1005
1006         self._print_aggregate_test_statistics(individual_test_timings)
1007         self._print_individual_test_times(individual_test_timings,
1008                                           result_summary)
1009         self._print_directory_timings(directory_test_timings)
1010
1011     def _print_aggregate_test_statistics(self, individual_test_timings):
1012         """Prints aggregate statistics (e.g. median, mean, etc.) for all tests.
1013         Args:
1014           individual_test_timings: List of dump_render_tree_thread.TestStats
1015               for all tests.
1016         """
1017         test_types = []  # Unit tests don't actually produce any timings.
1018         if individual_test_timings:
1019             test_types = individual_test_timings[0].time_for_diffs.keys()
1020         times_for_dump_render_tree = []
1021         times_for_diff_processing = []
1022         times_per_test_type = {}
1023         for test_type in test_types:
1024             times_per_test_type[test_type] = []
1025
1026         for test_stats in individual_test_timings:
1027             times_for_dump_render_tree.append(test_stats.test_run_time)
1028             times_for_diff_processing.append(
1029                 test_stats.total_time_for_all_diffs)
1030             time_for_diffs = test_stats.time_for_diffs
1031             for test_type in test_types:
1032                 times_per_test_type[test_type].append(
1033                     time_for_diffs[test_type])
1034
1035         self._print_statistics_for_test_timings(
1036             "PER TEST TIME IN TESTSHELL (seconds):",
1037             times_for_dump_render_tree)
1038         self._print_statistics_for_test_timings(
1039             "PER TEST DIFF PROCESSING TIMES (seconds):",
1040             times_for_diff_processing)
1041         for test_type in test_types:
1042             self._print_statistics_for_test_timings(
1043                 "PER TEST TIMES BY TEST TYPE: %s" % test_type,
1044                 times_per_test_type[test_type])
1045
1046     def _print_individual_test_times(self, individual_test_timings,
1047                                   result_summary):
1048         """Prints the run times for slow, timeout and crash tests.
1049         Args:
1050           individual_test_timings: List of dump_render_tree_thread.TestStats
1051               for all tests.
1052           result_summary: summary object for test run
1053         """
1054         # Reverse-sort by the time spent in DumpRenderTree.
1055         individual_test_timings.sort(lambda a, b:
1056             cmp(b.test_run_time, a.test_run_time))
1057
1058         num_printed = 0
1059         slow_tests = []
1060         timeout_or_crash_tests = []
1061         unexpected_slow_tests = []
1062         for test_tuple in individual_test_timings:
1063             filename = test_tuple.filename
1064             is_timeout_crash_or_slow = False
1065             if self._test_is_slow(filename):
1066                 is_timeout_crash_or_slow = True
1067                 slow_tests.append(test_tuple)
1068
1069             if filename in result_summary.failures:
1070                 result = result_summary.results[filename].type
1071                 if (result == test_expectations.TIMEOUT or
1072                     result == test_expectations.CRASH):
1073                     is_timeout_crash_or_slow = True
1074                     timeout_or_crash_tests.append(test_tuple)
1075
1076             if (not is_timeout_crash_or_slow and
1077                 num_printed < printing.NUM_SLOW_TESTS_TO_LOG):
1078                 num_printed = num_printed + 1
1079                 unexpected_slow_tests.append(test_tuple)
1080
1081         self._printer.print_timing("")
1082         self._print_test_list_timing("%s slowest tests that are not "
1083             "marked as SLOW and did not timeout/crash:" %
1084             printing.NUM_SLOW_TESTS_TO_LOG, unexpected_slow_tests)
1085         self._printer.print_timing("")
1086         self._print_test_list_timing("Tests marked as SLOW:", slow_tests)
1087         self._printer.print_timing("")
1088         self._print_test_list_timing("Tests that timed out or crashed:",
1089                                      timeout_or_crash_tests)
1090         self._printer.print_timing("")
1091
1092     def _print_test_list_timing(self, title, test_list):
1093         """Print timing info for each test.
1094
1095         Args:
1096           title: section heading
1097           test_list: tests that fall in this section
1098         """
1099         if self._printer.disabled('slowest'):
1100             return
1101
1102         self._printer.print_timing(title)
1103         for test_tuple in test_list:
1104             filename = test_tuple.filename[len(
1105                 self._port.layout_tests_dir()) + 1:]
1106             filename = filename.replace('\\', '/')
1107             test_run_time = round(test_tuple.test_run_time, 1)
1108             self._printer.print_timing("  %s took %s seconds" %
1109                                        (filename, test_run_time))
1110
1111     def _print_directory_timings(self, directory_test_timings):
1112         """Print timing info by directory for any directories that
1113         take > 10 seconds to run.
1114
1115         Args:
1116           directory_test_timing: time info for each directory
1117         """
1118         timings = []
1119         for directory in directory_test_timings:
1120             num_tests, time_for_directory = directory_test_timings[directory]
1121             timings.append((round(time_for_directory, 1), directory,
1122                             num_tests))
1123         timings.sort()
1124
1125         self._printer.print_timing("Time to process slowest subdirectories:")
1126         min_seconds_to_print = 10
1127         for timing in timings:
1128             if timing[0] > min_seconds_to_print:
1129                 self._printer.print_timing(
1130                     "  %s took %s seconds to run %s tests." % (timing[1],
1131                     timing[0], timing[2]))
1132         self._printer.print_timing("")
1133
1134     def _print_statistics_for_test_timings(self, title, timings):
1135         """Prints the median, mean and standard deviation of the values in
1136         timings.
1137
1138         Args:
1139           title: Title for these timings.
1140           timings: A list of floats representing times.
1141         """
1142         self._printer.print_timing(title)
1143         timings.sort()
1144
1145         num_tests = len(timings)
1146         if not num_tests:
1147             return
1148         percentile90 = timings[int(.9 * num_tests)]
1149         percentile99 = timings[int(.99 * num_tests)]
1150
1151         if num_tests % 2 == 1:
1152             median = timings[((num_tests - 1) / 2) - 1]
1153         else:
1154             lower = timings[num_tests / 2 - 1]
1155             upper = timings[num_tests / 2]
1156             median = (float(lower + upper)) / 2
1157
1158         mean = sum(timings) / num_tests
1159
1160         for time in timings:
1161             sum_of_deviations = math.pow(time - mean, 2)
1162
1163         std_deviation = math.sqrt(sum_of_deviations / num_tests)
1164         self._printer.print_timing("  Median:          %6.3f" % median)
1165         self._printer.print_timing("  Mean:            %6.3f" % mean)
1166         self._printer.print_timing("  90th percentile: %6.3f" % percentile90)
1167         self._printer.print_timing("  99th percentile: %6.3f" % percentile99)
1168         self._printer.print_timing("  Standard dev:    %6.3f" % std_deviation)
1169         self._printer.print_timing("")
1170
1171     def _print_result_summary(self, result_summary):
1172         """Print a short summary about how many tests passed.
1173
1174         Args:
1175           result_summary: information to log
1176         """
1177         failed = len(result_summary.failures)
1178         skipped = len(
1179             result_summary.tests_by_expectation[test_expectations.SKIP])
1180         total = result_summary.total
1181         passed = total - failed - skipped
1182         pct_passed = 0.0
1183         if total > 0:
1184             pct_passed = float(passed) * 100 / total
1185
1186         self._printer.print_actual("")
1187         self._printer.print_actual("=> Results: %d/%d tests passed (%.1f%%)" %
1188                      (passed, total, pct_passed))
1189         self._printer.print_actual("")
1190         self._print_result_summary_entry(result_summary,
1191             test_expectations.NOW, "Tests to be fixed")
1192
1193         self._printer.print_actual("")
1194         self._print_result_summary_entry(result_summary,
1195             test_expectations.WONTFIX,
1196             "Tests that will only be fixed if they crash (WONTFIX)")
1197         self._printer.print_actual("")
1198
1199     def _print_result_summary_entry(self, result_summary, timeline,
1200                                     heading):
1201         """Print a summary block of results for a particular timeline of test.
1202
1203         Args:
1204           result_summary: summary to print results for
1205           timeline: the timeline to print results for (NOT, WONTFIX, etc.)
1206           heading: a textual description of the timeline
1207         """
1208         total = len(result_summary.tests_by_timeline[timeline])
1209         not_passing = (total -
1210            len(result_summary.tests_by_expectation[test_expectations.PASS] &
1211                result_summary.tests_by_timeline[timeline]))
1212         self._printer.print_actual("=> %s (%d):" % (heading, not_passing))
1213
1214         for result in TestExpectationsFile.EXPECTATION_ORDER:
1215             if result == test_expectations.PASS:
1216                 continue
1217             results = (result_summary.tests_by_expectation[result] &
1218                        result_summary.tests_by_timeline[timeline])
1219             desc = TestExpectationsFile.EXPECTATION_DESCRIPTIONS[result]
1220             if not_passing and len(results):
1221                 pct = len(results) * 100.0 / not_passing
1222                 self._printer.print_actual("  %5d %-24s (%4.1f%%)" %
1223                     (len(results), desc[len(results) != 1], pct))
1224
1225     def _results_html(self, test_files, failures, title="Test Failures", override_time=None):
1226         """
1227         test_files = a list of file paths
1228         failures = dictionary mapping test paths to failure objects
1229         title = title printed at top of test
1230         override_time = current time (used by unit tests)
1231         """
1232         page = """<html>
1233   <head>
1234     <title>Layout Test Results (%(time)s)</title>
1235   </head>
1236   <body>
1237     <h2>%(title)s (%(time)s)</h2>
1238         """ % {'title': title, 'time': override_time or time.asctime()}
1239
1240         for test_file in sorted(test_files):
1241             test_name = self._port.relative_test_filename(test_file)
1242             test_url = self._port.filename_to_uri(test_file)
1243             page += u"<p><a href='%s'>%s</a><br />\n" % (test_url, test_name)
1244             test_failures = failures.get(test_file, [])
1245             for failure in test_failures:
1246                 page += (u"&nbsp;&nbsp;%s<br/>" %
1247                          failure.result_html_output(test_name))
1248             page += "</p>\n"
1249         page += "</body></html>\n"
1250         return page
1251
1252     def _write_results_html_file(self, result_summary):
1253         """Write results.html which is a summary of tests that failed.
1254
1255         Args:
1256           result_summary: a summary of the results :)
1257
1258         Returns:
1259           True if any results were written (since expected failures may be
1260           omitted)
1261         """
1262         # test failures
1263         if self._options.full_results_html:
1264             results_title = "Test Failures"
1265             test_files = result_summary.failures.keys()
1266         else:
1267             results_title = "Unexpected Test Failures"
1268             unexpected_failures = self._get_failures(result_summary,
1269                 include_crashes=True)
1270             test_files = unexpected_failures.keys()
1271         if not len(test_files):
1272             return False
1273
1274         out_filename = os.path.join(self._options.results_directory,
1275                                     "results.html")
1276         with codecs.open(out_filename, "w", "utf-8") as results_file:
1277             html = self._results_html(test_files, result_summary.failures, results_title)
1278             results_file.write(html)
1279
1280         return True
1281
1282     def _show_results_html_file(self):
1283         """Shows the results.html page."""
1284         results_filename = os.path.join(self._options.results_directory,
1285                                         "results.html")
1286         self._port.show_results_html_file(results_filename)
1287
1288
1289 def read_test_files(files):
1290     tests = []
1291     for file in files:
1292         try:
1293             with codecs.open(file, 'r', 'utf-8') as file_contents:
1294                 # FIXME: This could be cleaner using a list comprehension.
1295                 for line in file_contents:
1296                     line = test_expectations.strip_comments(line)
1297                     if line:
1298                         tests.append(line)
1299         except IOError, e:
1300             if e.errno == errno.ENOENT:
1301                 _log.critical('')
1302                 _log.critical('--test-list file "%s" not found' % file)
1303             raise
1304     return tests
1305
1306
1307 def run(port, options, args, regular_output=sys.stderr,
1308         buildbot_output=sys.stdout):
1309     """Run the tests.
1310
1311     Args:
1312       port: Port object for port-specific behavior
1313       options: a dictionary of command line options
1314       args: a list of sub directories or files to test
1315       regular_output: a stream-like object that we can send logging/debug
1316           output to
1317       buildbot_output: a stream-like object that we can write all output that
1318           is intended to be parsed by the buildbot to
1319     Returns:
1320       the number of unexpected results that occurred, or -1 if there is an
1321           error.
1322
1323     """
1324     _set_up_derived_options(port, options)
1325
1326     printer = printing.Printer(port, options, regular_output, buildbot_output,
1327         int(options.child_processes), options.experimental_fully_parallel)
1328     if options.help_printing:
1329         printer.help_printing()
1330         printer.cleanup()
1331         return 0
1332
1333     last_unexpected_results = _gather_unexpected_results(options)
1334     if options.print_last_failures:
1335         printer.write("\n".join(last_unexpected_results) + "\n")
1336         printer.cleanup()
1337         return 0
1338
1339     broker = message_broker.get(port, options)
1340
1341     # We wrap any parts of the run that are slow or likely to raise exceptions
1342     # in a try/finally to ensure that we clean up the logging configuration.
1343     num_unexpected_results = -1
1344     try:
1345         test_runner = TestRunner(port, options, printer, broker)
1346         test_runner._print_config()
1347
1348         printer.print_update("Collecting tests ...")
1349         try:
1350             test_runner.collect_tests(args, last_unexpected_results)
1351         except IOError, e:
1352             if e.errno == errno.ENOENT:
1353                 return -1
1354             raise
1355
1356         printer.print_update("Parsing expectations ...")
1357         if options.lint_test_files:
1358             return test_runner.lint()
1359         test_runner.parse_expectations(port.test_platform_name(),
1360                                        options.configuration == 'Debug')
1361
1362         printer.print_update("Checking build ...")
1363         if not port.check_build(test_runner.needs_http()):
1364             _log.error("Build check failed")
1365             return -1
1366
1367         result_summary = test_runner.set_up_run()
1368         if result_summary:
1369             num_unexpected_results = test_runner.run(result_summary)
1370             test_runner.clean_up_run()
1371             _log.debug("Testing completed, Exit status: %d" %
1372                        num_unexpected_results)
1373     finally:
1374         broker.cleanup()
1375         printer.cleanup()
1376
1377     return num_unexpected_results
1378
1379
1380 def _set_up_derived_options(port_obj, options):
1381     """Sets the options values that depend on other options values."""
1382
1383     if options.worker_model == 'inline':
1384         if options.child_processes and int(options.child_processes) > 1:
1385             _log.warning("--worker-model=inline overrides --child-processes")
1386         options.child_processes = "1"
1387     if not options.child_processes:
1388         options.child_processes = os.environ.get("WEBKIT_TEST_CHILD_PROCESSES",
1389                                                  str(port_obj.default_child_processes()))
1390
1391     if not options.configuration:
1392         options.configuration = port_obj.default_configuration()
1393
1394     if options.pixel_tests is None:
1395         options.pixel_tests = True
1396
1397     if not options.use_apache:
1398         options.use_apache = sys.platform in ('darwin', 'linux2')
1399
1400     if not os.path.isabs(options.results_directory):
1401         # This normalizes the path to the build dir.
1402         # FIXME: how this happens is not at all obvious; this is a dumb
1403         # interface and should be cleaned up.
1404         options.results_directory = port_obj.results_directory()
1405
1406     if not options.time_out_ms:
1407         if options.configuration == "Debug":
1408             options.time_out_ms = str(2 * TestRunner.DEFAULT_TEST_TIMEOUT_MS)
1409         else:
1410             options.time_out_ms = str(TestRunner.DEFAULT_TEST_TIMEOUT_MS)
1411
1412     options.slow_time_out_ms = str(5 * int(options.time_out_ms))
1413
1414
1415 def _gather_unexpected_results(options):
1416     """Returns the unexpected results from the previous run, if any."""
1417     last_unexpected_results = []
1418     if options.print_last_failures or options.retest_last_failures:
1419         unexpected_results_filename = os.path.join(
1420         options.results_directory, "unexpected_results.json")
1421         with codecs.open(unexpected_results_filename, "r", "utf-8") as file:
1422             results = simplejson.load(file)
1423         last_unexpected_results = results['tests'].keys()
1424     return last_unexpected_results
1425
1426
1427 def _compat_shim_callback(option, opt_str, value, parser):
1428     print "Ignoring unsupported option: %s" % opt_str
1429
1430
1431 def _compat_shim_option(option_name, **kwargs):
1432     return optparse.make_option(option_name, action="callback",
1433         callback=_compat_shim_callback,
1434         help="Ignored, for old-run-webkit-tests compat only.", **kwargs)
1435
1436
1437 def parse_args(args=None):
1438     """Provides a default set of command line args.
1439
1440     Returns a tuple of options, args from optparse"""
1441
1442     # FIXME: All of these options should be stored closer to the code which
1443     # FIXME: actually uses them. configuration_options should move
1444     # FIXME: to WebKitPort and be shared across all scripts.
1445     configuration_options = [
1446         optparse.make_option("-t", "--target", dest="configuration",
1447                              help="(DEPRECATED)"),
1448         # FIXME: --help should display which configuration is default.
1449         optparse.make_option('--debug', action='store_const', const='Debug',
1450                              dest="configuration",
1451                              help='Set the configuration to Debug'),
1452         optparse.make_option('--release', action='store_const',
1453                              const='Release', dest="configuration",
1454                              help='Set the configuration to Release'),
1455         # old-run-webkit-tests also accepts -c, --configuration CONFIGURATION.
1456     ]
1457
1458     print_options = printing.print_options()
1459
1460     # FIXME: These options should move onto the ChromiumPort.
1461     chromium_options = [
1462         optparse.make_option("--chromium", action="store_true", default=False,
1463             help="use the Chromium port"),
1464         optparse.make_option("--startup-dialog", action="store_true",
1465             default=False, help="create a dialog on DumpRenderTree startup"),
1466         optparse.make_option("--gp-fault-error-box", action="store_true",
1467             default=False, help="enable Windows GP fault error box"),
1468         optparse.make_option("--multiple-loads",
1469             type="int", help="turn on multiple loads of each test"),
1470         optparse.make_option("--js-flags",
1471             type="string", help="JavaScript flags to pass to tests"),
1472         optparse.make_option("--nocheck-sys-deps", action="store_true",
1473             default=False,
1474             help="Don't check the system dependencies (themes)"),
1475         optparse.make_option("--use-test-shell", action="store_true",
1476             default=False,
1477             help="Use test_shell instead of DRT"),
1478         optparse.make_option("--accelerated-compositing",
1479             action="store_true",
1480             help="Use hardware-accelated compositing for rendering"),
1481         optparse.make_option("--no-accelerated-compositing",
1482             action="store_false",
1483             dest="accelerated_compositing",
1484             help="Don't use hardware-accelerated compositing for rendering"),
1485         optparse.make_option("--accelerated-2d-canvas",
1486             action="store_true",
1487             help="Use hardware-accelerated 2D Canvas calls"),
1488         optparse.make_option("--no-accelerated-2d-canvas",
1489             action="store_false",
1490             dest="accelerated_2d_canvas",
1491             help="Don't use hardware-accelerated 2D Canvas calls"),
1492     ]
1493
1494     # Missing Mac-specific old-run-webkit-tests options:
1495     # FIXME: Need: -g, --guard for guard malloc support on Mac.
1496     # FIXME: Need: -l --leaks    Enable leaks checking.
1497     # FIXME: Need: --sample-on-timeout Run sample on timeout
1498
1499     old_run_webkit_tests_compat = [
1500         # NRWT doesn't generate results by default anyway.
1501         _compat_shim_option("--no-new-test-results"),
1502         # NRWT doesn't sample on timeout yet anyway.
1503         _compat_shim_option("--no-sample-on-timeout"),
1504         # FIXME: NRWT needs to support remote links eventually.
1505         _compat_shim_option("--use-remote-links-to-tests"),
1506     ]
1507
1508     results_options = [
1509         # NEED for bots: --use-remote-links-to-tests Link to test files
1510         # within the SVN repository in the results.
1511         optparse.make_option("-p", "--pixel-tests", action="store_true",
1512             dest="pixel_tests", help="Enable pixel-to-pixel PNG comparisons"),
1513         optparse.make_option("--no-pixel-tests", action="store_false",
1514             dest="pixel_tests", help="Disable pixel-to-pixel PNG comparisons"),
1515         optparse.make_option("--tolerance",
1516             help="Ignore image differences less than this percentage (some "
1517                 "ports may ignore this option)", type="float"),
1518         optparse.make_option("--results-directory",
1519             default="layout-test-results",
1520             help="Output results directory source dir, relative to Debug or "
1521                  "Release"),
1522         optparse.make_option("--new-baseline", action="store_true",
1523             default=False, help="Save all generated results as new baselines "
1524                  "into the platform directory, overwriting whatever's "
1525                  "already there."),
1526         optparse.make_option("--reset-results", action="store_true",
1527             default=False, help="Reset any existing baselines to the "
1528                  "generated results"),
1529         optparse.make_option("--no-show-results", action="store_false",
1530             default=True, dest="show_results",
1531             help="Don't launch a browser with results after the tests "
1532                  "are done"),
1533         # FIXME: We should have a helper function to do this sort of
1534         # deprectated mapping and automatically log, etc.
1535         optparse.make_option("--noshow-results", action="store_false",
1536             dest="show_results",
1537             help="Deprecated, same as --no-show-results."),
1538         optparse.make_option("--no-launch-safari", action="store_false",
1539             dest="show_results",
1540             help="old-run-webkit-tests compat, same as --noshow-results."),
1541         # old-run-webkit-tests:
1542         # --[no-]launch-safari    Launch (or do not launch) Safari to display
1543         #                         test results (default: launch)
1544         optparse.make_option("--full-results-html", action="store_true",
1545             default=False,
1546             help="Show all failures in results.html, rather than only "
1547                  "regressions"),
1548         optparse.make_option("--clobber-old-results", action="store_true",
1549             default=False, help="Clobbers test results from previous runs."),
1550         optparse.make_option("--platform",
1551             help="Override the platform for expected results"),
1552         optparse.make_option("--no-record-results", action="store_false",
1553             default=True, dest="record_results",
1554             help="Don't record the results."),
1555         # old-run-webkit-tests also has HTTP toggle options:
1556         # --[no-]http                     Run (or do not run) http tests
1557         #                                 (default: run)
1558     ]
1559
1560     test_options = [
1561         optparse.make_option("--build", dest="build",
1562             action="store_true", default=True,
1563             help="Check to ensure the DumpRenderTree build is up-to-date "
1564                  "(default)."),
1565         optparse.make_option("--no-build", dest="build",
1566             action="store_false", help="Don't check to see if the "
1567                                        "DumpRenderTree build is up-to-date."),
1568         optparse.make_option("-n", "--dry-run", action="store_true",
1569             default=False,
1570             help="Do everything but actually run the tests or upload results."),
1571         # old-run-webkit-tests has --valgrind instead of wrapper.
1572         optparse.make_option("--wrapper",
1573             help="wrapper command to insert before invocations of "
1574                  "DumpRenderTree; option is split on whitespace before "
1575                  "running. (Example: --wrapper='valgrind --smc-check=all')"),
1576         # old-run-webkit-tests:
1577         # -i|--ignore-tests               Comma-separated list of directories
1578         #                                 or tests to ignore
1579         optparse.make_option("--test-list", action="append",
1580             help="read list of tests to run from file", metavar="FILE"),
1581         # old-run-webkit-tests uses --skipped==[default|ignore|only]
1582         # instead of --force:
1583         optparse.make_option("--force", action="store_true", default=False,
1584             help="Run all tests, even those marked SKIP in the test list"),
1585         optparse.make_option("--use-apache", action="store_true",
1586             default=False, help="Whether to use apache instead of lighttpd."),
1587         optparse.make_option("--time-out-ms",
1588             help="Set the timeout for each test"),
1589         # old-run-webkit-tests calls --randomize-order --random:
1590         optparse.make_option("--randomize-order", action="store_true",
1591             default=False, help=("Run tests in random order (useful "
1592                                 "for tracking down corruption)")),
1593         optparse.make_option("--run-chunk",
1594             help=("Run a specified chunk (n:l), the nth of len l, "
1595                  "of the layout tests")),
1596         optparse.make_option("--run-part", help=("Run a specified part (n:m), "
1597                   "the nth of m parts, of the layout tests")),
1598         # old-run-webkit-tests calls --batch-size: --nthly n
1599         #   Restart DumpRenderTree every n tests (default: 1000)
1600         optparse.make_option("--batch-size",
1601             help=("Run a the tests in batches (n), after every n tests, "
1602                   "DumpRenderTree is relaunched."), type="int", default=0),
1603         # old-run-webkit-tests calls --run-singly: -1|--singly
1604         # Isolate each test case run (implies --nthly 1 --verbose)
1605         optparse.make_option("--run-singly", action="store_true",
1606             default=False, help="run a separate DumpRenderTree for each test"),
1607         optparse.make_option("--child-processes",
1608             help="Number of DumpRenderTrees to run in parallel."),
1609         # FIXME: Display default number of child processes that will run.
1610         optparse.make_option("--worker-model", action="store",
1611             default="threads", help=("controls worker model. Valid values are "
1612             "'inline' and 'threads' (default).")),
1613         optparse.make_option("--experimental-fully-parallel",
1614             action="store_true", default=False,
1615             help="run all tests in parallel"),
1616         optparse.make_option("--exit-after-n-failures", type="int", nargs=1,
1617             help="Exit after the first N failures instead of running all "
1618             "tests"),
1619         optparse.make_option("--exit-after-n-crashes-or-timeouts", type="int",
1620             nargs=1, help="Exit after the first N crashes instead of running "
1621             "all tests"),
1622         # FIXME: consider: --iterations n
1623         #      Number of times to run the set of tests (e.g. ABCABCABC)
1624         optparse.make_option("--print-last-failures", action="store_true",
1625             default=False, help="Print the tests in the last run that "
1626             "had unexpected failures (or passes) and then exit."),
1627         optparse.make_option("--retest-last-failures", action="store_true",
1628             default=False, help="re-test the tests in the last run that "
1629             "had unexpected failures (or passes)."),
1630         optparse.make_option("--retry-failures", action="store_true",
1631             default=True,
1632             help="Re-try any tests that produce unexpected results (default)"),
1633         optparse.make_option("--no-retry-failures", action="store_false",
1634             dest="retry_failures",
1635             help="Don't re-try any tests that produce unexpected results."),
1636     ]
1637
1638     misc_options = [
1639         optparse.make_option("--lint-test-files", action="store_true",
1640         default=False, help=("Makes sure the test files parse for all "
1641                             "configurations. Does not run any tests.")),
1642     ]
1643
1644     # FIXME: Move these into json_results_generator.py
1645     results_json_options = [
1646         optparse.make_option("--master-name", help="The name of the buildbot master."),
1647         optparse.make_option("--builder-name", default="DUMMY_BUILDER_NAME",
1648             help=("The name of the builder shown on the waterfall running "
1649                   "this script e.g. WebKit.")),
1650         optparse.make_option("--build-name", default="DUMMY_BUILD_NAME",
1651             help=("The name of the builder used in its path, e.g. "
1652                   "webkit-rel.")),
1653         optparse.make_option("--build-number", default="DUMMY_BUILD_NUMBER",
1654             help=("The build number of the builder running this script.")),
1655         optparse.make_option("--test-results-server", default="",
1656             help=("If specified, upload results json files to this appengine "
1657                   "server.")),
1658         optparse.make_option("--upload-full-results",
1659             action="store_true",
1660             default=False,
1661             help="If true, upload full json results to server."),
1662     ]
1663
1664     option_list = (configuration_options + print_options +
1665                    chromium_options + results_options + test_options +
1666                    misc_options + results_json_options +
1667                    old_run_webkit_tests_compat)
1668     option_parser = optparse.OptionParser(option_list=option_list)
1669
1670     return option_parser.parse_args(args)
1671
1672
1673 def main():
1674     options, args = parse_args()
1675     port_obj = port.get(options.platform, options)
1676     return run(port_obj, options, args)
1677
1678
1679 if '__main__' == __name__:
1680     try:
1681         sys.exit(main())
1682     except KeyboardInterrupt:
1683         # this mirrors what the shell normally does
1684         sys.exit(signal.SIGINT + 128)