WebKitTools/Scripts/webkitpy/layout_tests/run_webkit_tests.py

   1 #!/usr/bin/env python
   2 # Copyright (C) 2010 Google Inc. All rights reserved.
   3 # Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Szeged
   4 #
   5 # Redistribution and use in source and binary forms, with or without
   6 # modification, are permitted provided that the following conditions are
   7 # met:
   8 #
   9 #     * Redistributions of source code must retain the above copyright
  10 # notice, this list of conditions and the following disclaimer.
  11 #     * Redistributions in binary form must reproduce the above
  12 # copyright notice, this list of conditions and the following disclaimer
  13 # in the documentation and/or other materials provided with the
  14 # distribution.
  15 #     * Neither the name of Google Inc. nor the names of its
  16 # contributors may be used to endorse or promote products derived from
  17 # this software without specific prior written permission.
  18 #
  19 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  20 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  21 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  22 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  23 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  24 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  25 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  26 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  27 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  28 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  29 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  30
  31 """Run layout tests.
  32
  33 This is a port of the existing webkit test script run-webkit-tests.
  34
  35 The TestRunner class runs a series of tests (TestType interface) against a set
  36 of test files.  If a test file fails a TestType, it returns a list TestFailure
  37 objects to the TestRunner.  The TestRunner then aggregates the TestFailures to
  38 create a final report.
  39
  40 This script reads several files, if they exist in the test_lists subdirectory
  41 next to this script itself.  Each should contain a list of paths to individual
  42 tests or entire subdirectories of tests, relative to the outermost test
  43 directory.  Entire lines starting with '//' (comments) will be ignored.
  44
  45 For details of the files' contents and purposes, see test_lists/README.
  46 """
  47
  48 from __future__ import with_statement
  49
  50 import codecs
  51 import errno
  52 import glob
  53 import logging
  54 import math
  55 import optparse
  56 import os
  57 import platform
  58 import Queue
  59 import random
  60 import re
  61 import shutil
  62 import signal
  63 import sys
  64 import time
  65 import traceback
  66
  67 from layout_package import dump_render_tree_thread
  68 from layout_package import json_layout_results_generator
  69 from layout_package import message_broker
  70 from layout_package import printing
  71 from layout_package import test_expectations
  72 from layout_package import test_failures
  73 from layout_package import test_results
  74 from layout_package import test_results_uploader
  75
  76 from webkitpy.common.system import user
  77 from webkitpy.thirdparty import simplejson
  78 from webkitpy.tool import grammar
  79
  80 import port
  81
  82 _log = logging.getLogger("webkitpy.layout_tests.run_webkit_tests")
  83
  84 # Builder base URL where we have the archived test results.
  85 BUILDER_BASE_URL = "http://build.chromium.org/buildbot/layout_test_results/"
  86
  87 LAYOUT_TESTS_DIRECTORY = "LayoutTests" + os.sep
  88
  89 TestExpectationsFile = test_expectations.TestExpectationsFile
  90
  91
  92 class TestInput:
  93     """Groups information about a test for easy passing of data."""
  94
  95     def __init__(self, filename, timeout):
  96         """Holds the input parameters for a test.
  97         Args:
  98           filename: Full path to the test.
  99           timeout: Timeout in msecs the driver should use while running the test
 100           """
 101         # FIXME: filename should really be test_name as a relative path.
 102         self.filename = filename
 103         self.timeout = timeout
 104         # The image_hash is used to avoid doing an image dump if the
 105         # checksums match. The image_hash is set later, and only if it is needed
 106         # for the test.
 107         self.image_hash = None
 108
 109
 110 class ResultSummary(object):
 111     """A class for partitioning the test results we get into buckets.
 112
 113     This class is basically a glorified struct and it's private to this file
 114     so we don't bother with any information hiding."""
 115
 116     def __init__(self, expectations, test_files):
 117         self.total = len(test_files)
 118         self.remaining = self.total
 119         self.expectations = expectations
 120         self.expected = 0
 121         self.unexpected = 0
 122         self.tests_by_expectation = {}
 123         self.tests_by_timeline = {}
 124         self.results = {}
 125         self.unexpected_results = {}
 126         self.failures = {}
 127         self.tests_by_expectation[test_expectations.SKIP] = set()
 128         for expectation in TestExpectationsFile.EXPECTATIONS.values():
 129             self.tests_by_expectation[expectation] = set()
 130         for timeline in TestExpectationsFile.TIMELINES.values():
 131             self.tests_by_timeline[timeline] = (
 132                 expectations.get_tests_with_timeline(timeline))
 133
 134     def add(self, result, expected):
 135         """Add a TestResult into the appropriate bin.
 136
 137         Args:
 138           result: TestResult from dump_render_tree_thread.
 139           expected: whether the result was what we expected it to be.
 140         """
 141
 142         self.tests_by_expectation[result.type].add(result.filename)
 143         self.results[result.filename] = result
 144         self.remaining -= 1
 145         if len(result.failures):
 146             self.failures[result.filename] = result.failures
 147         if expected:
 148             self.expected += 1
 149         else:
 150             self.unexpected_results[result.filename] = result.type
 151             self.unexpected += 1
 152
 153
 154 def summarize_unexpected_results(port_obj, expectations, result_summary,
 155                                  retry_summary):
 156     """Summarize any unexpected results as a dict.
 157
 158     FIXME: split this data structure into a separate class?
 159
 160     Args:
 161         port_obj: interface to port-specific hooks
 162         expectations: test_expectations.TestExpectations object
 163         result_summary: summary object from initial test runs
 164         retry_summary: summary object from final test run of retried tests
 165     Returns:
 166         A dictionary containing a summary of the unexpected results from the
 167         run, with the following fields:
 168         'version': a version indicator (1 in this version)
 169         'fixable': # of fixable tests (NOW - PASS)
 170         'skipped': # of skipped tests (NOW & SKIPPED)
 171         'num_regressions': # of non-flaky failures
 172         'num_flaky': # of flaky failures
 173         'num_passes': # of unexpected passes
 174         'tests': a dict of tests -> {'expected': '...', 'actual': '...'}
 175     """
 176     results = {}
 177     results['version'] = 1
 178
 179     tbe = result_summary.tests_by_expectation
 180     tbt = result_summary.tests_by_timeline
 181     results['fixable'] = len(tbt[test_expectations.NOW] -
 182                                 tbe[test_expectations.PASS])
 183     results['skipped'] = len(tbt[test_expectations.NOW] &
 184                                 tbe[test_expectations.SKIP])
 185
 186     num_passes = 0
 187     num_flaky = 0
 188     num_regressions = 0
 189     keywords = {}
 190     for k, v in TestExpectationsFile.EXPECTATIONS.iteritems():
 191         keywords[v] = k.upper()
 192
 193     tests = {}
 194     for filename, result in result_summary.unexpected_results.iteritems():
 195         # Note that if a test crashed in the original run, we ignore
 196         # whether or not it crashed when we retried it (if we retried it),
 197         # and always consider the result not flaky.
 198         test = port_obj.relative_test_filename(filename)
 199         expected = expectations.get_expectations_string(filename)
 200         actual = [keywords[result]]
 201
 202         if result == test_expectations.PASS:
 203             num_passes += 1
 204         elif result == test_expectations.CRASH:
 205             num_regressions += 1
 206         else:
 207             if filename not in retry_summary.unexpected_results:
 208                 actual.extend(expectations.get_expectations_string(
 209                     filename).split(" "))
 210                 num_flaky += 1
 211             else:
 212                 retry_result = retry_summary.unexpected_results[filename]
 213                 if result != retry_result:
 214                     actual.append(keywords[retry_result])
 215                     num_flaky += 1
 216                 else:
 217                     num_regressions += 1
 218
 219         tests[test] = {}
 220         tests[test]['expected'] = expected
 221         tests[test]['actual'] = " ".join(actual)
 222
 223     results['tests'] = tests
 224     results['num_passes'] = num_passes
 225     results['num_flaky'] = num_flaky
 226     results['num_regressions'] = num_regressions
 227
 228     return results
 229
 230
 231 class TestRunner:
 232     """A class for managing running a series of tests on a series of layout
 233     test files."""
 234
 235     HTTP_SUBDIR = os.sep.join(['', 'http', ''])
 236     WEBSOCKET_SUBDIR = os.sep.join(['', 'websocket', ''])
 237
 238     # The per-test timeout in milliseconds, if no --time-out-ms option was
 239     # given to run_webkit_tests. This should correspond to the default timeout
 240     # in DumpRenderTree.
 241     DEFAULT_TEST_TIMEOUT_MS = 6 * 1000
 242
 243     def __init__(self, port, options, printer, message_broker):
 244         """Initialize test runner data structures.
 245
 246         Args:
 247           port: an object implementing port-specific
 248           options: a dictionary of command line options
 249           printer: a Printer object to record updates to.
 250           message_broker: object used to communicate with workers.
 251         """
 252         self._port = port
 253         self._options = options
 254         self._printer = printer
 255         self._message_broker = message_broker
 256
 257         # disable wss server. need to install pyOpenSSL on buildbots.
 258         # self._websocket_secure_server = websocket_server.PyWebSocket(
 259         #        options.results_directory, use_tls=True, port=9323)
 260
 261         # a set of test files, and the same tests as a list
 262         self._test_files = set()
 263         self._test_files_list = None
 264         self._result_queue = Queue.Queue()
 265         self._retrying = False
 266
 267     def collect_tests(self, args, last_unexpected_results):
 268         """Find all the files to test.
 269
 270         Args:
 271           args: list of test arguments from the command line
 272           last_unexpected_results: list of unexpected results to retest, if any
 273
 274         """
 275         paths = [self._strip_test_dir_prefix(arg) for arg in args if arg and arg != '']
 276         paths += last_unexpected_results
 277         if self._options.test_list:
 278             paths += read_test_files(self._options.test_list)
 279         self._test_files = self._port.tests(paths)
 280
 281     def _strip_test_dir_prefix(self, path):
 282         if path.startswith(LAYOUT_TESTS_DIRECTORY):
 283             return path[len(LAYOUT_TESTS_DIRECTORY):]
 284         return path
 285
 286     def lint(self):
 287         # Creating the expecations for each platform/configuration pair does
 288         # all the test list parsing and ensures it's correct syntax (e.g. no
 289         # dupes).
 290         for platform_name in self._port.test_platform_names():
 291             self.parse_expectations(platform_name, is_debug_mode=True)
 292             self.parse_expectations(platform_name, is_debug_mode=False)
 293         self._printer.write("")
 294         _log.info("If there are no fail messages, errors or exceptions, "
 295                   "then the lint succeeded.")
 296         return 0
 297
 298     def parse_expectations(self, test_platform_name, is_debug_mode):
 299         """Parse the expectations from the test_list files and return a data
 300         structure holding them. Throws an error if the test_list files have
 301         invalid syntax."""
 302         if self._options.lint_test_files:
 303             test_files = None
 304         else:
 305             test_files = self._test_files
 306
 307         try:
 308             expectations_str = self._port.test_expectations()
 309             overrides_str = self._port.test_expectations_overrides()
 310             self._expectations = test_expectations.TestExpectations(
 311                 self._port, test_files, expectations_str, test_platform_name,
 312                 is_debug_mode, self._options.lint_test_files,
 313                 overrides=overrides_str)
 314             return self._expectations
 315         except SyntaxError, err:
 316             if self._options.lint_test_files:
 317                 print str(err)
 318             else:
 319                 raise err
 320
 321     def prepare_lists_and_print_output(self):
 322         """Create appropriate subsets of test lists and returns a
 323         ResultSummary object. Also prints expected test counts.
 324         """
 325
 326         # Remove skipped - both fixable and ignored - files from the
 327         # top-level list of files to test.
 328         num_all_test_files = len(self._test_files)
 329         self._printer.print_expected("Found:  %d tests" %
 330                                      (len(self._test_files)))
 331         if not num_all_test_files:
 332             _log.critical('No tests to run.')
 333             return None
 334
 335         skipped = set()
 336         if num_all_test_files > 1 and not self._options.force:
 337             skipped = self._expectations.get_tests_with_result_type(
 338                            test_expectations.SKIP)
 339             self._test_files -= skipped
 340
 341         # Create a sorted list of test files so the subset chunk,
 342         # if used, contains alphabetically consecutive tests.
 343         self._test_files_list = list(self._test_files)
 344         if self._options.randomize_order:
 345             random.shuffle(self._test_files_list)
 346         else:
 347             self._test_files_list.sort()
 348
 349         # If the user specifies they just want to run a subset of the tests,
 350         # just grab a subset of the non-skipped tests.
 351         if self._options.run_chunk or self._options.run_part:
 352             chunk_value = self._options.run_chunk or self._options.run_part
 353             test_files = self._test_files_list
 354             try:
 355                 (chunk_num, chunk_len) = chunk_value.split(":")
 356                 chunk_num = int(chunk_num)
 357                 assert(chunk_num >= 0)
 358                 test_size = int(chunk_len)
 359                 assert(test_size > 0)
 360             except:
 361                 _log.critical("invalid chunk '%s'" % chunk_value)
 362                 return None
 363
 364             # Get the number of tests
 365             num_tests = len(test_files)
 366
 367             # Get the start offset of the slice.
 368             if self._options.run_chunk:
 369                 chunk_len = test_size
 370                 # In this case chunk_num can be really large. We need
 371                 # to make the slave fit in the current number of tests.
 372                 slice_start = (chunk_num * chunk_len) % num_tests
 373             else:
 374                 # Validate the data.
 375                 assert(test_size <= num_tests)
 376                 assert(chunk_num <= test_size)
 377
 378                 # To count the chunk_len, and make sure we don't skip
 379                 # some tests, we round to the next value that fits exactly
 380                 # all the parts.
 381                 rounded_tests = num_tests
 382                 if rounded_tests % test_size != 0:
 383                     rounded_tests = (num_tests + test_size -
 384                                      (num_tests % test_size))
 385
 386                 chunk_len = rounded_tests / test_size
 387                 slice_start = chunk_len * (chunk_num - 1)
 388                 # It does not mind if we go over test_size.
 389
 390             # Get the end offset of the slice.
 391             slice_end = min(num_tests, slice_start + chunk_len)
 392
 393             files = test_files[slice_start:slice_end]
 394
 395             tests_run_msg = 'Running: %d tests (chunk slice [%d:%d] of %d)' % (
 396                 (slice_end - slice_start), slice_start, slice_end, num_tests)
 397             self._printer.print_expected(tests_run_msg)
 398
 399             # If we reached the end and we don't have enough tests, we run some
 400             # from the beginning.
 401             if slice_end - slice_start < chunk_len:
 402                 extra = chunk_len - (slice_end - slice_start)
 403                 extra_msg = ('   last chunk is partial, appending [0:%d]' %
 404                             extra)
 405                 self._printer.print_expected(extra_msg)
 406                 tests_run_msg += "\n" + extra_msg
 407                 files.extend(test_files[0:extra])
 408             tests_run_filename = os.path.join(self._options.results_directory,
 409                                               "tests_run.txt")
 410             with codecs.open(tests_run_filename, "w", "utf-8") as file:
 411                 file.write(tests_run_msg + "\n")
 412
 413             len_skip_chunk = int(len(files) * len(skipped) /
 414                                  float(len(self._test_files)))
 415             skip_chunk_list = list(skipped)[0:len_skip_chunk]
 416             skip_chunk = set(skip_chunk_list)
 417
 418             # Update expectations so that the stats are calculated correctly.
 419             # We need to pass a list that includes the right # of skipped files
 420             # to ParseExpectations so that ResultSummary() will get the correct
 421             # stats. So, we add in the subset of skipped files, and then
 422             # subtract them back out.
 423             self._test_files_list = files + skip_chunk_list
 424             self._test_files = set(self._test_files_list)
 425
 426             self._expectations = self.parse_expectations(
 427                 self._port.test_platform_name(),
 428                 self._options.configuration == 'Debug')
 429
 430             self._test_files = set(files)
 431             self._test_files_list = files
 432         else:
 433             skip_chunk = skipped
 434
 435         result_summary = ResultSummary(self._expectations,
 436             self._test_files | skip_chunk)
 437         self._print_expected_results_of_type(result_summary,
 438             test_expectations.PASS, "passes")
 439         self._print_expected_results_of_type(result_summary,
 440             test_expectations.FAIL, "failures")
 441         self._print_expected_results_of_type(result_summary,
 442             test_expectations.FLAKY, "flaky")
 443         self._print_expected_results_of_type(result_summary,
 444             test_expectations.SKIP, "skipped")
 445
 446         if self._options.force:
 447             self._printer.print_expected('Running all tests, including '
 448                                          'skips (--force)')
 449         else:
 450             # Note that we don't actually run the skipped tests (they were
 451             # subtracted out of self._test_files, above), but we stub out the
 452             # results here so the statistics can remain accurate.
 453             for test in skip_chunk:
 454                 result = test_results.TestResult(test,
 455                     failures=[], test_run_time=0, total_time_for_all_diffs=0,
 456                     time_for_diffs=0)
 457                 result.type = test_expectations.SKIP
 458                 result_summary.add(result, expected=True)
 459         self._printer.print_expected('')
 460
 461         return result_summary
 462
 463     def _get_dir_for_test_file(self, test_file):
 464         """Returns the highest-level directory by which to shard the given
 465         test file."""
 466         index = test_file.rfind(os.sep + LAYOUT_TESTS_DIRECTORY)
 467
 468         test_file = test_file[index + len(LAYOUT_TESTS_DIRECTORY):]
 469         test_file_parts = test_file.split(os.sep, 1)
 470         directory = test_file_parts[0]
 471         test_file = test_file_parts[1]
 472
 473         # The http tests are very stable on mac/linux.
 474         # TODO(ojan): Make the http server on Windows be apache so we can
 475         # turn shard the http tests there as well. Switching to apache is
 476         # what made them stable on linux/mac.
 477         return_value = directory
 478         while ((directory != 'http' or sys.platform in ('darwin', 'linux2'))
 479                 and test_file.find(os.sep) >= 0):
 480             test_file_parts = test_file.split(os.sep, 1)
 481             directory = test_file_parts[0]
 482             return_value = os.path.join(return_value, directory)
 483             test_file = test_file_parts[1]
 484
 485         return return_value
 486
 487     def _get_test_input_for_file(self, test_file):
 488         """Returns the appropriate TestInput object for the file. Mostly this
 489         is used for looking up the timeout value (in ms) to use for the given
 490         test."""
 491         if self._test_is_slow(test_file):
 492             return TestInput(test_file, self._options.slow_time_out_ms)
 493         return TestInput(test_file, self._options.time_out_ms)
 494
 495     def _test_requires_lock(self, test_file):
 496         """Return True if the test needs to be locked when
 497         running multiple copies of NRWTs."""
 498         split_path = test_file.split(os.sep)
 499         return 'http' in split_path or 'websocket' in split_path
 500
 501     def _test_is_slow(self, test_file):
 502         return self._expectations.has_modifier(test_file,
 503                                                test_expectations.SLOW)
 504
 505     def _shard_tests(self, test_files, use_real_shards):
 506         """Groups tests into batches.
 507         This helps ensure that tests that depend on each other (aka bad tests!)
 508         continue to run together as most cross-tests dependencies tend to
 509         occur within the same directory. If use_real_shards is false, we
 510         put each (non-HTTP/websocket) test into its own shard for maximum
 511         concurrency instead of trying to do any sort of real sharding.
 512
 513         Return:
 514             A list of lists of TestInput objects.
 515         """
 516         # FIXME: when we added http locking, we changed how this works such
 517         # that we always lump all of the HTTP threads into a single shard.
 518         # That will slow down experimental-fully-parallel, but it's unclear
 519         # what the best alternative is completely revamping how we track
 520         # when to grab the lock.
 521
 522         test_lists = []
 523         tests_to_http_lock = []
 524         if not use_real_shards:
 525             for test_file in test_files:
 526                 test_input = self._get_test_input_for_file(test_file)
 527                 if self._test_requires_lock(test_file):
 528                     tests_to_http_lock.append(test_input)
 529                 else:
 530                     test_lists.append((".", [test_input]))
 531         else:
 532             tests_by_dir = {}
 533             for test_file in test_files:
 534                 directory = self._get_dir_for_test_file(test_file)
 535                 test_input = self._get_test_input_for_file(test_file)
 536                 if self._test_requires_lock(test_file):
 537                     tests_to_http_lock.append(test_input)
 538                 else:
 539                     tests_by_dir.setdefault(directory, [])
 540                     tests_by_dir[directory].append(test_input)
 541             # Sort by the number of tests in the dir so that the ones with the
 542             # most tests get run first in order to maximize parallelization.
 543             # Number of tests is a good enough, but not perfect, approximation
 544             # of how long that set of tests will take to run. We can't just use
 545             # a PriorityQueue until we move to Python 2.6.
 546             for directory in tests_by_dir:
 547                 test_list = tests_by_dir[directory]
 548                 # Keep the tests in alphabetical order.
 549                 # FIXME: Remove once tests are fixed so they can be run in any
 550                 # order.
 551                 test_list.reverse()
 552                 test_list_tuple = (directory, test_list)
 553                 test_lists.append(test_list_tuple)
 554             test_lists.sort(lambda a, b: cmp(len(b[1]), len(a[1])))
 555
 556         # Put the http tests first. There are only a couple hundred of them,
 557         # but each http test takes a very long time to run, so sorting by the
 558         # number of tests doesn't accurately capture how long they take to run.
 559         if tests_to_http_lock:
 560             tests_to_http_lock.reverse()
 561             test_lists.insert(0, ("tests_to_http_lock", tests_to_http_lock))
 562
 563         return test_lists
 564
 565     def _contains_tests(self, subdir):
 566         for test_file in self._test_files:
 567             if test_file.find(subdir) >= 0:
 568                 return True
 569         return False
 570
 571     def _num_workers(self):
 572         return int(self._options.child_processes)
 573
 574     def _run_tests(self, file_list, result_summary):
 575         """Runs the tests in the file_list.
 576
 577         Return: A tuple (keyboard_interrupted, thread_timings, test_timings,
 578             individual_test_timings)
 579             keyboard_interrupted is whether someone typed Ctrl^C
 580             thread_timings is a list of dicts with the total runtime
 581               of each thread with 'name', 'num_tests', 'total_time' properties
 582             test_timings is a list of timings for each sharded subdirectory
 583               of the form [time, directory_name, num_tests]
 584             individual_test_timings is a list of run times for each test
 585               in the form {filename:filename, test_run_time:test_run_time}
 586             result_summary: summary object to populate with the results
 587         """
 588
 589         self._printer.print_update('Sharding tests ...')
 590         num_workers = self._num_workers()
 591         test_lists = self._shard_tests(file_list,
 592             num_workers > 1 and not self._options.experimental_fully_parallel)
 593         filename_queue = Queue.Queue()
 594         for item in test_lists:
 595             filename_queue.put(item)
 596
 597         self._printer.print_update('Starting %s ...' %
 598                                    grammar.pluralize('worker', num_workers))
 599         message_broker = self._message_broker
 600         self._current_filename_queue = filename_queue
 601         self._current_result_summary = result_summary
 602
 603         if not self._options.dry_run:
 604             threads = message_broker.start_workers(self)
 605         else:
 606             threads = []
 607
 608         self._printer.print_update("Starting testing ...")
 609         keyboard_interrupted = False
 610         if not self._options.dry_run:
 611             try:
 612                 message_broker.run_message_loop()
 613             except KeyboardInterrupt:
 614                 _log.info("Interrupted, exiting")
 615                 message_broker.cancel_workers()
 616                 keyboard_interrupted = True
 617             except:
 618                 # Unexpected exception; don't try to clean up workers.
 619                 _log.info("Exception raised, exiting")
 620                 raise
 621
 622         thread_timings, test_timings, individual_test_timings = \
 623             self._collect_timing_info(threads)
 624
 625         return (keyboard_interrupted, thread_timings, test_timings,
 626                 individual_test_timings)
 627
 628     def update(self):
 629         self.update_summary(self._current_result_summary)
 630
 631     def _collect_timing_info(self, threads):
 632         test_timings = {}
 633         individual_test_timings = []
 634         thread_timings = []
 635
 636         for thread in threads:
 637             thread_timings.append({'name': thread.getName(),
 638                                    'num_tests': thread.get_num_tests(),
 639                                    'total_time': thread.get_total_time()})
 640             test_timings.update(thread.get_test_group_timing_stats())
 641             individual_test_timings.extend(thread.get_test_results())
 642
 643         return (thread_timings, test_timings, individual_test_timings)
 644
 645     def needs_http(self):
 646         """Returns whether the test runner needs an HTTP server."""
 647         return self._contains_tests(self.HTTP_SUBDIR)
 648
 649     def needs_websocket(self):
 650         """Returns whether the test runner needs a WEBSOCKET server."""
 651         return self._contains_tests(self.WEBSOCKET_SUBDIR)
 652
 653     def set_up_run(self):
 654         """Configures the system to be ready to run tests.
 655
 656         Returns a ResultSummary object if we should continue to run tests,
 657         or None if we should abort.
 658
 659         """
 660         # This must be started before we check the system dependencies,
 661         # since the helper may do things to make the setup correct.
 662         self._printer.print_update("Starting helper ...")
 663         self._port.start_helper()
 664
 665         # Check that the system dependencies (themes, fonts, ...) are correct.
 666         if not self._options.nocheck_sys_deps:
 667             self._printer.print_update("Checking system dependencies ...")
 668             if not self._port.check_sys_deps(self.needs_http()):
 669                 self._port.stop_helper()
 670                 return None
 671
 672         if self._options.clobber_old_results:
 673             self._clobber_old_results()
 674
 675         # Create the output directory if it doesn't already exist.
 676         self._port.maybe_make_directory(self._options.results_directory)
 677
 678         self._port.setup_test_run()
 679
 680         self._printer.print_update("Preparing tests ...")
 681         result_summary = self.prepare_lists_and_print_output()
 682         if not result_summary:
 683             return None
 684
 685         return result_summary
 686
 687     def run(self, result_summary):
 688         """Run all our tests on all our test files.
 689
 690         For each test file, we run each test type. If there are any failures,
 691         we collect them for reporting.
 692
 693         Args:
 694           result_summary: a summary object tracking the test results.
 695
 696         Return:
 697           The number of unexpected results (0 == success)
 698         """
 699         # gather_test_files() must have been called first to initialize us.
 700         # If we didn't find any files to test, we've errored out already in
 701         # prepare_lists_and_print_output().
 702         assert(len(self._test_files))
 703
 704         start_time = time.time()
 705
 706         keyboard_interrupted, thread_timings, test_timings, \
 707             individual_test_timings = (
 708             self._run_tests(self._test_files_list, result_summary))
 709
 710         # We exclude the crashes from the list of results to retry, because
 711         # we want to treat even a potentially flaky crash as an error.
 712         failures = self._get_failures(result_summary, include_crashes=False)
 713         retry_summary = result_summary
 714         while (len(failures) and self._options.retry_failures and
 715             not self._retrying and not keyboard_interrupted):
 716             _log.info('')
 717             _log.info("Retrying %d unexpected failure(s) ..." % len(failures))
 718             _log.info('')
 719             self._retrying = True
 720             retry_summary = ResultSummary(self._expectations, failures.keys())
 721             # Note that we intentionally ignore the return value here.
 722             self._run_tests(failures.keys(), retry_summary)
 723             failures = self._get_failures(retry_summary, include_crashes=True)
 724
 725         end_time = time.time()
 726
 727         self._print_timing_statistics(end_time - start_time,
 728                                       thread_timings, test_timings,
 729                                       individual_test_timings,
 730                                       result_summary)
 731
 732         self._print_result_summary(result_summary)
 733
 734         sys.stdout.flush()
 735         sys.stderr.flush()
 736
 737         self._printer.print_one_line_summary(result_summary.total,
 738                                              result_summary.expected,
 739                                              result_summary.unexpected)
 740
 741         unexpected_results = summarize_unexpected_results(self._port,
 742             self._expectations, result_summary, retry_summary)
 743         self._printer.print_unexpected_results(unexpected_results)
 744
 745         if (self._options.record_results and not self._options.dry_run and
 746             not keyboard_interrupted):
 747             # Write the same data to log files and upload generated JSON files
 748             # to appengine server.
 749             self._upload_json_files(unexpected_results, result_summary,
 750                                     individual_test_timings)
 751
 752         # Write the summary to disk (results.html) and display it if requested.
 753         if not self._options.dry_run:
 754             wrote_results = self._write_results_html_file(result_summary)
 755             if self._options.show_results and wrote_results:
 756                 self._show_results_html_file()
 757
 758         # Now that we've completed all the processing we can, we re-raise
 759         # a KeyboardInterrupt if necessary so the caller can handle it.
 760         if keyboard_interrupted:
 761             raise KeyboardInterrupt
 762
 763         # Ignore flaky failures and unexpected passes so we don't turn the
 764         # bot red for those.
 765         return unexpected_results['num_regressions']
 766
 767     def clean_up_run(self):
 768         """Restores the system after we're done running tests."""
 769
 770         _log.debug("flushing stdout")
 771         sys.stdout.flush()
 772         _log.debug("flushing stderr")
 773         sys.stderr.flush()
 774         _log.debug("stopping helper")
 775         self._port.stop_helper()
 776
 777     def update_summary(self, result_summary):
 778         """Update the summary and print results with any completed tests."""
 779         while True:
 780             try:
 781                 result = test_results.TestResult.loads(self._result_queue.get_nowait())
 782             except Queue.Empty:
 783                 return
 784
 785             expected = self._expectations.matches_an_expected_result(
 786                 result.filename, result.type, self._options.pixel_tests)
 787             result_summary.add(result, expected)
 788             exp_str = self._expectations.get_expectations_string(
 789                 result.filename)
 790             got_str = self._expectations.expectation_to_string(result.type)
 791             self._printer.print_test_result(result, expected, exp_str, got_str)
 792             self._printer.print_progress(result_summary, self._retrying,
 793                                          self._test_files_list)
 794
 795     def _clobber_old_results(self):
 796         # Just clobber the actual test results directories since the other
 797         # files in the results directory are explicitly used for cross-run
 798         # tracking.
 799         self._printer.print_update("Clobbering old results in %s" %
 800                                    self._options.results_directory)
 801         layout_tests_dir = self._port.layout_tests_dir()
 802         possible_dirs = self._port.test_dirs()
 803         for dirname in possible_dirs:
 804             if os.path.isdir(os.path.join(layout_tests_dir, dirname)):
 805                 shutil.rmtree(os.path.join(self._options.results_directory,
 806                                            dirname),
 807                               ignore_errors=True)
 808
 809     def _get_failures(self, result_summary, include_crashes):
 810         """Filters a dict of results and returns only the failures.
 811
 812         Args:
 813           result_summary: the results of the test run
 814           include_crashes: whether crashes are included in the output.
 815             We use False when finding the list of failures to retry
 816             to see if the results were flaky. Although the crashes may also be
 817             flaky, we treat them as if they aren't so that they're not ignored.
 818         Returns:
 819           a dict of files -> results
 820         """
 821         failed_results = {}
 822         for test, result in result_summary.unexpected_results.iteritems():
 823             if (result == test_expectations.PASS or
 824                 result == test_expectations.CRASH and not include_crashes):
 825                 continue
 826             failed_results[test] = result
 827
 828         return failed_results
 829
 830     def _upload_json_files(self, unexpected_results, result_summary,
 831                         individual_test_timings):
 832         """Writes the results of the test run as JSON files into the results
 833         dir and upload the files to the appengine server.
 834
 835         There are three different files written into the results dir:
 836           unexpected_results.json: A short list of any unexpected results.
 837             This is used by the buildbots to display results.
 838           expectations.json: This is used by the flakiness dashboard.
 839           results.json: A full list of the results - used by the flakiness
 840             dashboard and the aggregate results dashboard.
 841
 842         Args:
 843           unexpected_results: dict of unexpected results
 844           result_summary: full summary object
 845           individual_test_timings: list of test times (used by the flakiness
 846             dashboard).
 847         """
 848         results_directory = self._options.results_directory
 849         _log.debug("Writing JSON files in %s." % results_directory)
 850         unexpected_json_path = os.path.join(results_directory, "unexpected_results.json")
 851         with codecs.open(unexpected_json_path, "w", "utf-8") as file:
 852             simplejson.dump(unexpected_results, file, sort_keys=True, indent=2)
 853
 854         # Write a json file of the test_expectations.txt file for the layout
 855         # tests dashboard.
 856         expectations_path = os.path.join(results_directory, "expectations.json")
 857         expectations_json = \
 858             self._expectations.get_expectations_json_for_all_platforms()
 859         with codecs.open(expectations_path, "w", "utf-8") as file:
 860             file.write(u"ADD_EXPECTATIONS(%s);" % expectations_json)
 861
 862         generator = json_layout_results_generator.JSONLayoutResultsGenerator(
 863             self._port, self._options.builder_name, self._options.build_name,
 864             self._options.build_number, self._options.results_directory,
 865             BUILDER_BASE_URL, individual_test_timings,
 866             self._expectations, result_summary, self._test_files_list,
 867             not self._options.upload_full_results,
 868             self._options.test_results_server,
 869             "layout-tests",
 870             self._options.master_name)
 871
 872         _log.debug("Finished writing JSON files.")
 873
 874         json_files = ["expectations.json"]
 875         if self._options.upload_full_results:
 876             json_files.append("results.json")
 877         else:
 878             json_files.append("incremental_results.json")
 879
 880         generator.upload_json_files(json_files)
 881
 882     def _print_config(self):
 883         """Prints the configuration for the test run."""
 884         p = self._printer
 885         p.print_config("Using port '%s'" % self._port.name())
 886         p.print_config("Placing test results in %s" %
 887                        self._options.results_directory)
 888         if self._options.new_baseline:
 889             p.print_config("Placing new baselines in %s" %
 890                            self._port.baseline_path())
 891         p.print_config("Using %s build" % self._options.configuration)
 892         if self._options.pixel_tests:
 893             p.print_config("Pixel tests enabled")
 894         else:
 895             p.print_config("Pixel tests disabled")
 896
 897         p.print_config("Regular timeout: %s, slow test timeout: %s" %
 898                        (self._options.time_out_ms,
 899                         self._options.slow_time_out_ms))
 900
 901         if self._num_workers() == 1:
 902             p.print_config("Running one %s" % self._port.driver_name())
 903         else:
 904             p.print_config("Running %s %ss in parallel" %
 905                            (self._options.child_processes,
 906                             self._port.driver_name()))
 907         p.print_config('Command line: ' +
 908                        ' '.join(self._port.driver_cmd_line()))
 909         p.print_config("Worker model: %s" % self._options.worker_model)
 910         p.print_config("")
 911
 912     def _print_expected_results_of_type(self, result_summary,
 913                                         result_type, result_type_str):
 914         """Print the number of the tests in a given result class.
 915
 916         Args:
 917           result_summary - the object containing all the results to report on
 918           result_type - the particular result type to report in the summary.
 919           result_type_str - a string description of the result_type.
 920         """
 921         tests = self._expectations.get_tests_with_result_type(result_type)
 922         now = result_summary.tests_by_timeline[test_expectations.NOW]
 923         wontfix = result_summary.tests_by_timeline[test_expectations.WONTFIX]
 924
 925         # We use a fancy format string in order to print the data out in a
 926         # nicely-aligned table.
 927         fmtstr = ("Expect: %%5d %%-8s (%%%dd now, %%%dd wontfix)"
 928                   % (self._num_digits(now), self._num_digits(wontfix)))
 929         self._printer.print_expected(fmtstr %
 930             (len(tests), result_type_str, len(tests & now), len(tests & wontfix)))
 931
 932     def _num_digits(self, num):
 933         """Returns the number of digits needed to represent the length of a
 934         sequence."""
 935         ndigits = 1
 936         if len(num):
 937             ndigits = int(math.log10(len(num))) + 1
 938         return ndigits
 939
 940     def _print_timing_statistics(self, total_time, thread_timings,
 941                                directory_test_timings, individual_test_timings,
 942                                result_summary):
 943         """Record timing-specific information for the test run.
 944
 945         Args:
 946           total_time: total elapsed time (in seconds) for the test run
 947           thread_timings: wall clock time each thread ran for
 948           directory_test_timings: timing by directory
 949           individual_test_timings: timing by file
 950           result_summary: summary object for the test run
 951         """
 952         self._printer.print_timing("Test timing:")
 953         self._printer.print_timing("  %6.2f total testing time" % total_time)
 954         self._printer.print_timing("")
 955         self._printer.print_timing("Thread timing:")
 956         cuml_time = 0
 957         for t in thread_timings:
 958             self._printer.print_timing("    %10s: %5d tests, %6.2f secs" %
 959                   (t['name'], t['num_tests'], t['total_time']))
 960             cuml_time += t['total_time']
 961         self._printer.print_timing("   %6.2f cumulative, %6.2f optimal" %
 962               (cuml_time, cuml_time / int(self._options.child_processes)))
 963         self._printer.print_timing("")
 964
 965         self._print_aggregate_test_statistics(individual_test_timings)
 966         self._print_individual_test_times(individual_test_timings,
 967                                           result_summary)
 968         self._print_directory_timings(directory_test_timings)
 969
 970     def _print_aggregate_test_statistics(self, individual_test_timings):
 971         """Prints aggregate statistics (e.g. median, mean, etc.) for all tests.
 972         Args:
 973           individual_test_timings: List of dump_render_tree_thread.TestStats
 974               for all tests.
 975         """
 976         test_types = []  # Unit tests don't actually produce any timings.
 977         if individual_test_timings:
 978             test_types = individual_test_timings[0].time_for_diffs.keys()
 979         times_for_dump_render_tree = []
 980         times_for_diff_processing = []
 981         times_per_test_type = {}
 982         for test_type in test_types:
 983             times_per_test_type[test_type] = []
 984
 985         for test_stats in individual_test_timings:
 986             times_for_dump_render_tree.append(test_stats.test_run_time)
 987             times_for_diff_processing.append(
 988                 test_stats.total_time_for_all_diffs)
 989             time_for_diffs = test_stats.time_for_diffs
 990             for test_type in test_types:
 991                 times_per_test_type[test_type].append(
 992                     time_for_diffs[test_type])
 993
 994         self._print_statistics_for_test_timings(
 995             "PER TEST TIME IN TESTSHELL (seconds):",
 996             times_for_dump_render_tree)
 997         self._print_statistics_for_test_timings(
 998             "PER TEST DIFF PROCESSING TIMES (seconds):",
 999             times_for_diff_processing)
1000         for test_type in test_types:
1001             self._print_statistics_for_test_timings(
1002                 "PER TEST TIMES BY TEST TYPE: %s" % test_type,
1003                 times_per_test_type[test_type])
1004
1005     def _print_individual_test_times(self, individual_test_timings,
1006                                   result_summary):
1007         """Prints the run times for slow, timeout and crash tests.
1008         Args:
1009           individual_test_timings: List of dump_render_tree_thread.TestStats
1010               for all tests.
1011           result_summary: summary object for test run
1012         """
1013         # Reverse-sort by the time spent in DumpRenderTree.
1014         individual_test_timings.sort(lambda a, b:
1015             cmp(b.test_run_time, a.test_run_time))
1016
1017         num_printed = 0
1018         slow_tests = []
1019         timeout_or_crash_tests = []
1020         unexpected_slow_tests = []
1021         for test_tuple in individual_test_timings:
1022             filename = test_tuple.filename
1023             is_timeout_crash_or_slow = False
1024             if self._test_is_slow(filename):
1025                 is_timeout_crash_or_slow = True
1026                 slow_tests.append(test_tuple)
1027
1028             if filename in result_summary.failures:
1029                 result = result_summary.results[filename].type
1030                 if (result == test_expectations.TIMEOUT or
1031                     result == test_expectations.CRASH):
1032                     is_timeout_crash_or_slow = True
1033                     timeout_or_crash_tests.append(test_tuple)
1034
1035             if (not is_timeout_crash_or_slow and
1036                 num_printed < printing.NUM_SLOW_TESTS_TO_LOG):
1037                 num_printed = num_printed + 1
1038                 unexpected_slow_tests.append(test_tuple)
1039
1040         self._printer.print_timing("")
1041         self._print_test_list_timing("%s slowest tests that are not "
1042             "marked as SLOW and did not timeout/crash:" %
1043             printing.NUM_SLOW_TESTS_TO_LOG, unexpected_slow_tests)
1044         self._printer.print_timing("")
1045         self._print_test_list_timing("Tests marked as SLOW:", slow_tests)
1046         self._printer.print_timing("")
1047         self._print_test_list_timing("Tests that timed out or crashed:",
1048                                      timeout_or_crash_tests)
1049         self._printer.print_timing("")
1050
1051     def _print_test_list_timing(self, title, test_list):
1052         """Print timing info for each test.
1053
1054         Args:
1055           title: section heading
1056           test_list: tests that fall in this section
1057         """
1058         if self._printer.disabled('slowest'):
1059             return
1060
1061         self._printer.print_timing(title)
1062         for test_tuple in test_list:
1063             filename = test_tuple.filename[len(
1064                 self._port.layout_tests_dir()) + 1:]
1065             filename = filename.replace('\\', '/')
1066             test_run_time = round(test_tuple.test_run_time, 1)
1067             self._printer.print_timing("  %s took %s seconds" %
1068                                        (filename, test_run_time))
1069
1070     def _print_directory_timings(self, directory_test_timings):
1071         """Print timing info by directory for any directories that
1072         take > 10 seconds to run.
1073
1074         Args:
1075           directory_test_timing: time info for each directory
1076         """
1077         timings = []
1078         for directory in directory_test_timings:
1079             num_tests, time_for_directory = directory_test_timings[directory]
1080             timings.append((round(time_for_directory, 1), directory,
1081                             num_tests))
1082         timings.sort()
1083
1084         self._printer.print_timing("Time to process slowest subdirectories:")
1085         min_seconds_to_print = 10
1086         for timing in timings:
1087             if timing[0] > min_seconds_to_print:
1088                 self._printer.print_timing(
1089                     "  %s took %s seconds to run %s tests." % (timing[1],
1090                     timing[0], timing[2]))
1091         self._printer.print_timing("")
1092
1093     def _print_statistics_for_test_timings(self, title, timings):
1094         """Prints the median, mean and standard deviation of the values in
1095         timings.
1096
1097         Args:
1098           title: Title for these timings.
1099           timings: A list of floats representing times.
1100         """
1101         self._printer.print_timing(title)
1102         timings.sort()
1103
1104         num_tests = len(timings)
1105         if not num_tests:
1106             return
1107         percentile90 = timings[int(.9 * num_tests)]
1108         percentile99 = timings[int(.99 * num_tests)]
1109
1110         if num_tests % 2 == 1:
1111             median = timings[((num_tests - 1) / 2) - 1]
1112         else:
1113             lower = timings[num_tests / 2 - 1]
1114             upper = timings[num_tests / 2]
1115             median = (float(lower + upper)) / 2
1116
1117         mean = sum(timings) / num_tests
1118
1119         for time in timings:
1120             sum_of_deviations = math.pow(time - mean, 2)
1121
1122         std_deviation = math.sqrt(sum_of_deviations / num_tests)
1123         self._printer.print_timing("  Median:          %6.3f" % median)
1124         self._printer.print_timing("  Mean:            %6.3f" % mean)
1125         self._printer.print_timing("  90th percentile: %6.3f" % percentile90)
1126         self._printer.print_timing("  99th percentile: %6.3f" % percentile99)
1127         self._printer.print_timing("  Standard dev:    %6.3f" % std_deviation)
1128         self._printer.print_timing("")
1129
1130     def _print_result_summary(self, result_summary):
1131         """Print a short summary about how many tests passed.
1132
1133         Args:
1134           result_summary: information to log
1135         """
1136         failed = len(result_summary.failures)
1137         skipped = len(
1138             result_summary.tests_by_expectation[test_expectations.SKIP])
1139         total = result_summary.total
1140         passed = total - failed - skipped
1141         pct_passed = 0.0
1142         if total > 0:
1143             pct_passed = float(passed) * 100 / total
1144
1145         self._printer.print_actual("")
1146         self._printer.print_actual("=> Results: %d/%d tests passed (%.1f%%)" %
1147                      (passed, total, pct_passed))
1148         self._printer.print_actual("")
1149         self._print_result_summary_entry(result_summary,
1150             test_expectations.NOW, "Tests to be fixed")
1151
1152         self._printer.print_actual("")
1153         self._print_result_summary_entry(result_summary,
1154             test_expectations.WONTFIX,
1155             "Tests that will only be fixed if they crash (WONTFIX)")
1156         self._printer.print_actual("")
1157
1158     def _print_result_summary_entry(self, result_summary, timeline,
1159                                     heading):
1160         """Print a summary block of results for a particular timeline of test.
1161
1162         Args:
1163           result_summary: summary to print results for
1164           timeline: the timeline to print results for (NOT, WONTFIX, etc.)
1165           heading: a textual description of the timeline
1166         """
1167         total = len(result_summary.tests_by_timeline[timeline])
1168         not_passing = (total -
1169            len(result_summary.tests_by_expectation[test_expectations.PASS] &
1170                result_summary.tests_by_timeline[timeline]))
1171         self._printer.print_actual("=> %s (%d):" % (heading, not_passing))
1172
1173         for result in TestExpectationsFile.EXPECTATION_ORDER:
1174             if result == test_expectations.PASS:
1175                 continue
1176             results = (result_summary.tests_by_expectation[result] &
1177                        result_summary.tests_by_timeline[timeline])
1178             desc = TestExpectationsFile.EXPECTATION_DESCRIPTIONS[result]
1179             if not_passing and len(results):
1180                 pct = len(results) * 100.0 / not_passing
1181                 self._printer.print_actual("  %5d %-24s (%4.1f%%)" %
1182                     (len(results), desc[len(results) != 1], pct))
1183
1184     def _results_html(self, test_files, failures, title="Test Failures", override_time=None):
1185         """
1186         test_files = a list of file paths
1187         failures = dictionary mapping test paths to failure objects
1188         title = title printed at top of test
1189         override_time = current time (used by unit tests)
1190         """
1191         page = """<html>
1192   <head>
1193     <title>Layout Test Results (%(time)s)</title>
1194   </head>
1195   <body>
1196     <h2>%(title)s (%(time)s)</h2>
1197         """ % {'title': title, 'time': override_time or time.asctime()}
1198
1199         for test_file in sorted(test_files):
1200             test_name = self._port.relative_test_filename(test_file)
1201             test_url = self._port.filename_to_uri(test_file)
1202             page += u"<p><a href='%s'>%s</a><br />\n" % (test_url, test_name)
1203             test_failures = failures.get(test_file, [])
1204             for failure in test_failures:
1205                 page += (u"&nbsp;&nbsp;%s<br/>" %
1206                          failure.result_html_output(test_name))
1207             page += "</p>\n"
1208         page += "</body></html>\n"
1209         return page
1210
1211     def _write_results_html_file(self, result_summary):
1212         """Write results.html which is a summary of tests that failed.
1213
1214         Args:
1215           result_summary: a summary of the results :)
1216
1217         Returns:
1218           True if any results were written (since expected failures may be
1219           omitted)
1220         """
1221         # test failures
1222         if self._options.full_results_html:
1223             results_title = "Test Failures"
1224             test_files = result_summary.failures.keys()
1225         else:
1226             results_title = "Unexpected Test Failures"
1227             unexpected_failures = self._get_failures(result_summary,
1228                 include_crashes=True)
1229             test_files = unexpected_failures.keys()
1230         if not len(test_files):
1231             return False
1232
1233         out_filename = os.path.join(self._options.results_directory,
1234                                     "results.html")
1235         with codecs.open(out_filename, "w", "utf-8") as results_file:
1236             html = self._results_html(test_files, result_summary.failures, results_title)
1237             results_file.write(html)
1238
1239         return True
1240
1241     def _show_results_html_file(self):
1242         """Shows the results.html page."""
1243         results_filename = os.path.join(self._options.results_directory,
1244                                         "results.html")
1245         self._port.show_results_html_file(results_filename)
1246
1247
1248 def read_test_files(files):
1249     tests = []
1250     for file in files:
1251         try:
1252             with codecs.open(file, 'r', 'utf-8') as file_contents:
1253                 # FIXME: This could be cleaner using a list comprehension.
1254                 for line in file_contents:
1255                     line = test_expectations.strip_comments(line)
1256                     if line:
1257                         tests.append(line)
1258         except IOError, e:
1259             if e.errno == errno.ENOENT:
1260                 _log.critical('')
1261                 _log.critical('--test-list file "%s" not found' % file)
1262             raise
1263     return tests
1264
1265
1266 def run(port, options, args, regular_output=sys.stderr,
1267         buildbot_output=sys.stdout):
1268     """Run the tests.
1269
1270     Args:
1271       port: Port object for port-specific behavior
1272       options: a dictionary of command line options
1273       args: a list of sub directories or files to test
1274       regular_output: a stream-like object that we can send logging/debug
1275           output to
1276       buildbot_output: a stream-like object that we can write all output that
1277           is intended to be parsed by the buildbot to
1278     Returns:
1279       the number of unexpected results that occurred, or -1 if there is an
1280           error.
1281
1282     """
1283     _set_up_derived_options(port, options)
1284
1285     printer = printing.Printer(port, options, regular_output, buildbot_output,
1286         int(options.child_processes), options.experimental_fully_parallel)
1287     if options.help_printing:
1288         printer.help_printing()
1289         printer.cleanup()
1290         return 0
1291
1292     last_unexpected_results = _gather_unexpected_results(options)
1293     if options.print_last_failures:
1294         printer.write("\n".join(last_unexpected_results) + "\n")
1295         printer.cleanup()
1296         return 0
1297
1298     broker = message_broker.get(port, options)
1299
1300     # We wrap any parts of the run that are slow or likely to raise exceptions
1301     # in a try/finally to ensure that we clean up the logging configuration.
1302     num_unexpected_results = -1
1303     try:
1304         test_runner = TestRunner(port, options, printer, broker)
1305         test_runner._print_config()
1306
1307         printer.print_update("Collecting tests ...")
1308         try:
1309             test_runner.collect_tests(args, last_unexpected_results)
1310         except IOError, e:
1311             if e.errno == errno.ENOENT:
1312                 return -1
1313             raise
1314
1315         printer.print_update("Parsing expectations ...")
1316         if options.lint_test_files:
1317             return test_runner.lint()
1318         test_runner.parse_expectations(port.test_platform_name(),
1319                                        options.configuration == 'Debug')
1320
1321         printer.print_update("Checking build ...")
1322         if not port.check_build(test_runner.needs_http()):
1323             _log.error("Build check failed")
1324             return -1
1325
1326         result_summary = test_runner.set_up_run()
1327         if result_summary:
1328             num_unexpected_results = test_runner.run(result_summary)
1329             test_runner.clean_up_run()
1330             _log.debug("Testing completed, Exit status: %d" %
1331                        num_unexpected_results)
1332     finally:
1333         broker.cleanup()
1334         printer.cleanup()
1335
1336     return num_unexpected_results
1337
1338
1339 def _set_up_derived_options(port_obj, options):
1340     """Sets the options values that depend on other options values."""
1341     if options.worker_model is None:
1342         options.worker_model = port_obj.default_worker_model()
1343     if options.worker_model == 'inline':
1344         if options.child_processes and int(options.child_processes) > 1:
1345             _log.warning("--worker-model=inline overrides --child-processes")
1346         options.child_processes = "1"
1347     if not options.child_processes:
1348         options.child_processes = os.environ.get("WEBKIT_TEST_CHILD_PROCESSES",
1349                                                  str(port_obj.default_child_processes()))
1350
1351     if not options.configuration:
1352         options.configuration = port_obj.default_configuration()
1353
1354     if options.pixel_tests is None:
1355         options.pixel_tests = True
1356
1357     if not options.use_apache:
1358         options.use_apache = sys.platform in ('darwin', 'linux2')
1359
1360     if not os.path.isabs(options.results_directory):
1361         # This normalizes the path to the build dir.
1362         # FIXME: how this happens is not at all obvious; this is a dumb
1363         # interface and should be cleaned up.
1364         options.results_directory = port_obj.results_directory()
1365
1366     if not options.time_out_ms:
1367         if options.configuration == "Debug":
1368             options.time_out_ms = str(2 * TestRunner.DEFAULT_TEST_TIMEOUT_MS)
1369         else:
1370             options.time_out_ms = str(TestRunner.DEFAULT_TEST_TIMEOUT_MS)
1371
1372     options.slow_time_out_ms = str(5 * int(options.time_out_ms))
1373
1374
1375 def _gather_unexpected_results(options):
1376     """Returns the unexpected results from the previous run, if any."""
1377     last_unexpected_results = []
1378     if options.print_last_failures or options.retest_last_failures:
1379         unexpected_results_filename = os.path.join(
1380         options.results_directory, "unexpected_results.json")
1381         with codecs.open(unexpected_results_filename, "r", "utf-8") as file:
1382             results = simplejson.load(file)
1383         last_unexpected_results = results['tests'].keys()
1384     return last_unexpected_results
1385
1386
1387 def _compat_shim_callback(option, opt_str, value, parser):
1388     print "Ignoring unsupported option: %s" % opt_str
1389
1390
1391 def _compat_shim_option(option_name, **kwargs):
1392     return optparse.make_option(option_name, action="callback",
1393         callback=_compat_shim_callback,
1394         help="Ignored, for old-run-webkit-tests compat only.", **kwargs)
1395
1396
1397 def parse_args(args=None):
1398     """Provides a default set of command line args.
1399
1400     Returns a tuple of options, args from optparse"""
1401
1402     # FIXME: All of these options should be stored closer to the code which
1403     # FIXME: actually uses them. configuration_options should move
1404     # FIXME: to WebKitPort and be shared across all scripts.
1405     configuration_options = [
1406         optparse.make_option("-t", "--target", dest="configuration",
1407                              help="(DEPRECATED)"),
1408         # FIXME: --help should display which configuration is default.
1409         optparse.make_option('--debug', action='store_const', const='Debug',
1410                              dest="configuration",
1411                              help='Set the configuration to Debug'),
1412         optparse.make_option('--release', action='store_const',
1413                              const='Release', dest="configuration",
1414                              help='Set the configuration to Release'),
1415         # old-run-webkit-tests also accepts -c, --configuration CONFIGURATION.
1416     ]
1417
1418     print_options = printing.print_options()
1419
1420     # FIXME: These options should move onto the ChromiumPort.
1421     chromium_options = [
1422         optparse.make_option("--chromium", action="store_true", default=False,
1423             help="use the Chromium port"),
1424         optparse.make_option("--startup-dialog", action="store_true",
1425             default=False, help="create a dialog on DumpRenderTree startup"),
1426         optparse.make_option("--gp-fault-error-box", action="store_true",
1427             default=False, help="enable Windows GP fault error box"),
1428         optparse.make_option("--multiple-loads",
1429             type="int", help="turn on multiple loads of each test"),
1430         optparse.make_option("--js-flags",
1431             type="string", help="JavaScript flags to pass to tests"),
1432         optparse.make_option("--nocheck-sys-deps", action="store_true",
1433             default=False,
1434             help="Don't check the system dependencies (themes)"),
1435         optparse.make_option("--use-drt", action="store_true",
1436             default=None,
1437             help="Use DumpRenderTree instead of test_shell"),
1438         optparse.make_option("--accelerated-compositing",
1439             action="store_true",
1440             help="Use hardware-accelated compositing for rendering"),
1441         optparse.make_option("--no-accelerated-compositing",
1442             action="store_false",
1443             dest="accelerated_compositing",
1444             help="Don't use hardware-accelerated compositing for rendering"),
1445         optparse.make_option("--accelerated-2d-canvas",
1446             action="store_true",
1447             help="Use hardware-accelerated 2D Canvas calls"),
1448         optparse.make_option("--no-accelerated-2d-canvas",
1449             action="store_false",
1450             dest="accelerated_2d_canvas",
1451             help="Don't use hardware-accelerated 2D Canvas calls"),
1452     ]
1453
1454     # Missing Mac-specific old-run-webkit-tests options:
1455     # FIXME: Need: -g, --guard for guard malloc support on Mac.
1456     # FIXME: Need: -l --leaks    Enable leaks checking.
1457     # FIXME: Need: --sample-on-timeout Run sample on timeout
1458
1459     old_run_webkit_tests_compat = [
1460         # NRWT doesn't generate results by default anyway.
1461         _compat_shim_option("--no-new-test-results"),
1462         # NRWT doesn't sample on timeout yet anyway.
1463         _compat_shim_option("--no-sample-on-timeout"),
1464         # FIXME: NRWT needs to support remote links eventually.
1465         _compat_shim_option("--use-remote-links-to-tests"),
1466         # FIXME: NRWT doesn't need this option as much since failures are
1467         # designed to be cheap.  We eventually plan to add this support.
1468         _compat_shim_option("--exit-after-n-failures", nargs=1, type="int"),
1469     ]
1470
1471     results_options = [
1472         # NEED for bots: --use-remote-links-to-tests Link to test files
1473         # within the SVN repository in the results.
1474         optparse.make_option("-p", "--pixel-tests", action="store_true",
1475             dest="pixel_tests", help="Enable pixel-to-pixel PNG comparisons"),
1476         optparse.make_option("--no-pixel-tests", action="store_false",
1477             dest="pixel_tests", help="Disable pixel-to-pixel PNG comparisons"),
1478         optparse.make_option("--tolerance",
1479             help="Ignore image differences less than this percentage (some "
1480                 "ports may ignore this option)", type="float"),
1481         optparse.make_option("--results-directory",
1482             default="layout-test-results",
1483             help="Output results directory source dir, relative to Debug or "
1484                  "Release"),
1485         optparse.make_option("--new-baseline", action="store_true",
1486             default=False, help="Save all generated results as new baselines "
1487                  "into the platform directory, overwriting whatever's "
1488                  "already there."),
1489         optparse.make_option("--reset-results", action="store_true",
1490             default=False, help="Reset any existing baselines to the "
1491                  "generated results"),
1492         optparse.make_option("--no-show-results", action="store_false",
1493             default=True, dest="show_results",
1494             help="Don't launch a browser with results after the tests "
1495                  "are done"),
1496         # FIXME: We should have a helper function to do this sort of
1497         # deprectated mapping and automatically log, etc.
1498         optparse.make_option("--noshow-results", action="store_false",
1499             dest="show_results",
1500             help="Deprecated, same as --no-show-results."),
1501         optparse.make_option("--no-launch-safari", action="store_false",
1502             dest="show_results",
1503             help="old-run-webkit-tests compat, same as --noshow-results."),
1504         # old-run-webkit-tests:
1505         # --[no-]launch-safari    Launch (or do not launch) Safari to display
1506         #                         test results (default: launch)
1507         optparse.make_option("--full-results-html", action="store_true",
1508             default=False,
1509             help="Show all failures in results.html, rather than only "
1510                  "regressions"),
1511         optparse.make_option("--clobber-old-results", action="store_true",
1512             default=False, help="Clobbers test results from previous runs."),
1513         optparse.make_option("--platform",
1514             help="Override the platform for expected results"),
1515         optparse.make_option("--no-record-results", action="store_false",
1516             default=True, dest="record_results",
1517             help="Don't record the results."),
1518         # old-run-webkit-tests also has HTTP toggle options:
1519         # --[no-]http                     Run (or do not run) http tests
1520         #                                 (default: run)
1521     ]
1522
1523     test_options = [
1524         optparse.make_option("--build", dest="build",
1525             action="store_true", default=True,
1526             help="Check to ensure the DumpRenderTree build is up-to-date "
1527                  "(default)."),
1528         optparse.make_option("--no-build", dest="build",
1529             action="store_false", help="Don't check to see if the "
1530                                        "DumpRenderTree build is up-to-date."),
1531         optparse.make_option("-n", "--dry-run", action="store_true",
1532             default=False,
1533             help="Do everything but actually run the tests or upload results."),
1534         # old-run-webkit-tests has --valgrind instead of wrapper.
1535         optparse.make_option("--wrapper",
1536             help="wrapper command to insert before invocations of "
1537                  "DumpRenderTree; option is split on whitespace before "
1538                  "running. (Example: --wrapper='valgrind --smc-check=all')"),
1539         # old-run-webkit-tests:
1540         # -i|--ignore-tests               Comma-separated list of directories
1541         #                                 or tests to ignore
1542         optparse.make_option("--test-list", action="append",
1543             help="read list of tests to run from file", metavar="FILE"),
1544         # old-run-webkit-tests uses --skipped==[default|ignore|only]
1545         # instead of --force:
1546         optparse.make_option("--force", action="store_true", default=False,
1547             help="Run all tests, even those marked SKIP in the test list"),
1548         optparse.make_option("--use-apache", action="store_true",
1549             default=False, help="Whether to use apache instead of lighttpd."),
1550         optparse.make_option("--time-out-ms",
1551             help="Set the timeout for each test"),
1552         # old-run-webkit-tests calls --randomize-order --random:
1553         optparse.make_option("--randomize-order", action="store_true",
1554             default=False, help=("Run tests in random order (useful "
1555                                 "for tracking down corruption)")),
1556         optparse.make_option("--run-chunk",
1557             help=("Run a specified chunk (n:l), the nth of len l, "
1558                  "of the layout tests")),
1559         optparse.make_option("--run-part", help=("Run a specified part (n:m), "
1560                   "the nth of m parts, of the layout tests")),
1561         # old-run-webkit-tests calls --batch-size: --nthly n
1562         #   Restart DumpRenderTree every n tests (default: 1000)
1563         optparse.make_option("--batch-size",
1564             help=("Run a the tests in batches (n), after every n tests, "
1565                   "DumpRenderTree is relaunched."), type="int", default=0),
1566         # old-run-webkit-tests calls --run-singly: -1|--singly
1567         # Isolate each test case run (implies --nthly 1 --verbose)
1568         optparse.make_option("--run-singly", action="store_true",
1569             default=False, help="run a separate DumpRenderTree for each test"),
1570         optparse.make_option("--child-processes",
1571             help="Number of DumpRenderTrees to run in parallel."),
1572         # FIXME: Display default number of child processes that will run.
1573         # FIXME: Display default threading model (will be port-specific).
1574         optparse.make_option("--worker-model", action="store",
1575             help=("controls worker model. Valid values are "
1576             "'inline' and 'threads'.")),
1577         optparse.make_option("--experimental-fully-parallel",
1578             action="store_true", default=False,
1579             help="run all tests in parallel"),
1580         # FIXME: Need --exit-after-n-failures N
1581         #      Exit after the first N failures instead of running all tests
1582         # FIXME: Need --exit-after-n-crashes N
1583         #      Exit after the first N crashes instead of running all tests
1584         # FIXME: consider: --iterations n
1585         #      Number of times to run the set of tests (e.g. ABCABCABC)
1586         optparse.make_option("--print-last-failures", action="store_true",
1587             default=False, help="Print the tests in the last run that "
1588             "had unexpected failures (or passes) and then exit."),
1589         optparse.make_option("--retest-last-failures", action="store_true",
1590             default=False, help="re-test the tests in the last run that "
1591             "had unexpected failures (or passes)."),
1592         optparse.make_option("--retry-failures", action="store_true",
1593             default=True,
1594             help="Re-try any tests that produce unexpected results (default)"),
1595         optparse.make_option("--no-retry-failures", action="store_false",
1596             dest="retry_failures",
1597             help="Don't re-try any tests that produce unexpected results."),
1598     ]
1599
1600     misc_options = [
1601         optparse.make_option("--lint-test-files", action="store_true",
1602         default=False, help=("Makes sure the test files parse for all "
1603                             "configurations. Does not run any tests.")),
1604     ]
1605
1606     # FIXME: Move these into json_results_generator.py
1607     results_json_options = [
1608         optparse.make_option("--master-name", help="The name of the buildbot master."),
1609         optparse.make_option("--builder-name", default="DUMMY_BUILDER_NAME",
1610             help=("The name of the builder shown on the waterfall running "
1611                   "this script e.g. WebKit.")),
1612         optparse.make_option("--build-name", default="DUMMY_BUILD_NAME",
1613             help=("The name of the builder used in its path, e.g. "
1614                   "webkit-rel.")),
1615         optparse.make_option("--build-number", default="DUMMY_BUILD_NUMBER",
1616             help=("The build number of the builder running this script.")),
1617         optparse.make_option("--test-results-server", default="",
1618             help=("If specified, upload results json files to this appengine "
1619                   "server.")),
1620         optparse.make_option("--upload-full-results",
1621             action="store_true",
1622             default=False,
1623             help="If true, upload full json results to server."),
1624     ]
1625
1626     option_list = (configuration_options + print_options +
1627                    chromium_options + results_options + test_options +
1628                    misc_options + results_json_options +
1629                    old_run_webkit_tests_compat)
1630     option_parser = optparse.OptionParser(option_list=option_list)
1631
1632     return option_parser.parse_args(args)
1633
1634
1635 def main():
1636     options, args = parse_args()
1637     port_obj = port.get(options.platform, options)
1638     return run(port_obj, options, args)
1639
1640
1641 if '__main__' == __name__:
1642     try:
1643         sys.exit(main())
1644     except KeyboardInterrupt:
1645         # this mirrors what the shell normally does
1646         sys.exit(signal.SIGINT + 128)