Tools/Scripts/webkitpy/layout_tests/run_webkit_tests.py

   1 #!/usr/bin/env python
   2 # Copyright (C) 2010 Google Inc. All rights reserved.
   3 # Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Szeged
   4 #
   5 # Redistribution and use in source and binary forms, with or without
   6 # modification, are permitted provided that the following conditions are
   7 # met:
   8 #
   9 #     * Redistributions of source code must retain the above copyright
  10 # notice, this list of conditions and the following disclaimer.
  11 #     * Redistributions in binary form must reproduce the above
  12 # copyright notice, this list of conditions and the following disclaimer
  13 # in the documentation and/or other materials provided with the
  14 # distribution.
  15 #     * Neither the name of Google Inc. nor the names of its
  16 # contributors may be used to endorse or promote products derived from
  17 # this software without specific prior written permission.
  18 #
  19 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  20 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  21 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  22 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  23 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  24 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  25 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  26 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  27 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  28 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  29 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  30
  31 """Run layout tests.
  32
  33 This is a port of the existing webkit test script run-webkit-tests.
  34
  35 The TestRunner class runs a series of tests (TestType interface) against a set
  36 of test files.  If a test file fails a TestType, it returns a list TestFailure
  37 objects to the TestRunner.  The TestRunner then aggregates the TestFailures to
  38 create a final report.
  39
  40 This script reads several files, if they exist in the test_lists subdirectory
  41 next to this script itself.  Each should contain a list of paths to individual
  42 tests or entire subdirectories of tests, relative to the outermost test
  43 directory.  Entire lines starting with '//' (comments) will be ignored.
  44
  45 For details of the files' contents and purposes, see test_lists/README.
  46 """
  47
  48 from __future__ import with_statement
  49
  50 import codecs
  51 import errno
  52 import glob
  53 import logging
  54 import math
  55 import optparse
  56 import os
  57 import platform
  58 import Queue
  59 import random
  60 import re
  61 import shutil
  62 import signal
  63 import sys
  64 import time
  65 import traceback
  66
  67 from layout_package import dump_render_tree_thread
  68 from layout_package import json_layout_results_generator
  69 from layout_package import message_broker
  70 from layout_package import printing
  71 from layout_package import test_expectations
  72 from layout_package import test_failures
  73 from layout_package import test_results
  74 from layout_package import test_results_uploader
  75
  76 from webkitpy.common.system import user
  77 from webkitpy.thirdparty import simplejson
  78 from webkitpy.tool import grammar
  79
  80 import port
  81
  82 _log = logging.getLogger("webkitpy.layout_tests.run_webkit_tests")
  83
  84 # Builder base URL where we have the archived test results.
  85 BUILDER_BASE_URL = "http://build.chromium.org/buildbot/layout_test_results/"
  86
  87 LAYOUT_TESTS_DIRECTORY = "LayoutTests" + os.sep
  88
  89 TestExpectationsFile = test_expectations.TestExpectationsFile
  90
  91
  92 class TestInput:
  93     """Groups information about a test for easy passing of data."""
  94
  95     def __init__(self, filename, timeout):
  96         """Holds the input parameters for a test.
  97         Args:
  98           filename: Full path to the test.
  99           timeout: Timeout in msecs the driver should use while running the test
 100           """
 101         # FIXME: filename should really be test_name as a relative path.
 102         self.filename = filename
 103         self.timeout = timeout
 104         # The image_hash is used to avoid doing an image dump if the
 105         # checksums match. The image_hash is set later, and only if it is needed
 106         # for the test.
 107         self.image_hash = None
 108
 109
 110 class TestRunInterruptedException(Exception):
 111     """Raised when a test run should be stopped immediately."""
 112     def __init__(self, reason):
 113         self.reason = reason
 114
 115
 116 class ResultSummary(object):
 117     """A class for partitioning the test results we get into buckets.
 118
 119     This class is basically a glorified struct and it's private to this file
 120     so we don't bother with any information hiding."""
 121
 122     def __init__(self, expectations, test_files):
 123         self.total = len(test_files)
 124         self.remaining = self.total
 125         self.expectations = expectations
 126         self.expected = 0
 127         self.unexpected = 0
 128         self.unexpected_failures = 0
 129         self.unexpected_crashes_or_timeouts = 0
 130         self.tests_by_expectation = {}
 131         self.tests_by_timeline = {}
 132         self.results = {}
 133         self.unexpected_results = {}
 134         self.failures = {}
 135         self.tests_by_expectation[test_expectations.SKIP] = set()
 136         for expectation in TestExpectationsFile.EXPECTATIONS.values():
 137             self.tests_by_expectation[expectation] = set()
 138         for timeline in TestExpectationsFile.TIMELINES.values():
 139             self.tests_by_timeline[timeline] = (
 140                 expectations.get_tests_with_timeline(timeline))
 141
 142     def add(self, result, expected):
 143         """Add a TestResult into the appropriate bin.
 144
 145         Args:
 146           result: TestResult
 147           expected: whether the result was what we expected it to be.
 148         """
 149
 150         self.tests_by_expectation[result.type].add(result.filename)
 151         self.results[result.filename] = result
 152         self.remaining -= 1
 153         if len(result.failures):
 154             self.failures[result.filename] = result.failures
 155         if expected:
 156             self.expected += 1
 157         else:
 158             self.unexpected_results[result.filename] = result.type
 159             self.unexpected += 1
 160             if len(result.failures):
 161                 self.unexpected_failures += 1
 162             if result.type == test_expectations.CRASH or result.type == test_expectations.TIMEOUT:
 163                 self.unexpected_crashes_or_timeouts += 1
 164
 165
 166 def summarize_unexpected_results(port_obj, expectations, result_summary,
 167                                  retry_summary):
 168     """Summarize any unexpected results as a dict.
 169
 170     FIXME: split this data structure into a separate class?
 171
 172     Args:
 173         port_obj: interface to port-specific hooks
 174         expectations: test_expectations.TestExpectations object
 175         result_summary: summary object from initial test runs
 176         retry_summary: summary object from final test run of retried tests
 177     Returns:
 178         A dictionary containing a summary of the unexpected results from the
 179         run, with the following fields:
 180         'version': a version indicator (1 in this version)
 181         'fixable': # of fixable tests (NOW - PASS)
 182         'skipped': # of skipped tests (NOW & SKIPPED)
 183         'num_regressions': # of non-flaky failures
 184         'num_flaky': # of flaky failures
 185         'num_passes': # of unexpected passes
 186         'tests': a dict of tests -> {'expected': '...', 'actual': '...'}
 187     """
 188     results = {}
 189     results['version'] = 1
 190
 191     tbe = result_summary.tests_by_expectation
 192     tbt = result_summary.tests_by_timeline
 193     results['fixable'] = len(tbt[test_expectations.NOW] -
 194                                 tbe[test_expectations.PASS])
 195     results['skipped'] = len(tbt[test_expectations.NOW] &
 196                                 tbe[test_expectations.SKIP])
 197
 198     num_passes = 0
 199     num_flaky = 0
 200     num_regressions = 0
 201     keywords = {}
 202     for k, v in TestExpectationsFile.EXPECTATIONS.iteritems():
 203         keywords[v] = k.upper()
 204
 205     tests = {}
 206     for filename, result in result_summary.unexpected_results.iteritems():
 207         # Note that if a test crashed in the original run, we ignore
 208         # whether or not it crashed when we retried it (if we retried it),
 209         # and always consider the result not flaky.
 210         test = port_obj.relative_test_filename(filename)
 211         expected = expectations.get_expectations_string(filename)
 212         actual = [keywords[result]]
 213
 214         if result == test_expectations.PASS:
 215             num_passes += 1
 216         elif result == test_expectations.CRASH:
 217             num_regressions += 1
 218         else:
 219             if filename not in retry_summary.unexpected_results:
 220                 actual.extend(expectations.get_expectations_string(
 221                     filename).split(" "))
 222                 num_flaky += 1
 223             else:
 224                 retry_result = retry_summary.unexpected_results[filename]
 225                 if result != retry_result:
 226                     actual.append(keywords[retry_result])
 227                     num_flaky += 1
 228                 else:
 229                     num_regressions += 1
 230
 231         tests[test] = {}
 232         tests[test]['expected'] = expected
 233         tests[test]['actual'] = " ".join(actual)
 234
 235     results['tests'] = tests
 236     results['num_passes'] = num_passes
 237     results['num_flaky'] = num_flaky
 238     results['num_regressions'] = num_regressions
 239
 240     return results
 241
 242
 243 class TestRunner:
 244     """A class for managing running a series of tests on a series of layout
 245     test files."""
 246
 247     HTTP_SUBDIR = os.sep.join(['', 'http', ''])
 248     WEBSOCKET_SUBDIR = os.sep.join(['', 'websocket', ''])
 249
 250     # The per-test timeout in milliseconds, if no --time-out-ms option was
 251     # given to run_webkit_tests. This should correspond to the default timeout
 252     # in DumpRenderTree.
 253     DEFAULT_TEST_TIMEOUT_MS = 6 * 1000
 254
 255     def __init__(self, port, options, printer):
 256         """Initialize test runner data structures.
 257
 258         Args:
 259           port: an object implementing port-specific
 260           options: a dictionary of command line options
 261           printer: a Printer object to record updates to.
 262         """
 263         self._port = port
 264         self._options = options
 265         self._printer = printer
 266         self._message_broker = None
 267
 268         # disable wss server. need to install pyOpenSSL on buildbots.
 269         # self._websocket_secure_server = websocket_server.PyWebSocket(
 270         #        options.results_directory, use_tls=True, port=9323)
 271
 272         # a set of test files, and the same tests as a list
 273         self._test_files = set()
 274         self._test_files_list = None
 275         self._result_queue = Queue.Queue()
 276         self._retrying = False
 277
 278     def collect_tests(self, args, last_unexpected_results):
 279         """Find all the files to test.
 280
 281         Args:
 282           args: list of test arguments from the command line
 283           last_unexpected_results: list of unexpected results to retest, if any
 284
 285         """
 286         paths = [self._strip_test_dir_prefix(arg) for arg in args if arg and arg != '']
 287         paths += last_unexpected_results
 288         if self._options.test_list:
 289             paths += read_test_files(self._options.test_list)
 290         self._test_files = self._port.tests(paths)
 291
 292     def _strip_test_dir_prefix(self, path):
 293         if path.startswith(LAYOUT_TESTS_DIRECTORY):
 294             return path[len(LAYOUT_TESTS_DIRECTORY):]
 295         return path
 296
 297     def lint(self):
 298         lint_failed = False
 299
 300         # Creating the expecations for each platform/configuration pair does
 301         # all the test list parsing and ensures it's correct syntax (e.g. no
 302         # dupes).
 303         for platform_name in self._port.test_platform_names():
 304             try:
 305                 self.parse_expectations(platform_name, is_debug_mode=True)
 306             except test_expectations.ParseError:
 307                 lint_failed = True
 308             try:
 309                 self.parse_expectations(platform_name, is_debug_mode=False)
 310             except test_expectations.ParseError:
 311                 lint_failed = True
 312
 313         self._printer.write("")
 314         if lint_failed:
 315             _log.error("Lint failed.")
 316             return -1
 317
 318         _log.info("Lint succeeded.")
 319         return 0
 320
 321     def parse_expectations(self, test_platform_name, is_debug_mode):
 322         """Parse the expectations from the test_list files and return a data
 323         structure holding them. Throws an error if the test_list files have
 324         invalid syntax."""
 325         if self._options.lint_test_files:
 326             test_files = None
 327         else:
 328             test_files = self._test_files
 329
 330         expectations_str = self._port.test_expectations()
 331         overrides_str = self._port.test_expectations_overrides()
 332         self._expectations = test_expectations.TestExpectations(
 333             self._port, test_files, expectations_str, test_platform_name,
 334             is_debug_mode, self._options.lint_test_files,
 335             overrides=overrides_str)
 336         return self._expectations
 337
 338
 339     def prepare_lists_and_print_output(self):
 340         """Create appropriate subsets of test lists and returns a
 341         ResultSummary object. Also prints expected test counts.
 342         """
 343
 344         # Remove skipped - both fixable and ignored - files from the
 345         # top-level list of files to test.
 346         num_all_test_files = len(self._test_files)
 347         self._printer.print_expected("Found:  %d tests" %
 348                                      (len(self._test_files)))
 349         if not num_all_test_files:
 350             _log.critical('No tests to run.')
 351             return None
 352
 353         skipped = set()
 354         if num_all_test_files > 1 and not self._options.force:
 355             skipped = self._expectations.get_tests_with_result_type(
 356                            test_expectations.SKIP)
 357             self._test_files -= skipped
 358
 359         # Create a sorted list of test files so the subset chunk,
 360         # if used, contains alphabetically consecutive tests.
 361         self._test_files_list = list(self._test_files)
 362         if self._options.randomize_order:
 363             random.shuffle(self._test_files_list)
 364         else:
 365             self._test_files_list.sort()
 366
 367         # If the user specifies they just want to run a subset of the tests,
 368         # just grab a subset of the non-skipped tests.
 369         if self._options.run_chunk or self._options.run_part:
 370             chunk_value = self._options.run_chunk or self._options.run_part
 371             test_files = self._test_files_list
 372             try:
 373                 (chunk_num, chunk_len) = chunk_value.split(":")
 374                 chunk_num = int(chunk_num)
 375                 assert(chunk_num >= 0)
 376                 test_size = int(chunk_len)
 377                 assert(test_size > 0)
 378             except:
 379                 _log.critical("invalid chunk '%s'" % chunk_value)
 380                 return None
 381
 382             # Get the number of tests
 383             num_tests = len(test_files)
 384
 385             # Get the start offset of the slice.
 386             if self._options.run_chunk:
 387                 chunk_len = test_size
 388                 # In this case chunk_num can be really large. We need
 389                 # to make the slave fit in the current number of tests.
 390                 slice_start = (chunk_num * chunk_len) % num_tests
 391             else:
 392                 # Validate the data.
 393                 assert(test_size <= num_tests)
 394                 assert(chunk_num <= test_size)
 395
 396                 # To count the chunk_len, and make sure we don't skip
 397                 # some tests, we round to the next value that fits exactly
 398                 # all the parts.
 399                 rounded_tests = num_tests
 400                 if rounded_tests % test_size != 0:
 401                     rounded_tests = (num_tests + test_size -
 402                                      (num_tests % test_size))
 403
 404                 chunk_len = rounded_tests / test_size
 405                 slice_start = chunk_len * (chunk_num - 1)
 406                 # It does not mind if we go over test_size.
 407
 408             # Get the end offset of the slice.
 409             slice_end = min(num_tests, slice_start + chunk_len)
 410
 411             files = test_files[slice_start:slice_end]
 412
 413             tests_run_msg = 'Running: %d tests (chunk slice [%d:%d] of %d)' % (
 414                 (slice_end - slice_start), slice_start, slice_end, num_tests)
 415             self._printer.print_expected(tests_run_msg)
 416
 417             # If we reached the end and we don't have enough tests, we run some
 418             # from the beginning.
 419             if slice_end - slice_start < chunk_len:
 420                 extra = chunk_len - (slice_end - slice_start)
 421                 extra_msg = ('   last chunk is partial, appending [0:%d]' %
 422                             extra)
 423                 self._printer.print_expected(extra_msg)
 424                 tests_run_msg += "\n" + extra_msg
 425                 files.extend(test_files[0:extra])
 426             tests_run_filename = os.path.join(self._options.results_directory,
 427                                               "tests_run.txt")
 428             with codecs.open(tests_run_filename, "w", "utf-8") as file:
 429                 file.write(tests_run_msg + "\n")
 430
 431             len_skip_chunk = int(len(files) * len(skipped) /
 432                                  float(len(self._test_files)))
 433             skip_chunk_list = list(skipped)[0:len_skip_chunk]
 434             skip_chunk = set(skip_chunk_list)
 435
 436             # Update expectations so that the stats are calculated correctly.
 437             # We need to pass a list that includes the right # of skipped files
 438             # to ParseExpectations so that ResultSummary() will get the correct
 439             # stats. So, we add in the subset of skipped files, and then
 440             # subtract them back out.
 441             self._test_files_list = files + skip_chunk_list
 442             self._test_files = set(self._test_files_list)
 443
 444             self._expectations = self.parse_expectations(
 445                 self._port.test_platform_name(),
 446                 self._options.configuration == 'Debug')
 447
 448             self._test_files = set(files)
 449             self._test_files_list = files
 450         else:
 451             skip_chunk = skipped
 452
 453         result_summary = ResultSummary(self._expectations,
 454             self._test_files | skip_chunk)
 455         self._print_expected_results_of_type(result_summary,
 456             test_expectations.PASS, "passes")
 457         self._print_expected_results_of_type(result_summary,
 458             test_expectations.FAIL, "failures")
 459         self._print_expected_results_of_type(result_summary,
 460             test_expectations.FLAKY, "flaky")
 461         self._print_expected_results_of_type(result_summary,
 462             test_expectations.SKIP, "skipped")
 463
 464         if self._options.force:
 465             self._printer.print_expected('Running all tests, including '
 466                                          'skips (--force)')
 467         else:
 468             # Note that we don't actually run the skipped tests (they were
 469             # subtracted out of self._test_files, above), but we stub out the
 470             # results here so the statistics can remain accurate.
 471             for test in skip_chunk:
 472                 result = test_results.TestResult(test,
 473                     failures=[], test_run_time=0, total_time_for_all_diffs=0,
 474                     time_for_diffs=0)
 475                 result.type = test_expectations.SKIP
 476                 result_summary.add(result, expected=True)
 477         self._printer.print_expected('')
 478
 479         return result_summary
 480
 481     def _get_dir_for_test_file(self, test_file):
 482         """Returns the highest-level directory by which to shard the given
 483         test file."""
 484         index = test_file.rfind(os.sep + LAYOUT_TESTS_DIRECTORY)
 485
 486         test_file = test_file[index + len(LAYOUT_TESTS_DIRECTORY):]
 487         test_file_parts = test_file.split(os.sep, 1)
 488         directory = test_file_parts[0]
 489         test_file = test_file_parts[1]
 490
 491         # The http tests are very stable on mac/linux.
 492         # TODO(ojan): Make the http server on Windows be apache so we can
 493         # turn shard the http tests there as well. Switching to apache is
 494         # what made them stable on linux/mac.
 495         return_value = directory
 496         while ((directory != 'http' or sys.platform in ('darwin', 'linux2'))
 497                 and test_file.find(os.sep) >= 0):
 498             test_file_parts = test_file.split(os.sep, 1)
 499             directory = test_file_parts[0]
 500             return_value = os.path.join(return_value, directory)
 501             test_file = test_file_parts[1]
 502
 503         return return_value
 504
 505     def _get_test_input_for_file(self, test_file):
 506         """Returns the appropriate TestInput object for the file. Mostly this
 507         is used for looking up the timeout value (in ms) to use for the given
 508         test."""
 509         if self._test_is_slow(test_file):
 510             return TestInput(test_file, self._options.slow_time_out_ms)
 511         return TestInput(test_file, self._options.time_out_ms)
 512
 513     def _test_requires_lock(self, test_file):
 514         """Return True if the test needs to be locked when
 515         running multiple copies of NRWTs."""
 516         split_path = test_file.split(os.sep)
 517         return 'http' in split_path or 'websocket' in split_path
 518
 519     def _test_is_slow(self, test_file):
 520         return self._expectations.has_modifier(test_file,
 521                                                test_expectations.SLOW)
 522
 523     def _shard_tests(self, test_files, use_real_shards):
 524         """Groups tests into batches.
 525         This helps ensure that tests that depend on each other (aka bad tests!)
 526         continue to run together as most cross-tests dependencies tend to
 527         occur within the same directory. If use_real_shards is False, we
 528         put each (non-HTTP/websocket) test into its own shard for maximum
 529         concurrency instead of trying to do any sort of real sharding.
 530
 531         Return:
 532             A list of lists of TestInput objects.
 533         """
 534         # FIXME: when we added http locking, we changed how this works such
 535         # that we always lump all of the HTTP threads into a single shard.
 536         # That will slow down experimental-fully-parallel, but it's unclear
 537         # what the best alternative is completely revamping how we track
 538         # when to grab the lock.
 539
 540         test_lists = []
 541         tests_to_http_lock = []
 542         if not use_real_shards:
 543             for test_file in test_files:
 544                 test_input = self._get_test_input_for_file(test_file)
 545                 if self._test_requires_lock(test_file):
 546                     tests_to_http_lock.append(test_input)
 547                 else:
 548                     test_lists.append((".", [test_input]))
 549         else:
 550             tests_by_dir = {}
 551             for test_file in test_files:
 552                 directory = self._get_dir_for_test_file(test_file)
 553                 test_input = self._get_test_input_for_file(test_file)
 554                 if self._test_requires_lock(test_file):
 555                     tests_to_http_lock.append(test_input)
 556                 else:
 557                     tests_by_dir.setdefault(directory, [])
 558                     tests_by_dir[directory].append(test_input)
 559             # Sort by the number of tests in the dir so that the ones with the
 560             # most tests get run first in order to maximize parallelization.
 561             # Number of tests is a good enough, but not perfect, approximation
 562             # of how long that set of tests will take to run. We can't just use
 563             # a PriorityQueue until we move to Python 2.6.
 564             for directory in tests_by_dir:
 565                 test_list = tests_by_dir[directory]
 566                 # Keep the tests in alphabetical order.
 567                 # FIXME: Remove once tests are fixed so they can be run in any
 568                 # order.
 569                 test_list.reverse()
 570                 test_list_tuple = (directory, test_list)
 571                 test_lists.append(test_list_tuple)
 572             test_lists.sort(lambda a, b: cmp(len(b[1]), len(a[1])))
 573
 574         # Put the http tests first. There are only a couple hundred of them,
 575         # but each http test takes a very long time to run, so sorting by the
 576         # number of tests doesn't accurately capture how long they take to run.
 577         if tests_to_http_lock:
 578             tests_to_http_lock.reverse()
 579             test_lists.insert(0, ("tests_to_http_lock", tests_to_http_lock))
 580
 581         return test_lists
 582
 583     def _contains_tests(self, subdir):
 584         for test_file in self._test_files:
 585             if test_file.find(subdir) >= 0:
 586                 return True
 587         return False
 588
 589     def _num_workers(self):
 590         return int(self._options.child_processes)
 591
 592     def _run_tests(self, file_list, result_summary):
 593         """Runs the tests in the file_list.
 594
 595         Return: A tuple (interrupted, keyboard_interrupted, thread_timings,
 596             test_timings, individual_test_timings)
 597             interrupted is whether the run was interrupted
 598             keyboard_interrupted is whether the interruption was because someone
 599               typed Ctrl^C
 600             thread_timings is a list of dicts with the total runtime
 601               of each thread with 'name', 'num_tests', 'total_time' properties
 602             test_timings is a list of timings for each sharded subdirectory
 603               of the form [time, directory_name, num_tests]
 604             individual_test_timings is a list of run times for each test
 605               in the form {filename:filename, test_run_time:test_run_time}
 606             result_summary: summary object to populate with the results
 607         """
 608
 609         self._printer.print_update('Sharding tests ...')
 610         num_workers = self._num_workers()
 611         test_lists = self._shard_tests(file_list,
 612             num_workers > 1 and not self._options.experimental_fully_parallel)
 613         filename_queue = Queue.Queue()
 614         for item in test_lists:
 615             filename_queue.put(item)
 616
 617         self._printer.print_update('Starting %s ...' %
 618                                    grammar.pluralize('worker', num_workers))
 619         self._message_broker = message_broker.get(self._port, self._options)
 620         broker = self._message_broker
 621         self._current_filename_queue = filename_queue
 622         self._current_result_summary = result_summary
 623
 624         if not self._options.dry_run:
 625             threads = broker.start_workers(self)
 626         else:
 627             threads = {}
 628
 629         self._printer.print_update("Starting testing ...")
 630         keyboard_interrupted = False
 631         interrupted = False
 632         if not self._options.dry_run:
 633             try:
 634                 broker.run_message_loop()
 635             except KeyboardInterrupt:
 636                 _log.info("Interrupted, exiting")
 637                 broker.cancel_workers()
 638                 keyboard_interrupted = True
 639                 interrupted = True
 640             except TestRunInterruptedException, e:
 641                 _log.info(e.reason)
 642                 broker.cancel_workers()
 643                 interrupted = True
 644             except:
 645                 # Unexpected exception; don't try to clean up workers.
 646                 _log.info("Exception raised, exiting")
 647                 raise
 648
 649         thread_timings, test_timings, individual_test_timings = \
 650             self._collect_timing_info(threads)
 651
 652         broker.cleanup()
 653         self._message_broker = None
 654         return (interrupted, keyboard_interrupted, thread_timings, test_timings,
 655                 individual_test_timings)
 656
 657     def update(self):
 658         self.update_summary(self._current_result_summary)
 659
 660     def _collect_timing_info(self, threads):
 661         test_timings = {}
 662         individual_test_timings = []
 663         thread_timings = []
 664
 665         for thread in threads:
 666             thread_timings.append({'name': thread.getName(),
 667                                    'num_tests': thread.get_num_tests(),
 668                                    'total_time': thread.get_total_time()})
 669             test_timings.update(thread.get_test_group_timing_stats())
 670             individual_test_timings.extend(thread.get_test_results())
 671
 672         return (thread_timings, test_timings, individual_test_timings)
 673
 674     def needs_http(self):
 675         """Returns whether the test runner needs an HTTP server."""
 676         return self._contains_tests(self.HTTP_SUBDIR)
 677
 678     def needs_websocket(self):
 679         """Returns whether the test runner needs a WEBSOCKET server."""
 680         return self._contains_tests(self.WEBSOCKET_SUBDIR)
 681
 682     def set_up_run(self):
 683         """Configures the system to be ready to run tests.
 684
 685         Returns a ResultSummary object if we should continue to run tests,
 686         or None if we should abort.
 687
 688         """
 689         # This must be started before we check the system dependencies,
 690         # since the helper may do things to make the setup correct.
 691         self._printer.print_update("Starting helper ...")
 692         self._port.start_helper()
 693
 694         # Check that the system dependencies (themes, fonts, ...) are correct.
 695         if not self._options.nocheck_sys_deps:
 696             self._printer.print_update("Checking system dependencies ...")
 697             if not self._port.check_sys_deps(self.needs_http()):
 698                 self._port.stop_helper()
 699                 return None
 700
 701         if self._options.clobber_old_results:
 702             self._clobber_old_results()
 703
 704         # Create the output directory if it doesn't already exist.
 705         self._port.maybe_make_directory(self._options.results_directory)
 706
 707         self._port.setup_test_run()
 708
 709         self._printer.print_update("Preparing tests ...")
 710         result_summary = self.prepare_lists_and_print_output()
 711         if not result_summary:
 712             return None
 713
 714         return result_summary
 715
 716     def run(self, result_summary):
 717         """Run all our tests on all our test files.
 718
 719         For each test file, we run each test type. If there are any failures,
 720         we collect them for reporting.
 721
 722         Args:
 723           result_summary: a summary object tracking the test results.
 724
 725         Return:
 726           The number of unexpected results (0 == success)
 727         """
 728         # gather_test_files() must have been called first to initialize us.
 729         # If we didn't find any files to test, we've errored out already in
 730         # prepare_lists_and_print_output().
 731         assert(len(self._test_files))
 732
 733         start_time = time.time()
 734
 735         interrupted, keyboard_interrupted, thread_timings, test_timings, \
 736             individual_test_timings = (
 737             self._run_tests(self._test_files_list, result_summary))
 738
 739         # We exclude the crashes from the list of results to retry, because
 740         # we want to treat even a potentially flaky crash as an error.
 741         failures = self._get_failures(result_summary, include_crashes=False)
 742         retry_summary = result_summary
 743         while (len(failures) and self._options.retry_failures and
 744             not self._retrying and not interrupted):
 745             _log.info('')
 746             _log.info("Retrying %d unexpected failure(s) ..." % len(failures))
 747             _log.info('')
 748             self._retrying = True
 749             retry_summary = ResultSummary(self._expectations, failures.keys())
 750             # Note that we intentionally ignore the return value here.
 751             self._run_tests(failures.keys(), retry_summary)
 752             failures = self._get_failures(retry_summary, include_crashes=True)
 753
 754         end_time = time.time()
 755
 756         self._print_timing_statistics(end_time - start_time,
 757                                       thread_timings, test_timings,
 758                                       individual_test_timings,
 759                                       result_summary)
 760
 761         self._print_result_summary(result_summary)
 762
 763         sys.stdout.flush()
 764         sys.stderr.flush()
 765
 766         self._printer.print_one_line_summary(result_summary.total,
 767                                              result_summary.expected,
 768                                              result_summary.unexpected)
 769
 770         unexpected_results = summarize_unexpected_results(self._port,
 771             self._expectations, result_summary, retry_summary)
 772         self._printer.print_unexpected_results(unexpected_results)
 773
 774         if (self._options.record_results and not self._options.dry_run and
 775             not interrupted):
 776             # Write the same data to log files and upload generated JSON files
 777             # to appengine server.
 778             self._upload_json_files(unexpected_results, result_summary,
 779                                     individual_test_timings)
 780
 781         # Write the summary to disk (results.html) and display it if requested.
 782         if not self._options.dry_run:
 783             wrote_results = self._write_results_html_file(result_summary)
 784             if self._options.show_results and wrote_results:
 785                 self._show_results_html_file()
 786
 787         # Now that we've completed all the processing we can, we re-raise
 788         # a KeyboardInterrupt if necessary so the caller can handle it.
 789         if keyboard_interrupted:
 790             raise KeyboardInterrupt
 791
 792         # Ignore flaky failures and unexpected passes so we don't turn the
 793         # bot red for those.
 794         return unexpected_results['num_regressions']
 795
 796     def clean_up_run(self):
 797         """Restores the system after we're done running tests."""
 798
 799         _log.debug("flushing stdout")
 800         sys.stdout.flush()
 801         _log.debug("flushing stderr")
 802         sys.stderr.flush()
 803         _log.debug("stopping helper")
 804         self._port.stop_helper()
 805
 806     def update_summary(self, result_summary):
 807         """Update the summary and print results with any completed tests."""
 808         while True:
 809             try:
 810                 result = test_results.TestResult.loads(self._result_queue.get_nowait())
 811             except Queue.Empty:
 812                 return
 813
 814             expected = self._expectations.matches_an_expected_result(
 815                 result.filename, result.type, self._options.pixel_tests)
 816             result_summary.add(result, expected)
 817             exp_str = self._expectations.get_expectations_string(
 818                 result.filename)
 819             got_str = self._expectations.expectation_to_string(result.type)
 820             self._printer.print_test_result(result, expected, exp_str, got_str)
 821             self._printer.print_progress(result_summary, self._retrying,
 822                                          self._test_files_list)
 823
 824             def interrupt_if_at_failure_limit(limit, count, message):
 825                 if limit and count >= limit:
 826                     raise TestRunInterruptedException(message % count)
 827
 828             interrupt_if_at_failure_limit(
 829                 self._options.exit_after_n_failures,
 830                 result_summary.unexpected_failures,
 831                 "Aborting run since %d failures were reached")
 832             interrupt_if_at_failure_limit(
 833                 self._options.exit_after_n_crashes_or_timeouts,
 834                 result_summary.unexpected_crashes_or_timeouts,
 835                 "Aborting run since %d crashes or timeouts were reached")
 836
 837
 838     def _clobber_old_results(self):
 839         # Just clobber the actual test results directories since the other
 840         # files in the results directory are explicitly used for cross-run
 841         # tracking.
 842         self._printer.print_update("Clobbering old results in %s" %
 843                                    self._options.results_directory)
 844         layout_tests_dir = self._port.layout_tests_dir()
 845         possible_dirs = self._port.test_dirs()
 846         for dirname in possible_dirs:
 847             if os.path.isdir(os.path.join(layout_tests_dir, dirname)):
 848                 shutil.rmtree(os.path.join(self._options.results_directory,
 849                                            dirname),
 850                               ignore_errors=True)
 851
 852     def _get_failures(self, result_summary, include_crashes):
 853         """Filters a dict of results and returns only the failures.
 854
 855         Args:
 856           result_summary: the results of the test run
 857           include_crashes: whether crashes are included in the output.
 858             We use False when finding the list of failures to retry
 859             to see if the results were flaky. Although the crashes may also be
 860             flaky, we treat them as if they aren't so that they're not ignored.
 861         Returns:
 862           a dict of files -> results
 863         """
 864         failed_results = {}
 865         for test, result in result_summary.unexpected_results.iteritems():
 866             if (result == test_expectations.PASS or
 867                 result == test_expectations.CRASH and not include_crashes):
 868                 continue
 869             failed_results[test] = result
 870
 871         return failed_results
 872
 873     def _upload_json_files(self, unexpected_results, result_summary,
 874                         individual_test_timings):
 875         """Writes the results of the test run as JSON files into the results
 876         dir and upload the files to the appengine server.
 877
 878         There are three different files written into the results dir:
 879           unexpected_results.json: A short list of any unexpected results.
 880             This is used by the buildbots to display results.
 881           expectations.json: This is used by the flakiness dashboard.
 882           results.json: A full list of the results - used by the flakiness
 883             dashboard and the aggregate results dashboard.
 884
 885         Args:
 886           unexpected_results: dict of unexpected results
 887           result_summary: full summary object
 888           individual_test_timings: list of test times (used by the flakiness
 889             dashboard).
 890         """
 891         results_directory = self._options.results_directory
 892         _log.debug("Writing JSON files in %s." % results_directory)
 893         unexpected_json_path = os.path.join(results_directory, "unexpected_results.json")
 894         with codecs.open(unexpected_json_path, "w", "utf-8") as file:
 895             simplejson.dump(unexpected_results, file, sort_keys=True, indent=2)
 896
 897         # Write a json file of the test_expectations.txt file for the layout
 898         # tests dashboard.
 899         expectations_path = os.path.join(results_directory, "expectations.json")
 900         expectations_json = \
 901             self._expectations.get_expectations_json_for_all_platforms()
 902         with codecs.open(expectations_path, "w", "utf-8") as file:
 903             file.write(u"ADD_EXPECTATIONS(%s);" % expectations_json)
 904
 905         generator = json_layout_results_generator.JSONLayoutResultsGenerator(
 906             self._port, self._options.builder_name, self._options.build_name,
 907             self._options.build_number, self._options.results_directory,
 908             BUILDER_BASE_URL, individual_test_timings,
 909             self._expectations, result_summary, self._test_files_list,
 910             not self._options.upload_full_results,
 911             self._options.test_results_server,
 912             "layout-tests",
 913             self._options.master_name)
 914
 915         _log.debug("Finished writing JSON files.")
 916
 917         json_files = ["expectations.json"]
 918         if self._options.upload_full_results:
 919             json_files.append("results.json")
 920         else:
 921             json_files.append("incremental_results.json")
 922
 923         generator.upload_json_files(json_files)
 924
 925     def _print_config(self):
 926         """Prints the configuration for the test run."""
 927         p = self._printer
 928         p.print_config("Using port '%s'" % self._port.name())
 929         p.print_config("Placing test results in %s" %
 930                        self._options.results_directory)
 931         if self._options.new_baseline:
 932             p.print_config("Placing new baselines in %s" %
 933                            self._port.baseline_path())
 934         p.print_config("Using %s build" % self._options.configuration)
 935         if self._options.pixel_tests:
 936             p.print_config("Pixel tests enabled")
 937         else:
 938             p.print_config("Pixel tests disabled")
 939
 940         p.print_config("Regular timeout: %s, slow test timeout: %s" %
 941                        (self._options.time_out_ms,
 942                         self._options.slow_time_out_ms))
 943
 944         if self._num_workers() == 1:
 945             p.print_config("Running one %s" % self._port.driver_name())
 946         else:
 947             p.print_config("Running %s %ss in parallel" %
 948                            (self._options.child_processes,
 949                             self._port.driver_name()))
 950         p.print_config('Command line: ' +
 951                        ' '.join(self._port.driver_cmd_line()))
 952         p.print_config("Worker model: %s" % self._options.worker_model)
 953         p.print_config("")
 954
 955     def _print_expected_results_of_type(self, result_summary,
 956                                         result_type, result_type_str):
 957         """Print the number of the tests in a given result class.
 958
 959         Args:
 960           result_summary - the object containing all the results to report on
 961           result_type - the particular result type to report in the summary.
 962           result_type_str - a string description of the result_type.
 963         """
 964         tests = self._expectations.get_tests_with_result_type(result_type)
 965         now = result_summary.tests_by_timeline[test_expectations.NOW]
 966         wontfix = result_summary.tests_by_timeline[test_expectations.WONTFIX]
 967
 968         # We use a fancy format string in order to print the data out in a
 969         # nicely-aligned table.
 970         fmtstr = ("Expect: %%5d %%-8s (%%%dd now, %%%dd wontfix)"
 971                   % (self._num_digits(now), self._num_digits(wontfix)))
 972         self._printer.print_expected(fmtstr %
 973             (len(tests), result_type_str, len(tests & now), len(tests & wontfix)))
 974
 975     def _num_digits(self, num):
 976         """Returns the number of digits needed to represent the length of a
 977         sequence."""
 978         ndigits = 1
 979         if len(num):
 980             ndigits = int(math.log10(len(num))) + 1
 981         return ndigits
 982
 983     def _print_timing_statistics(self, total_time, thread_timings,
 984                                directory_test_timings, individual_test_timings,
 985                                result_summary):
 986         """Record timing-specific information for the test run.
 987
 988         Args:
 989           total_time: total elapsed time (in seconds) for the test run
 990           thread_timings: wall clock time each thread ran for
 991           directory_test_timings: timing by directory
 992           individual_test_timings: timing by file
 993           result_summary: summary object for the test run
 994         """
 995         self._printer.print_timing("Test timing:")
 996         self._printer.print_timing("  %6.2f total testing time" % total_time)
 997         self._printer.print_timing("")
 998         self._printer.print_timing("Thread timing:")
 999         cuml_time = 0
1000         for t in thread_timings:
1001             self._printer.print_timing("    %10s: %5d tests, %6.2f secs" %
1002                   (t['name'], t['num_tests'], t['total_time']))
1003             cuml_time += t['total_time']
1004         self._printer.print_timing("   %6.2f cumulative, %6.2f optimal" %
1005               (cuml_time, cuml_time / int(self._options.child_processes)))
1006         self._printer.print_timing("")
1007
1008         self._print_aggregate_test_statistics(individual_test_timings)
1009         self._print_individual_test_times(individual_test_timings,
1010                                           result_summary)
1011         self._print_directory_timings(directory_test_timings)
1012
1013     def _print_aggregate_test_statistics(self, individual_test_timings):
1014         """Prints aggregate statistics (e.g. median, mean, etc.) for all tests.
1015         Args:
1016           individual_test_timings: List of TestResults for all tests.
1017         """
1018         test_types = []  # Unit tests don't actually produce any timings.
1019         if individual_test_timings:
1020             test_types = individual_test_timings[0].time_for_diffs.keys()
1021         times_for_dump_render_tree = []
1022         times_for_diff_processing = []
1023         times_per_test_type = {}
1024         for test_type in test_types:
1025             times_per_test_type[test_type] = []
1026
1027         for test_stats in individual_test_timings:
1028             times_for_dump_render_tree.append(test_stats.test_run_time)
1029             times_for_diff_processing.append(
1030                 test_stats.total_time_for_all_diffs)
1031             time_for_diffs = test_stats.time_for_diffs
1032             for test_type in test_types:
1033                 times_per_test_type[test_type].append(
1034                     time_for_diffs[test_type])
1035
1036         self._print_statistics_for_test_timings(
1037             "PER TEST TIME IN TESTSHELL (seconds):",
1038             times_for_dump_render_tree)
1039         self._print_statistics_for_test_timings(
1040             "PER TEST DIFF PROCESSING TIMES (seconds):",
1041             times_for_diff_processing)
1042         for test_type in test_types:
1043             self._print_statistics_for_test_timings(
1044                 "PER TEST TIMES BY TEST TYPE: %s" % test_type,
1045                 times_per_test_type[test_type])
1046
1047     def _print_individual_test_times(self, individual_test_timings,
1048                                   result_summary):
1049         """Prints the run times for slow, timeout and crash tests.
1050         Args:
1051           individual_test_timings: List of TestStats for all tests.
1052           result_summary: summary object for test run
1053         """
1054         # Reverse-sort by the time spent in DumpRenderTree.
1055         individual_test_timings.sort(lambda a, b:
1056             cmp(b.test_run_time, a.test_run_time))
1057
1058         num_printed = 0
1059         slow_tests = []
1060         timeout_or_crash_tests = []
1061         unexpected_slow_tests = []
1062         for test_tuple in individual_test_timings:
1063             filename = test_tuple.filename
1064             is_timeout_crash_or_slow = False
1065             if self._test_is_slow(filename):
1066                 is_timeout_crash_or_slow = True
1067                 slow_tests.append(test_tuple)
1068
1069             if filename in result_summary.failures:
1070                 result = result_summary.results[filename].type
1071                 if (result == test_expectations.TIMEOUT or
1072                     result == test_expectations.CRASH):
1073                     is_timeout_crash_or_slow = True
1074                     timeout_or_crash_tests.append(test_tuple)
1075
1076             if (not is_timeout_crash_or_slow and
1077                 num_printed < printing.NUM_SLOW_TESTS_TO_LOG):
1078                 num_printed = num_printed + 1
1079                 unexpected_slow_tests.append(test_tuple)
1080
1081         self._printer.print_timing("")
1082         self._print_test_list_timing("%s slowest tests that are not "
1083             "marked as SLOW and did not timeout/crash:" %
1084             printing.NUM_SLOW_TESTS_TO_LOG, unexpected_slow_tests)
1085         self._printer.print_timing("")
1086         self._print_test_list_timing("Tests marked as SLOW:", slow_tests)
1087         self._printer.print_timing("")
1088         self._print_test_list_timing("Tests that timed out or crashed:",
1089                                      timeout_or_crash_tests)
1090         self._printer.print_timing("")
1091
1092     def _print_test_list_timing(self, title, test_list):
1093         """Print timing info for each test.
1094
1095         Args:
1096           title: section heading
1097           test_list: tests that fall in this section
1098         """
1099         if self._printer.disabled('slowest'):
1100             return
1101
1102         self._printer.print_timing(title)
1103         for test_tuple in test_list:
1104             filename = test_tuple.filename[len(
1105                 self._port.layout_tests_dir()) + 1:]
1106             filename = filename.replace('\\', '/')
1107             test_run_time = round(test_tuple.test_run_time, 1)
1108             self._printer.print_timing("  %s took %s seconds" %
1109                                        (filename, test_run_time))
1110
1111     def _print_directory_timings(self, directory_test_timings):
1112         """Print timing info by directory for any directories that
1113         take > 10 seconds to run.
1114
1115         Args:
1116           directory_test_timing: time info for each directory
1117         """
1118         timings = []
1119         for directory in directory_test_timings:
1120             num_tests, time_for_directory = directory_test_timings[directory]
1121             timings.append((round(time_for_directory, 1), directory,
1122                             num_tests))
1123         timings.sort()
1124
1125         self._printer.print_timing("Time to process slowest subdirectories:")
1126         min_seconds_to_print = 10
1127         for timing in timings:
1128             if timing[0] > min_seconds_to_print:
1129                 self._printer.print_timing(
1130                     "  %s took %s seconds to run %s tests." % (timing[1],
1131                     timing[0], timing[2]))
1132         self._printer.print_timing("")
1133
1134     def _print_statistics_for_test_timings(self, title, timings):
1135         """Prints the median, mean and standard deviation of the values in
1136         timings.
1137
1138         Args:
1139           title: Title for these timings.
1140           timings: A list of floats representing times.
1141         """
1142         self._printer.print_timing(title)
1143         timings.sort()
1144
1145         num_tests = len(timings)
1146         if not num_tests:
1147             return
1148         percentile90 = timings[int(.9 * num_tests)]
1149         percentile99 = timings[int(.99 * num_tests)]
1150
1151         if num_tests % 2 == 1:
1152             median = timings[((num_tests - 1) / 2) - 1]
1153         else:
1154             lower = timings[num_tests / 2 - 1]
1155             upper = timings[num_tests / 2]
1156             median = (float(lower + upper)) / 2
1157
1158         mean = sum(timings) / num_tests
1159
1160         for time in timings:
1161             sum_of_deviations = math.pow(time - mean, 2)
1162
1163         std_deviation = math.sqrt(sum_of_deviations / num_tests)
1164         self._printer.print_timing("  Median:          %6.3f" % median)
1165         self._printer.print_timing("  Mean:            %6.3f" % mean)
1166         self._printer.print_timing("  90th percentile: %6.3f" % percentile90)
1167         self._printer.print_timing("  99th percentile: %6.3f" % percentile99)
1168         self._printer.print_timing("  Standard dev:    %6.3f" % std_deviation)
1169         self._printer.print_timing("")
1170
1171     def _print_result_summary(self, result_summary):
1172         """Print a short summary about how many tests passed.
1173
1174         Args:
1175           result_summary: information to log
1176         """
1177         failed = len(result_summary.failures)
1178         skipped = len(
1179             result_summary.tests_by_expectation[test_expectations.SKIP])
1180         total = result_summary.total
1181         passed = total - failed - skipped
1182         pct_passed = 0.0
1183         if total > 0:
1184             pct_passed = float(passed) * 100 / total
1185
1186         self._printer.print_actual("")
1187         self._printer.print_actual("=> Results: %d/%d tests passed (%.1f%%)" %
1188                      (passed, total, pct_passed))
1189         self._printer.print_actual("")
1190         self._print_result_summary_entry(result_summary,
1191             test_expectations.NOW, "Tests to be fixed")
1192
1193         self._printer.print_actual("")
1194         self._print_result_summary_entry(result_summary,
1195             test_expectations.WONTFIX,
1196             "Tests that will only be fixed if they crash (WONTFIX)")
1197         self._printer.print_actual("")
1198
1199     def _print_result_summary_entry(self, result_summary, timeline,
1200                                     heading):
1201         """Print a summary block of results for a particular timeline of test.
1202
1203         Args:
1204           result_summary: summary to print results for
1205           timeline: the timeline to print results for (NOT, WONTFIX, etc.)
1206           heading: a textual description of the timeline
1207         """
1208         total = len(result_summary.tests_by_timeline[timeline])
1209         not_passing = (total -
1210            len(result_summary.tests_by_expectation[test_expectations.PASS] &
1211                result_summary.tests_by_timeline[timeline]))
1212         self._printer.print_actual("=> %s (%d):" % (heading, not_passing))
1213
1214         for result in TestExpectationsFile.EXPECTATION_ORDER:
1215             if result == test_expectations.PASS:
1216                 continue
1217             results = (result_summary.tests_by_expectation[result] &
1218                        result_summary.tests_by_timeline[timeline])
1219             desc = TestExpectationsFile.EXPECTATION_DESCRIPTIONS[result]
1220             if not_passing and len(results):
1221                 pct = len(results) * 100.0 / not_passing
1222                 self._printer.print_actual("  %5d %-24s (%4.1f%%)" %
1223                     (len(results), desc[len(results) != 1], pct))
1224
1225     def _results_html(self, test_files, failures, title="Test Failures", override_time=None):
1226         """
1227         test_files = a list of file paths
1228         failures = dictionary mapping test paths to failure objects
1229         title = title printed at top of test
1230         override_time = current time (used by unit tests)
1231         """
1232         page = """<html>
1233   <head>
1234     <title>Layout Test Results (%(time)s)</title>
1235   </head>
1236   <body>
1237     <h2>%(title)s (%(time)s)</h2>
1238         """ % {'title': title, 'time': override_time or time.asctime()}
1239
1240         for test_file in sorted(test_files):
1241             test_name = self._port.relative_test_filename(test_file)
1242             test_url = self._port.filename_to_uri(test_file)
1243             page += u"<p><a href='%s'>%s</a><br />\n" % (test_url, test_name)
1244             test_failures = failures.get(test_file, [])
1245             for failure in test_failures:
1246                 page += (u"&nbsp;&nbsp;%s<br/>" %
1247                          failure.result_html_output(test_name))
1248             page += "</p>\n"
1249         page += "</body></html>\n"
1250         return page
1251
1252     def _write_results_html_file(self, result_summary):
1253         """Write results.html which is a summary of tests that failed.
1254
1255         Args:
1256           result_summary: a summary of the results :)
1257
1258         Returns:
1259           True if any results were written (since expected failures may be
1260           omitted)
1261         """
1262         # test failures
1263         if self._options.full_results_html:
1264             results_title = "Test Failures"
1265             test_files = result_summary.failures.keys()
1266         else:
1267             results_title = "Unexpected Test Failures"
1268             unexpected_failures = self._get_failures(result_summary,
1269                 include_crashes=True)
1270             test_files = unexpected_failures.keys()
1271         if not len(test_files):
1272             return False
1273
1274         out_filename = os.path.join(self._options.results_directory,
1275                                     "results.html")
1276         with codecs.open(out_filename, "w", "utf-8") as results_file:
1277             html = self._results_html(test_files, result_summary.failures, results_title)
1278             results_file.write(html)
1279
1280         return True
1281
1282     def _show_results_html_file(self):
1283         """Shows the results.html page."""
1284         results_filename = os.path.join(self._options.results_directory,
1285                                         "results.html")
1286         self._port.show_results_html_file(results_filename)
1287
1288
1289 def read_test_files(files):
1290     tests = []
1291     for file in files:
1292         try:
1293             with codecs.open(file, 'r', 'utf-8') as file_contents:
1294                 # FIXME: This could be cleaner using a list comprehension.
1295                 for line in file_contents:
1296                     line = test_expectations.strip_comments(line)
1297                     if line:
1298                         tests.append(line)
1299         except IOError, e:
1300             if e.errno == errno.ENOENT:
1301                 _log.critical('')
1302                 _log.critical('--test-list file "%s" not found' % file)
1303             raise
1304     return tests
1305
1306
1307 def run(port, options, args, regular_output=sys.stderr,
1308         buildbot_output=sys.stdout):
1309     """Run the tests.
1310
1311     Args:
1312       port: Port object for port-specific behavior
1313       options: a dictionary of command line options
1314       args: a list of sub directories or files to test
1315       regular_output: a stream-like object that we can send logging/debug
1316           output to
1317       buildbot_output: a stream-like object that we can write all output that
1318           is intended to be parsed by the buildbot to
1319     Returns:
1320       the number of unexpected results that occurred, or -1 if there is an
1321           error.
1322
1323     """
1324     warnings = _set_up_derived_options(port, options)
1325
1326     printer = printing.Printer(port, options, regular_output, buildbot_output,
1327         int(options.child_processes), options.experimental_fully_parallel)
1328     for w in warnings:
1329         _log.warning(w)
1330
1331     if options.help_printing:
1332         printer.help_printing()
1333         printer.cleanup()
1334         return 0
1335
1336     last_unexpected_results = _gather_unexpected_results(options)
1337     if options.print_last_failures:
1338         printer.write("\n".join(last_unexpected_results) + "\n")
1339         printer.cleanup()
1340         return 0
1341
1342     # We wrap any parts of the run that are slow or likely to raise exceptions
1343     # in a try/finally to ensure that we clean up the logging configuration.
1344     num_unexpected_results = -1
1345     try:
1346         test_runner = TestRunner(port, options, printer)
1347         test_runner._print_config()
1348
1349         printer.print_update("Collecting tests ...")
1350         try:
1351             test_runner.collect_tests(args, last_unexpected_results)
1352         except IOError, e:
1353             if e.errno == errno.ENOENT:
1354                 return -1
1355             raise
1356
1357         printer.print_update("Parsing expectations ...")
1358         if options.lint_test_files:
1359             return test_runner.lint()
1360         test_runner.parse_expectations(port.test_platform_name(),
1361                                        options.configuration == 'Debug')
1362
1363         printer.print_update("Checking build ...")
1364         if not port.check_build(test_runner.needs_http()):
1365             _log.error("Build check failed")
1366             return -1
1367
1368         result_summary = test_runner.set_up_run()
1369         if result_summary:
1370             num_unexpected_results = test_runner.run(result_summary)
1371             test_runner.clean_up_run()
1372             _log.debug("Testing completed, Exit status: %d" %
1373                        num_unexpected_results)
1374     finally:
1375         printer.cleanup()
1376
1377     return num_unexpected_results
1378
1379
1380 def _set_up_derived_options(port_obj, options):
1381     """Sets the options values that depend on other options values."""
1382     # We return a list of warnings to print after the printer is initialized.
1383     warnings = []
1384
1385     if options.worker_model == 'old-inline':
1386         if options.child_processes and int(options.child_processes) > 1:
1387             warnings.append("--worker-model=old-inline overrides --child-processes")
1388         options.child_processes = "1"
1389     if not options.child_processes:
1390         options.child_processes = os.environ.get("WEBKIT_TEST_CHILD_PROCESSES",
1391                                                  str(port_obj.default_child_processes()))
1392
1393     if not options.configuration:
1394         options.configuration = port_obj.default_configuration()
1395
1396     if options.pixel_tests is None:
1397         options.pixel_tests = True
1398
1399     if not options.use_apache:
1400         options.use_apache = sys.platform in ('darwin', 'linux2')
1401
1402     if not os.path.isabs(options.results_directory):
1403         # This normalizes the path to the build dir.
1404         # FIXME: how this happens is not at all obvious; this is a dumb
1405         # interface and should be cleaned up.
1406         options.results_directory = port_obj.results_directory()
1407
1408     if not options.time_out_ms:
1409         if options.configuration == "Debug":
1410             options.time_out_ms = str(2 * TestRunner.DEFAULT_TEST_TIMEOUT_MS)
1411         else:
1412             options.time_out_ms = str(TestRunner.DEFAULT_TEST_TIMEOUT_MS)
1413
1414     options.slow_time_out_ms = str(5 * int(options.time_out_ms))
1415     return warnings
1416
1417
1418 def _gather_unexpected_results(options):
1419     """Returns the unexpected results from the previous run, if any."""
1420     last_unexpected_results = []
1421     if options.print_last_failures or options.retest_last_failures:
1422         unexpected_results_filename = os.path.join(
1423         options.results_directory, "unexpected_results.json")
1424         with codecs.open(unexpected_results_filename, "r", "utf-8") as file:
1425             results = simplejson.load(file)
1426         last_unexpected_results = results['tests'].keys()
1427     return last_unexpected_results
1428
1429
1430 def _compat_shim_callback(option, opt_str, value, parser):
1431     print "Ignoring unsupported option: %s" % opt_str
1432
1433
1434 def _compat_shim_option(option_name, **kwargs):
1435     return optparse.make_option(option_name, action="callback",
1436         callback=_compat_shim_callback,
1437         help="Ignored, for old-run-webkit-tests compat only.", **kwargs)
1438
1439
1440 def parse_args(args=None):
1441     """Provides a default set of command line args.
1442
1443     Returns a tuple of options, args from optparse"""
1444
1445     # FIXME: All of these options should be stored closer to the code which
1446     # FIXME: actually uses them. configuration_options should move
1447     # FIXME: to WebKitPort and be shared across all scripts.
1448     configuration_options = [
1449         optparse.make_option("-t", "--target", dest="configuration",
1450                              help="(DEPRECATED)"),
1451         # FIXME: --help should display which configuration is default.
1452         optparse.make_option('--debug', action='store_const', const='Debug',
1453                              dest="configuration",
1454                              help='Set the configuration to Debug'),
1455         optparse.make_option('--release', action='store_const',
1456                              const='Release', dest="configuration",
1457                              help='Set the configuration to Release'),
1458         # old-run-webkit-tests also accepts -c, --configuration CONFIGURATION.
1459     ]
1460
1461     print_options = printing.print_options()
1462
1463     # FIXME: These options should move onto the ChromiumPort.
1464     chromium_options = [
1465         optparse.make_option("--chromium", action="store_true", default=False,
1466             help="use the Chromium port"),
1467         optparse.make_option("--startup-dialog", action="store_true",
1468             default=False, help="create a dialog on DumpRenderTree startup"),
1469         optparse.make_option("--gp-fault-error-box", action="store_true",
1470             default=False, help="enable Windows GP fault error box"),
1471         optparse.make_option("--multiple-loads",
1472             type="int", help="turn on multiple loads of each test"),
1473         optparse.make_option("--js-flags",
1474             type="string", help="JavaScript flags to pass to tests"),
1475         optparse.make_option("--nocheck-sys-deps", action="store_true",
1476             default=False,
1477             help="Don't check the system dependencies (themes)"),
1478         optparse.make_option("--use-test-shell", action="store_true",
1479             default=False,
1480             help="Use test_shell instead of DRT"),
1481         optparse.make_option("--accelerated-compositing",
1482             action="store_true",
1483             help="Use hardware-accelated compositing for rendering"),
1484         optparse.make_option("--no-accelerated-compositing",
1485             action="store_false",
1486             dest="accelerated_compositing",
1487             help="Don't use hardware-accelerated compositing for rendering"),
1488         optparse.make_option("--accelerated-2d-canvas",
1489             action="store_true",
1490             help="Use hardware-accelerated 2D Canvas calls"),
1491         optparse.make_option("--no-accelerated-2d-canvas",
1492             action="store_false",
1493             dest="accelerated_2d_canvas",
1494             help="Don't use hardware-accelerated 2D Canvas calls"),
1495     ]
1496
1497     # Missing Mac-specific old-run-webkit-tests options:
1498     # FIXME: Need: -g, --guard for guard malloc support on Mac.
1499     # FIXME: Need: -l --leaks    Enable leaks checking.
1500     # FIXME: Need: --sample-on-timeout Run sample on timeout
1501
1502     old_run_webkit_tests_compat = [
1503         # NRWT doesn't generate results by default anyway.
1504         _compat_shim_option("--no-new-test-results"),
1505         # NRWT doesn't sample on timeout yet anyway.
1506         _compat_shim_option("--no-sample-on-timeout"),
1507         # FIXME: NRWT needs to support remote links eventually.
1508         _compat_shim_option("--use-remote-links-to-tests"),
1509     ]
1510
1511     results_options = [
1512         # NEED for bots: --use-remote-links-to-tests Link to test files
1513         # within the SVN repository in the results.
1514         optparse.make_option("-p", "--pixel-tests", action="store_true",
1515             dest="pixel_tests", help="Enable pixel-to-pixel PNG comparisons"),
1516         optparse.make_option("--no-pixel-tests", action="store_false",
1517             dest="pixel_tests", help="Disable pixel-to-pixel PNG comparisons"),
1518         optparse.make_option("--tolerance",
1519             help="Ignore image differences less than this percentage (some "
1520                 "ports may ignore this option)", type="float"),
1521         optparse.make_option("--results-directory",
1522             default="layout-test-results",
1523             help="Output results directory source dir, relative to Debug or "
1524                  "Release"),
1525         optparse.make_option("--new-baseline", action="store_true",
1526             default=False, help="Save all generated results as new baselines "
1527                  "into the platform directory, overwriting whatever's "
1528                  "already there."),
1529         optparse.make_option("--reset-results", action="store_true",
1530             default=False, help="Reset any existing baselines to the "
1531                  "generated results"),
1532         optparse.make_option("--no-show-results", action="store_false",
1533             default=True, dest="show_results",
1534             help="Don't launch a browser with results after the tests "
1535                  "are done"),
1536         # FIXME: We should have a helper function to do this sort of
1537         # deprectated mapping and automatically log, etc.
1538         optparse.make_option("--noshow-results", action="store_false",
1539             dest="show_results",
1540             help="Deprecated, same as --no-show-results."),
1541         optparse.make_option("--no-launch-safari", action="store_false",
1542             dest="show_results",
1543             help="old-run-webkit-tests compat, same as --noshow-results."),
1544         # old-run-webkit-tests:
1545         # --[no-]launch-safari    Launch (or do not launch) Safari to display
1546         #                         test results (default: launch)
1547         optparse.make_option("--full-results-html", action="store_true",
1548             default=False,
1549             help="Show all failures in results.html, rather than only "
1550                  "regressions"),
1551         optparse.make_option("--clobber-old-results", action="store_true",
1552             default=False, help="Clobbers test results from previous runs."),
1553         optparse.make_option("--platform",
1554             help="Override the platform for expected results"),
1555         optparse.make_option("--no-record-results", action="store_false",
1556             default=True, dest="record_results",
1557             help="Don't record the results."),
1558         # old-run-webkit-tests also has HTTP toggle options:
1559         # --[no-]http                     Run (or do not run) http tests
1560         #                                 (default: run)
1561     ]
1562
1563     test_options = [
1564         optparse.make_option("--build", dest="build",
1565             action="store_true", default=True,
1566             help="Check to ensure the DumpRenderTree build is up-to-date "
1567                  "(default)."),
1568         optparse.make_option("--no-build", dest="build",
1569             action="store_false", help="Don't check to see if the "
1570                                        "DumpRenderTree build is up-to-date."),
1571         optparse.make_option("-n", "--dry-run", action="store_true",
1572             default=False,
1573             help="Do everything but actually run the tests or upload results."),
1574         # old-run-webkit-tests has --valgrind instead of wrapper.
1575         optparse.make_option("--wrapper",
1576             help="wrapper command to insert before invocations of "
1577                  "DumpRenderTree; option is split on whitespace before "
1578                  "running. (Example: --wrapper='valgrind --smc-check=all')"),
1579         # old-run-webkit-tests:
1580         # -i|--ignore-tests               Comma-separated list of directories
1581         #                                 or tests to ignore
1582         optparse.make_option("--test-list", action="append",
1583             help="read list of tests to run from file", metavar="FILE"),
1584         # old-run-webkit-tests uses --skipped==[default|ignore|only]
1585         # instead of --force:
1586         optparse.make_option("--force", action="store_true", default=False,
1587             help="Run all tests, even those marked SKIP in the test list"),
1588         optparse.make_option("--use-apache", action="store_true",
1589             default=False, help="Whether to use apache instead of lighttpd."),
1590         optparse.make_option("--time-out-ms",
1591             help="Set the timeout for each test"),
1592         # old-run-webkit-tests calls --randomize-order --random:
1593         optparse.make_option("--randomize-order", action="store_true",
1594             default=False, help=("Run tests in random order (useful "
1595                                 "for tracking down corruption)")),
1596         optparse.make_option("--run-chunk",
1597             help=("Run a specified chunk (n:l), the nth of len l, "
1598                  "of the layout tests")),
1599         optparse.make_option("--run-part", help=("Run a specified part (n:m), "
1600                   "the nth of m parts, of the layout tests")),
1601         # old-run-webkit-tests calls --batch-size: --nthly n
1602         #   Restart DumpRenderTree every n tests (default: 1000)
1603         optparse.make_option("--batch-size",
1604             help=("Run a the tests in batches (n), after every n tests, "
1605                   "DumpRenderTree is relaunched."), type="int", default=0),
1606         # old-run-webkit-tests calls --run-singly: -1|--singly
1607         # Isolate each test case run (implies --nthly 1 --verbose)
1608         optparse.make_option("--run-singly", action="store_true",
1609             default=False, help="run a separate DumpRenderTree for each test"),
1610         optparse.make_option("--child-processes",
1611             help="Number of DumpRenderTrees to run in parallel."),
1612         # FIXME: Display default number of child processes that will run.
1613         optparse.make_option("--worker-model", action="store",
1614             default="old-threads", help=("controls worker model. Valid values "
1615             "are 'old-inline', 'old-threads'.")),
1616         optparse.make_option("--experimental-fully-parallel",
1617             action="store_true", default=False,
1618             help="run all tests in parallel"),
1619         optparse.make_option("--exit-after-n-failures", type="int", nargs=1,
1620             help="Exit after the first N failures instead of running all "
1621             "tests"),
1622         optparse.make_option("--exit-after-n-crashes-or-timeouts", type="int",
1623             nargs=1, help="Exit after the first N crashes instead of running "
1624             "all tests"),
1625         # FIXME: consider: --iterations n
1626         #      Number of times to run the set of tests (e.g. ABCABCABC)
1627         optparse.make_option("--print-last-failures", action="store_true",
1628             default=False, help="Print the tests in the last run that "
1629             "had unexpected failures (or passes) and then exit."),
1630         optparse.make_option("--retest-last-failures", action="store_true",
1631             default=False, help="re-test the tests in the last run that "
1632             "had unexpected failures (or passes)."),
1633         optparse.make_option("--retry-failures", action="store_true",
1634             default=True,
1635             help="Re-try any tests that produce unexpected results (default)"),
1636         optparse.make_option("--no-retry-failures", action="store_false",
1637             dest="retry_failures",
1638             help="Don't re-try any tests that produce unexpected results."),
1639     ]
1640
1641     misc_options = [
1642         optparse.make_option("--lint-test-files", action="store_true",
1643         default=False, help=("Makes sure the test files parse for all "
1644                             "configurations. Does not run any tests.")),
1645     ]
1646
1647     # FIXME: Move these into json_results_generator.py
1648     results_json_options = [
1649         optparse.make_option("--master-name", help="The name of the buildbot master."),
1650         optparse.make_option("--builder-name", default="DUMMY_BUILDER_NAME",
1651             help=("The name of the builder shown on the waterfall running "
1652                   "this script e.g. WebKit.")),
1653         optparse.make_option("--build-name", default="DUMMY_BUILD_NAME",
1654             help=("The name of the builder used in its path, e.g. "
1655                   "webkit-rel.")),
1656         optparse.make_option("--build-number", default="DUMMY_BUILD_NUMBER",
1657             help=("The build number of the builder running this script.")),
1658         optparse.make_option("--test-results-server", default="",
1659             help=("If specified, upload results json files to this appengine "
1660                   "server.")),
1661         optparse.make_option("--upload-full-results",
1662             action="store_true",
1663             default=False,
1664             help="If true, upload full json results to server."),
1665     ]
1666
1667     option_list = (configuration_options + print_options +
1668                    chromium_options + results_options + test_options +
1669                    misc_options + results_json_options +
1670                    old_run_webkit_tests_compat)
1671     option_parser = optparse.OptionParser(option_list=option_list)
1672
1673     return option_parser.parse_args(args)
1674
1675
1676 def main():
1677     options, args = parse_args()
1678     port_obj = port.get(options.platform, options)
1679     return run(port_obj, options, args)
1680
1681
1682 if '__main__' == __name__:
1683     try:
1684         sys.exit(main())
1685     except KeyboardInterrupt:
1686         # this mirrors what the shell normally does
1687         sys.exit(signal.SIGINT + 128)