Tools/Scripts/webkitpy/layout_tests/run_webkit_tests.py

   1 #!/usr/bin/env python
   2 # Copyright (C) 2010 Google Inc. All rights reserved.
   3 # Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Szeged
   4 #
   5 # Redistribution and use in source and binary forms, with or without
   6 # modification, are permitted provided that the following conditions are
   7 # met:
   8 #
   9 #     * Redistributions of source code must retain the above copyright
  10 # notice, this list of conditions and the following disclaimer.
  11 #     * Redistributions in binary form must reproduce the above
  12 # copyright notice, this list of conditions and the following disclaimer
  13 # in the documentation and/or other materials provided with the
  14 # distribution.
  15 #     * Neither the name of Google Inc. nor the names of its
  16 # contributors may be used to endorse or promote products derived from
  17 # this software without specific prior written permission.
  18 #
  19 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  20 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  21 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  22 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  23 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  24 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  25 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  26 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  27 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  28 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  29 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  30
  31 """Run layout tests.
  32
  33 This is a port of the existing webkit test script run-webkit-tests.
  34
  35 The TestRunner class runs a series of tests (TestType interface) against a set
  36 of test files.  If a test file fails a TestType, it returns a list TestFailure
  37 objects to the TestRunner.  The TestRunner then aggregates the TestFailures to
  38 create a final report.
  39
  40 This script reads several files, if they exist in the test_lists subdirectory
  41 next to this script itself.  Each should contain a list of paths to individual
  42 tests or entire subdirectories of tests, relative to the outermost test
  43 directory.  Entire lines starting with '//' (comments) will be ignored.
  44
  45 For details of the files' contents and purposes, see test_lists/README.
  46 """
  47
  48 from __future__ import with_statement
  49
  50 import codecs
  51 import errno
  52 import glob
  53 import logging
  54 import math
  55 import optparse
  56 import os
  57 import platform
  58 import Queue
  59 import random
  60 import re
  61 import shutil
  62 import signal
  63 import sys
  64 import time
  65 import traceback
  66
  67 from layout_package import dump_render_tree_thread
  68 from layout_package import json_layout_results_generator
  69 from layout_package import message_broker
  70 from layout_package import printing
  71 from layout_package import test_expectations
  72 from layout_package import test_failures
  73 from layout_package import test_results
  74 from layout_package import test_results_uploader
  75
  76 from webkitpy.common.system import user
  77 from webkitpy.thirdparty import simplejson
  78 from webkitpy.tool import grammar
  79
  80 import port
  81
  82 _log = logging.getLogger("webkitpy.layout_tests.run_webkit_tests")
  83
  84 # Builder base URL where we have the archived test results.
  85 BUILDER_BASE_URL = "http://build.chromium.org/buildbot/layout_test_results/"
  86
  87 LAYOUT_TESTS_DIRECTORY = "LayoutTests" + os.sep
  88
  89 TestExpectationsFile = test_expectations.TestExpectationsFile
  90
  91
  92 class TestInput:
  93     """Groups information about a test for easy passing of data."""
  94
  95     def __init__(self, filename, timeout):
  96         """Holds the input parameters for a test.
  97         Args:
  98           filename: Full path to the test.
  99           timeout: Timeout in msecs the driver should use while running the test
 100           """
 101         # FIXME: filename should really be test_name as a relative path.
 102         self.filename = filename
 103         self.timeout = timeout
 104         # The image_hash is used to avoid doing an image dump if the
 105         # checksums match. The image_hash is set later, and only if it is needed
 106         # for the test.
 107         self.image_hash = None
 108
 109
 110 class TestRunInterruptedException(Exception):
 111     """Raised when a test run should be stopped immediately."""
 112     def __init__(self, reason):
 113         self.reason = reason
 114
 115
 116 class ResultSummary(object):
 117     """A class for partitioning the test results we get into buckets.
 118
 119     This class is basically a glorified struct and it's private to this file
 120     so we don't bother with any information hiding."""
 121
 122     def __init__(self, expectations, test_files):
 123         self.total = len(test_files)
 124         self.remaining = self.total
 125         self.expectations = expectations
 126         self.expected = 0
 127         self.unexpected = 0
 128         self.unexpected_failures = 0
 129         self.unexpected_crashes_or_timeouts = 0
 130         self.tests_by_expectation = {}
 131         self.tests_by_timeline = {}
 132         self.results = {}
 133         self.unexpected_results = {}
 134         self.failures = {}
 135         self.tests_by_expectation[test_expectations.SKIP] = set()
 136         for expectation in TestExpectationsFile.EXPECTATIONS.values():
 137             self.tests_by_expectation[expectation] = set()
 138         for timeline in TestExpectationsFile.TIMELINES.values():
 139             self.tests_by_timeline[timeline] = (
 140                 expectations.get_tests_with_timeline(timeline))
 141
 142     def add(self, result, expected):
 143         """Add a TestResult into the appropriate bin.
 144
 145         Args:
 146           result: TestResult from dump_render_tree_thread.
 147           expected: whether the result was what we expected it to be.
 148         """
 149
 150         self.tests_by_expectation[result.type].add(result.filename)
 151         self.results[result.filename] = result
 152         self.remaining -= 1
 153         if len(result.failures):
 154             self.failures[result.filename] = result.failures
 155         if expected:
 156             self.expected += 1
 157         else:
 158             self.unexpected_results[result.filename] = result.type
 159             self.unexpected += 1
 160             if len(result.failures):
 161                 self.unexpected_failures += 1
 162             if result.type == test_expectations.CRASH or result.type == test_expectations.TIMEOUT:
 163                 self.unexpected_crashes_or_timeouts += 1
 164
 165
 166 def summarize_unexpected_results(port_obj, expectations, result_summary,
 167                                  retry_summary):
 168     """Summarize any unexpected results as a dict.
 169
 170     FIXME: split this data structure into a separate class?
 171
 172     Args:
 173         port_obj: interface to port-specific hooks
 174         expectations: test_expectations.TestExpectations object
 175         result_summary: summary object from initial test runs
 176         retry_summary: summary object from final test run of retried tests
 177     Returns:
 178         A dictionary containing a summary of the unexpected results from the
 179         run, with the following fields:
 180         'version': a version indicator (1 in this version)
 181         'fixable': # of fixable tests (NOW - PASS)
 182         'skipped': # of skipped tests (NOW & SKIPPED)
 183         'num_regressions': # of non-flaky failures
 184         'num_flaky': # of flaky failures
 185         'num_passes': # of unexpected passes
 186         'tests': a dict of tests -> {'expected': '...', 'actual': '...'}
 187     """
 188     results = {}
 189     results['version'] = 1
 190
 191     tbe = result_summary.tests_by_expectation
 192     tbt = result_summary.tests_by_timeline
 193     results['fixable'] = len(tbt[test_expectations.NOW] -
 194                                 tbe[test_expectations.PASS])
 195     results['skipped'] = len(tbt[test_expectations.NOW] &
 196                                 tbe[test_expectations.SKIP])
 197
 198     num_passes = 0
 199     num_flaky = 0
 200     num_regressions = 0
 201     keywords = {}
 202     for k, v in TestExpectationsFile.EXPECTATIONS.iteritems():
 203         keywords[v] = k.upper()
 204
 205     tests = {}
 206     for filename, result in result_summary.unexpected_results.iteritems():
 207         # Note that if a test crashed in the original run, we ignore
 208         # whether or not it crashed when we retried it (if we retried it),
 209         # and always consider the result not flaky.
 210         test = port_obj.relative_test_filename(filename)
 211         expected = expectations.get_expectations_string(filename)
 212         actual = [keywords[result]]
 213
 214         if result == test_expectations.PASS:
 215             num_passes += 1
 216         elif result == test_expectations.CRASH:
 217             num_regressions += 1
 218         else:
 219             if filename not in retry_summary.unexpected_results:
 220                 actual.extend(expectations.get_expectations_string(
 221                     filename).split(" "))
 222                 num_flaky += 1
 223             else:
 224                 retry_result = retry_summary.unexpected_results[filename]
 225                 if result != retry_result:
 226                     actual.append(keywords[retry_result])
 227                     num_flaky += 1
 228                 else:
 229                     num_regressions += 1
 230
 231         tests[test] = {}
 232         tests[test]['expected'] = expected
 233         tests[test]['actual'] = " ".join(actual)
 234
 235     results['tests'] = tests
 236     results['num_passes'] = num_passes
 237     results['num_flaky'] = num_flaky
 238     results['num_regressions'] = num_regressions
 239
 240     return results
 241
 242
 243 class TestRunner:
 244     """A class for managing running a series of tests on a series of layout
 245     test files."""
 246
 247     HTTP_SUBDIR = os.sep.join(['', 'http', ''])
 248     WEBSOCKET_SUBDIR = os.sep.join(['', 'websocket', ''])
 249
 250     # The per-test timeout in milliseconds, if no --time-out-ms option was
 251     # given to run_webkit_tests. This should correspond to the default timeout
 252     # in DumpRenderTree.
 253     DEFAULT_TEST_TIMEOUT_MS = 6 * 1000
 254
 255     def __init__(self, port, options, printer, message_broker):
 256         """Initialize test runner data structures.
 257
 258         Args:
 259           port: an object implementing port-specific
 260           options: a dictionary of command line options
 261           printer: a Printer object to record updates to.
 262           message_broker: object used to communicate with workers.
 263         """
 264         self._port = port
 265         self._options = options
 266         self._printer = printer
 267         self._message_broker = message_broker
 268
 269         # disable wss server. need to install pyOpenSSL on buildbots.
 270         # self._websocket_secure_server = websocket_server.PyWebSocket(
 271         #        options.results_directory, use_tls=True, port=9323)
 272
 273         # a set of test files, and the same tests as a list
 274         self._test_files = set()
 275         self._test_files_list = None
 276         self._result_queue = Queue.Queue()
 277         self._retrying = False
 278
 279     def collect_tests(self, args, last_unexpected_results):
 280         """Find all the files to test.
 281
 282         Args:
 283           args: list of test arguments from the command line
 284           last_unexpected_results: list of unexpected results to retest, if any
 285
 286         """
 287         paths = [self._strip_test_dir_prefix(arg) for arg in args if arg and arg != '']
 288         paths += last_unexpected_results
 289         if self._options.test_list:
 290             paths += read_test_files(self._options.test_list)
 291         self._test_files = self._port.tests(paths)
 292
 293     def _strip_test_dir_prefix(self, path):
 294         if path.startswith(LAYOUT_TESTS_DIRECTORY):
 295             return path[len(LAYOUT_TESTS_DIRECTORY):]
 296         return path
 297
 298     def lint(self):
 299         lint_failed = False
 300
 301         # Creating the expecations for each platform/configuration pair does
 302         # all the test list parsing and ensures it's correct syntax (e.g. no
 303         # dupes).
 304         for platform_name in self._port.test_platform_names():
 305             try:
 306                 self.parse_expectations(platform_name, is_debug_mode=True)
 307             except test_expectations.ParseError:
 308                 lint_failed = True
 309             try:
 310                 self.parse_expectations(platform_name, is_debug_mode=False)
 311             except test_expectations.ParseError:
 312                 lint_failed = True
 313
 314         self._printer.write("")
 315         if lint_failed:
 316             _log.error("Lint failed.")
 317             return -1
 318
 319         _log.info("Lint succeeded.")
 320         return 0
 321
 322     def parse_expectations(self, test_platform_name, is_debug_mode):
 323         """Parse the expectations from the test_list files and return a data
 324         structure holding them. Throws an error if the test_list files have
 325         invalid syntax."""
 326         if self._options.lint_test_files:
 327             test_files = None
 328         else:
 329             test_files = self._test_files
 330
 331         expectations_str = self._port.test_expectations()
 332         overrides_str = self._port.test_expectations_overrides()
 333         self._expectations = test_expectations.TestExpectations(
 334             self._port, test_files, expectations_str, test_platform_name,
 335             is_debug_mode, self._options.lint_test_files,
 336             overrides=overrides_str)
 337         return self._expectations
 338
 339
 340     def prepare_lists_and_print_output(self):
 341         """Create appropriate subsets of test lists and returns a
 342         ResultSummary object. Also prints expected test counts.
 343         """
 344
 345         # Remove skipped - both fixable and ignored - files from the
 346         # top-level list of files to test.
 347         num_all_test_files = len(self._test_files)
 348         self._printer.print_expected("Found:  %d tests" %
 349                                      (len(self._test_files)))
 350         if not num_all_test_files:
 351             _log.critical('No tests to run.')
 352             return None
 353
 354         skipped = set()
 355         if num_all_test_files > 1 and not self._options.force:
 356             skipped = self._expectations.get_tests_with_result_type(
 357                            test_expectations.SKIP)
 358             self._test_files -= skipped
 359
 360         # Create a sorted list of test files so the subset chunk,
 361         # if used, contains alphabetically consecutive tests.
 362         self._test_files_list = list(self._test_files)
 363         if self._options.randomize_order:
 364             random.shuffle(self._test_files_list)
 365         else:
 366             self._test_files_list.sort()
 367
 368         # If the user specifies they just want to run a subset of the tests,
 369         # just grab a subset of the non-skipped tests.
 370         if self._options.run_chunk or self._options.run_part:
 371             chunk_value = self._options.run_chunk or self._options.run_part
 372             test_files = self._test_files_list
 373             try:
 374                 (chunk_num, chunk_len) = chunk_value.split(":")
 375                 chunk_num = int(chunk_num)
 376                 assert(chunk_num >= 0)
 377                 test_size = int(chunk_len)
 378                 assert(test_size > 0)
 379             except:
 380                 _log.critical("invalid chunk '%s'" % chunk_value)
 381                 return None
 382
 383             # Get the number of tests
 384             num_tests = len(test_files)
 385
 386             # Get the start offset of the slice.
 387             if self._options.run_chunk:
 388                 chunk_len = test_size
 389                 # In this case chunk_num can be really large. We need
 390                 # to make the slave fit in the current number of tests.
 391                 slice_start = (chunk_num * chunk_len) % num_tests
 392             else:
 393                 # Validate the data.
 394                 assert(test_size <= num_tests)
 395                 assert(chunk_num <= test_size)
 396
 397                 # To count the chunk_len, and make sure we don't skip
 398                 # some tests, we round to the next value that fits exactly
 399                 # all the parts.
 400                 rounded_tests = num_tests
 401                 if rounded_tests % test_size != 0:
 402                     rounded_tests = (num_tests + test_size -
 403                                      (num_tests % test_size))
 404
 405                 chunk_len = rounded_tests / test_size
 406                 slice_start = chunk_len * (chunk_num - 1)
 407                 # It does not mind if we go over test_size.
 408
 409             # Get the end offset of the slice.
 410             slice_end = min(num_tests, slice_start + chunk_len)
 411
 412             files = test_files[slice_start:slice_end]
 413
 414             tests_run_msg = 'Running: %d tests (chunk slice [%d:%d] of %d)' % (
 415                 (slice_end - slice_start), slice_start, slice_end, num_tests)
 416             self._printer.print_expected(tests_run_msg)
 417
 418             # If we reached the end and we don't have enough tests, we run some
 419             # from the beginning.
 420             if slice_end - slice_start < chunk_len:
 421                 extra = chunk_len - (slice_end - slice_start)
 422                 extra_msg = ('   last chunk is partial, appending [0:%d]' %
 423                             extra)
 424                 self._printer.print_expected(extra_msg)
 425                 tests_run_msg += "\n" + extra_msg
 426                 files.extend(test_files[0:extra])
 427             tests_run_filename = os.path.join(self._options.results_directory,
 428                                               "tests_run.txt")
 429             with codecs.open(tests_run_filename, "w", "utf-8") as file:
 430                 file.write(tests_run_msg + "\n")
 431
 432             len_skip_chunk = int(len(files) * len(skipped) /
 433                                  float(len(self._test_files)))
 434             skip_chunk_list = list(skipped)[0:len_skip_chunk]
 435             skip_chunk = set(skip_chunk_list)
 436
 437             # Update expectations so that the stats are calculated correctly.
 438             # We need to pass a list that includes the right # of skipped files
 439             # to ParseExpectations so that ResultSummary() will get the correct
 440             # stats. So, we add in the subset of skipped files, and then
 441             # subtract them back out.
 442             self._test_files_list = files + skip_chunk_list
 443             self._test_files = set(self._test_files_list)
 444
 445             self._expectations = self.parse_expectations(
 446                 self._port.test_platform_name(),
 447                 self._options.configuration == 'Debug')
 448
 449             self._test_files = set(files)
 450             self._test_files_list = files
 451         else:
 452             skip_chunk = skipped
 453
 454         result_summary = ResultSummary(self._expectations,
 455             self._test_files | skip_chunk)
 456         self._print_expected_results_of_type(result_summary,
 457             test_expectations.PASS, "passes")
 458         self._print_expected_results_of_type(result_summary,
 459             test_expectations.FAIL, "failures")
 460         self._print_expected_results_of_type(result_summary,
 461             test_expectations.FLAKY, "flaky")
 462         self._print_expected_results_of_type(result_summary,
 463             test_expectations.SKIP, "skipped")
 464
 465         if self._options.force:
 466             self._printer.print_expected('Running all tests, including '
 467                                          'skips (--force)')
 468         else:
 469             # Note that we don't actually run the skipped tests (they were
 470             # subtracted out of self._test_files, above), but we stub out the
 471             # results here so the statistics can remain accurate.
 472             for test in skip_chunk:
 473                 result = test_results.TestResult(test,
 474                     failures=[], test_run_time=0, total_time_for_all_diffs=0,
 475                     time_for_diffs=0)
 476                 result.type = test_expectations.SKIP
 477                 result_summary.add(result, expected=True)
 478         self._printer.print_expected('')
 479
 480         return result_summary
 481
 482     def _get_dir_for_test_file(self, test_file):
 483         """Returns the highest-level directory by which to shard the given
 484         test file."""
 485         index = test_file.rfind(os.sep + LAYOUT_TESTS_DIRECTORY)
 486
 487         test_file = test_file[index + len(LAYOUT_TESTS_DIRECTORY):]
 488         test_file_parts = test_file.split(os.sep, 1)
 489         directory = test_file_parts[0]
 490         test_file = test_file_parts[1]
 491
 492         # The http tests are very stable on mac/linux.
 493         # TODO(ojan): Make the http server on Windows be apache so we can
 494         # turn shard the http tests there as well. Switching to apache is
 495         # what made them stable on linux/mac.
 496         return_value = directory
 497         while ((directory != 'http' or sys.platform in ('darwin', 'linux2'))
 498                 and test_file.find(os.sep) >= 0):
 499             test_file_parts = test_file.split(os.sep, 1)
 500             directory = test_file_parts[0]
 501             return_value = os.path.join(return_value, directory)
 502             test_file = test_file_parts[1]
 503
 504         return return_value
 505
 506     def _get_test_input_for_file(self, test_file):
 507         """Returns the appropriate TestInput object for the file. Mostly this
 508         is used for looking up the timeout value (in ms) to use for the given
 509         test."""
 510         if self._test_is_slow(test_file):
 511             return TestInput(test_file, self._options.slow_time_out_ms)
 512         return TestInput(test_file, self._options.time_out_ms)
 513
 514     def _test_requires_lock(self, test_file):
 515         """Return True if the test needs to be locked when
 516         running multiple copies of NRWTs."""
 517         split_path = test_file.split(os.sep)
 518         return 'http' in split_path or 'websocket' in split_path
 519
 520     def _test_is_slow(self, test_file):
 521         return self._expectations.has_modifier(test_file,
 522                                                test_expectations.SLOW)
 523
 524     def _shard_tests(self, test_files, use_real_shards):
 525         """Groups tests into batches.
 526         This helps ensure that tests that depend on each other (aka bad tests!)
 527         continue to run together as most cross-tests dependencies tend to
 528         occur within the same directory. If use_real_shards is false, we
 529         put each (non-HTTP/websocket) test into its own shard for maximum
 530         concurrency instead of trying to do any sort of real sharding.
 531
 532         Return:
 533             A list of lists of TestInput objects.
 534         """
 535         # FIXME: when we added http locking, we changed how this works such
 536         # that we always lump all of the HTTP threads into a single shard.
 537         # That will slow down experimental-fully-parallel, but it's unclear
 538         # what the best alternative is completely revamping how we track
 539         # when to grab the lock.
 540
 541         test_lists = []
 542         tests_to_http_lock = []
 543         if not use_real_shards:
 544             for test_file in test_files:
 545                 test_input = self._get_test_input_for_file(test_file)
 546                 if self._test_requires_lock(test_file):
 547                     tests_to_http_lock.append(test_input)
 548                 else:
 549                     test_lists.append((".", [test_input]))
 550         else:
 551             tests_by_dir = {}
 552             for test_file in test_files:
 553                 directory = self._get_dir_for_test_file(test_file)
 554                 test_input = self._get_test_input_for_file(test_file)
 555                 if self._test_requires_lock(test_file):
 556                     tests_to_http_lock.append(test_input)
 557                 else:
 558                     tests_by_dir.setdefault(directory, [])
 559                     tests_by_dir[directory].append(test_input)
 560             # Sort by the number of tests in the dir so that the ones with the
 561             # most tests get run first in order to maximize parallelization.
 562             # Number of tests is a good enough, but not perfect, approximation
 563             # of how long that set of tests will take to run. We can't just use
 564             # a PriorityQueue until we move to Python 2.6.
 565             for directory in tests_by_dir:
 566                 test_list = tests_by_dir[directory]
 567                 # Keep the tests in alphabetical order.
 568                 # FIXME: Remove once tests are fixed so they can be run in any
 569                 # order.
 570                 test_list.reverse()
 571                 test_list_tuple = (directory, test_list)
 572                 test_lists.append(test_list_tuple)
 573             test_lists.sort(lambda a, b: cmp(len(b[1]), len(a[1])))
 574
 575         # Put the http tests first. There are only a couple hundred of them,
 576         # but each http test takes a very long time to run, so sorting by the
 577         # number of tests doesn't accurately capture how long they take to run.
 578         if tests_to_http_lock:
 579             tests_to_http_lock.reverse()
 580             test_lists.insert(0, ("tests_to_http_lock", tests_to_http_lock))
 581
 582         return test_lists
 583
 584     def _contains_tests(self, subdir):
 585         for test_file in self._test_files:
 586             if test_file.find(subdir) >= 0:
 587                 return True
 588         return False
 589
 590     def _num_workers(self):
 591         return int(self._options.child_processes)
 592
 593     def _run_tests(self, file_list, result_summary):
 594         """Runs the tests in the file_list.
 595
 596         Return: A tuple (interrupted, keyboard_interrupted, thread_timings,
 597             test_timings, individual_test_timings)
 598             interrupted is whether the run was interrupted
 599             keyboard_interrupted is whether the interruption was because someone
 600               typed Ctrl^C
 601             thread_timings is a list of dicts with the total runtime
 602               of each thread with 'name', 'num_tests', 'total_time' properties
 603             test_timings is a list of timings for each sharded subdirectory
 604               of the form [time, directory_name, num_tests]
 605             individual_test_timings is a list of run times for each test
 606               in the form {filename:filename, test_run_time:test_run_time}
 607             result_summary: summary object to populate with the results
 608         """
 609
 610         self._printer.print_update('Sharding tests ...')
 611         num_workers = self._num_workers()
 612         test_lists = self._shard_tests(file_list,
 613             num_workers > 1 and not self._options.experimental_fully_parallel)
 614         filename_queue = Queue.Queue()
 615         for item in test_lists:
 616             filename_queue.put(item)
 617
 618         self._printer.print_update('Starting %s ...' %
 619                                    grammar.pluralize('worker', num_workers))
 620         message_broker = self._message_broker
 621         self._current_filename_queue = filename_queue
 622         self._current_result_summary = result_summary
 623
 624         if not self._options.dry_run:
 625             threads = message_broker.start_workers(self)
 626         else:
 627             threads = {}
 628
 629         self._printer.print_update("Starting testing ...")
 630         keyboard_interrupted = False
 631         interrupted = False
 632         if not self._options.dry_run:
 633             try:
 634                 message_broker.run_message_loop()
 635             except KeyboardInterrupt:
 636                 _log.info("Interrupted, exiting")
 637                 message_broker.cancel_workers()
 638                 keyboard_interrupted = True
 639                 interrupted = True
 640             except TestRunInterruptedException, e:
 641                 _log.info(e.reason)
 642                 message_broker.cancel_workers()
 643                 interrupted = True
 644             except:
 645                 # Unexpected exception; don't try to clean up workers.
 646                 _log.info("Exception raised, exiting")
 647                 raise
 648
 649         thread_timings, test_timings, individual_test_timings = \
 650             self._collect_timing_info(threads)
 651
 652         return (interrupted, keyboard_interrupted, thread_timings, test_timings,
 653                 individual_test_timings)
 654
 655     def update(self):
 656         self.update_summary(self._current_result_summary)
 657
 658     def _collect_timing_info(self, threads):
 659         test_timings = {}
 660         individual_test_timings = []
 661         thread_timings = []
 662
 663         for thread in threads:
 664             thread_timings.append({'name': thread.getName(),
 665                                    'num_tests': thread.get_num_tests(),
 666                                    'total_time': thread.get_total_time()})
 667             test_timings.update(thread.get_test_group_timing_stats())
 668             individual_test_timings.extend(thread.get_test_results())
 669
 670         return (thread_timings, test_timings, individual_test_timings)
 671
 672     def needs_http(self):
 673         """Returns whether the test runner needs an HTTP server."""
 674         return self._contains_tests(self.HTTP_SUBDIR)
 675
 676     def needs_websocket(self):
 677         """Returns whether the test runner needs a WEBSOCKET server."""
 678         return self._contains_tests(self.WEBSOCKET_SUBDIR)
 679
 680     def set_up_run(self):
 681         """Configures the system to be ready to run tests.
 682
 683         Returns a ResultSummary object if we should continue to run tests,
 684         or None if we should abort.
 685
 686         """
 687         # This must be started before we check the system dependencies,
 688         # since the helper may do things to make the setup correct.
 689         self._printer.print_update("Starting helper ...")
 690         self._port.start_helper()
 691
 692         # Check that the system dependencies (themes, fonts, ...) are correct.
 693         if not self._options.nocheck_sys_deps:
 694             self._printer.print_update("Checking system dependencies ...")
 695             if not self._port.check_sys_deps(self.needs_http()):
 696                 self._port.stop_helper()
 697                 return None
 698
 699         if self._options.clobber_old_results:
 700             self._clobber_old_results()
 701
 702         # Create the output directory if it doesn't already exist.
 703         self._port.maybe_make_directory(self._options.results_directory)
 704
 705         self._port.setup_test_run()
 706
 707         self._printer.print_update("Preparing tests ...")
 708         result_summary = self.prepare_lists_and_print_output()
 709         if not result_summary:
 710             return None
 711
 712         return result_summary
 713
 714     def run(self, result_summary):
 715         """Run all our tests on all our test files.
 716
 717         For each test file, we run each test type. If there are any failures,
 718         we collect them for reporting.
 719
 720         Args:
 721           result_summary: a summary object tracking the test results.
 722
 723         Return:
 724           The number of unexpected results (0 == success)
 725         """
 726         # gather_test_files() must have been called first to initialize us.
 727         # If we didn't find any files to test, we've errored out already in
 728         # prepare_lists_and_print_output().
 729         assert(len(self._test_files))
 730
 731         start_time = time.time()
 732
 733         interrupted, keyboard_interrupted, thread_timings, test_timings, \
 734             individual_test_timings = (
 735             self._run_tests(self._test_files_list, result_summary))
 736
 737         # We exclude the crashes from the list of results to retry, because
 738         # we want to treat even a potentially flaky crash as an error.
 739         failures = self._get_failures(result_summary, include_crashes=False)
 740         retry_summary = result_summary
 741         while (len(failures) and self._options.retry_failures and
 742             not self._retrying and not interrupted):
 743             _log.info('')
 744             _log.info("Retrying %d unexpected failure(s) ..." % len(failures))
 745             _log.info('')
 746             self._retrying = True
 747             retry_summary = ResultSummary(self._expectations, failures.keys())
 748             # Note that we intentionally ignore the return value here.
 749             self._run_tests(failures.keys(), retry_summary)
 750             failures = self._get_failures(retry_summary, include_crashes=True)
 751
 752         end_time = time.time()
 753
 754         self._print_timing_statistics(end_time - start_time,
 755                                       thread_timings, test_timings,
 756                                       individual_test_timings,
 757                                       result_summary)
 758
 759         self._print_result_summary(result_summary)
 760
 761         sys.stdout.flush()
 762         sys.stderr.flush()
 763
 764         self._printer.print_one_line_summary(result_summary.total,
 765                                              result_summary.expected,
 766                                              result_summary.unexpected)
 767
 768         unexpected_results = summarize_unexpected_results(self._port,
 769             self._expectations, result_summary, retry_summary)
 770         self._printer.print_unexpected_results(unexpected_results)
 771
 772         if (self._options.record_results and not self._options.dry_run and
 773             not interrupted):
 774             # Write the same data to log files and upload generated JSON files
 775             # to appengine server.
 776             self._upload_json_files(unexpected_results, result_summary,
 777                                     individual_test_timings)
 778
 779         # Write the summary to disk (results.html) and display it if requested.
 780         if not self._options.dry_run:
 781             wrote_results = self._write_results_html_file(result_summary)
 782             if self._options.show_results and wrote_results:
 783                 self._show_results_html_file()
 784
 785         # Now that we've completed all the processing we can, we re-raise
 786         # a KeyboardInterrupt if necessary so the caller can handle it.
 787         if keyboard_interrupted:
 788             raise KeyboardInterrupt
 789
 790         # Ignore flaky failures and unexpected passes so we don't turn the
 791         # bot red for those.
 792         return unexpected_results['num_regressions']
 793
 794     def clean_up_run(self):
 795         """Restores the system after we're done running tests."""
 796
 797         _log.debug("flushing stdout")
 798         sys.stdout.flush()
 799         _log.debug("flushing stderr")
 800         sys.stderr.flush()
 801         _log.debug("stopping helper")
 802         self._port.stop_helper()
 803
 804     def update_summary(self, result_summary):
 805         """Update the summary and print results with any completed tests."""
 806         while True:
 807             try:
 808                 result = test_results.TestResult.loads(self._result_queue.get_nowait())
 809             except Queue.Empty:
 810                 return
 811
 812             expected = self._expectations.matches_an_expected_result(
 813                 result.filename, result.type, self._options.pixel_tests)
 814             result_summary.add(result, expected)
 815             exp_str = self._expectations.get_expectations_string(
 816                 result.filename)
 817             got_str = self._expectations.expectation_to_string(result.type)
 818             self._printer.print_test_result(result, expected, exp_str, got_str)
 819             self._printer.print_progress(result_summary, self._retrying,
 820                                          self._test_files_list)
 821
 822             def interrupt_if_at_failure_limit(limit, count, message):
 823                 if limit and count >= limit:
 824                     raise TestRunInterruptedException(message % count)
 825
 826             interrupt_if_at_failure_limit(
 827                 self._options.exit_after_n_failures,
 828                 result_summary.unexpected_failures,
 829                 "Aborting run since %d failures were reached")
 830             interrupt_if_at_failure_limit(
 831                 self._options.exit_after_n_crashes_or_timeouts,
 832                 result_summary.unexpected_crashes_or_timeouts,
 833                 "Aborting run since %d crashes or timeouts were reached")
 834
 835
 836     def _clobber_old_results(self):
 837         # Just clobber the actual test results directories since the other
 838         # files in the results directory are explicitly used for cross-run
 839         # tracking.
 840         self._printer.print_update("Clobbering old results in %s" %
 841                                    self._options.results_directory)
 842         layout_tests_dir = self._port.layout_tests_dir()
 843         possible_dirs = self._port.test_dirs()
 844         for dirname in possible_dirs:
 845             if os.path.isdir(os.path.join(layout_tests_dir, dirname)):
 846                 shutil.rmtree(os.path.join(self._options.results_directory,
 847                                            dirname),
 848                               ignore_errors=True)
 849
 850     def _get_failures(self, result_summary, include_crashes):
 851         """Filters a dict of results and returns only the failures.
 852
 853         Args:
 854           result_summary: the results of the test run
 855           include_crashes: whether crashes are included in the output.
 856             We use False when finding the list of failures to retry
 857             to see if the results were flaky. Although the crashes may also be
 858             flaky, we treat them as if they aren't so that they're not ignored.
 859         Returns:
 860           a dict of files -> results
 861         """
 862         failed_results = {}
 863         for test, result in result_summary.unexpected_results.iteritems():
 864             if (result == test_expectations.PASS or
 865                 result == test_expectations.CRASH and not include_crashes):
 866                 continue
 867             failed_results[test] = result
 868
 869         return failed_results
 870
 871     def _upload_json_files(self, unexpected_results, result_summary,
 872                         individual_test_timings):
 873         """Writes the results of the test run as JSON files into the results
 874         dir and upload the files to the appengine server.
 875
 876         There are three different files written into the results dir:
 877           unexpected_results.json: A short list of any unexpected results.
 878             This is used by the buildbots to display results.
 879           expectations.json: This is used by the flakiness dashboard.
 880           results.json: A full list of the results - used by the flakiness
 881             dashboard and the aggregate results dashboard.
 882
 883         Args:
 884           unexpected_results: dict of unexpected results
 885           result_summary: full summary object
 886           individual_test_timings: list of test times (used by the flakiness
 887             dashboard).
 888         """
 889         results_directory = self._options.results_directory
 890         _log.debug("Writing JSON files in %s." % results_directory)
 891         unexpected_json_path = os.path.join(results_directory, "unexpected_results.json")
 892         with codecs.open(unexpected_json_path, "w", "utf-8") as file:
 893             simplejson.dump(unexpected_results, file, sort_keys=True, indent=2)
 894
 895         # Write a json file of the test_expectations.txt file for the layout
 896         # tests dashboard.
 897         expectations_path = os.path.join(results_directory, "expectations.json")
 898         expectations_json = \
 899             self._expectations.get_expectations_json_for_all_platforms()
 900         with codecs.open(expectations_path, "w", "utf-8") as file:
 901             file.write(u"ADD_EXPECTATIONS(%s);" % expectations_json)
 902
 903         generator = json_layout_results_generator.JSONLayoutResultsGenerator(
 904             self._port, self._options.builder_name, self._options.build_name,
 905             self._options.build_number, self._options.results_directory,
 906             BUILDER_BASE_URL, individual_test_timings,
 907             self._expectations, result_summary, self._test_files_list,
 908             not self._options.upload_full_results,
 909             self._options.test_results_server,
 910             "layout-tests",
 911             self._options.master_name)
 912
 913         _log.debug("Finished writing JSON files.")
 914
 915         json_files = ["expectations.json"]
 916         if self._options.upload_full_results:
 917             json_files.append("results.json")
 918         else:
 919             json_files.append("incremental_results.json")
 920
 921         generator.upload_json_files(json_files)
 922
 923     def _print_config(self):
 924         """Prints the configuration for the test run."""
 925         p = self._printer
 926         p.print_config("Using port '%s'" % self._port.name())
 927         p.print_config("Placing test results in %s" %
 928                        self._options.results_directory)
 929         if self._options.new_baseline:
 930             p.print_config("Placing new baselines in %s" %
 931                            self._port.baseline_path())
 932         p.print_config("Using %s build" % self._options.configuration)
 933         if self._options.pixel_tests:
 934             p.print_config("Pixel tests enabled")
 935         else:
 936             p.print_config("Pixel tests disabled")
 937
 938         p.print_config("Regular timeout: %s, slow test timeout: %s" %
 939                        (self._options.time_out_ms,
 940                         self._options.slow_time_out_ms))
 941
 942         if self._num_workers() == 1:
 943             p.print_config("Running one %s" % self._port.driver_name())
 944         else:
 945             p.print_config("Running %s %ss in parallel" %
 946                            (self._options.child_processes,
 947                             self._port.driver_name()))
 948         p.print_config('Command line: ' +
 949                        ' '.join(self._port.driver_cmd_line()))
 950         p.print_config("Worker model: %s" % self._options.worker_model)
 951         p.print_config("")
 952
 953     def _print_expected_results_of_type(self, result_summary,
 954                                         result_type, result_type_str):
 955         """Print the number of the tests in a given result class.
 956
 957         Args:
 958           result_summary - the object containing all the results to report on
 959           result_type - the particular result type to report in the summary.
 960           result_type_str - a string description of the result_type.
 961         """
 962         tests = self._expectations.get_tests_with_result_type(result_type)
 963         now = result_summary.tests_by_timeline[test_expectations.NOW]
 964         wontfix = result_summary.tests_by_timeline[test_expectations.WONTFIX]
 965
 966         # We use a fancy format string in order to print the data out in a
 967         # nicely-aligned table.
 968         fmtstr = ("Expect: %%5d %%-8s (%%%dd now, %%%dd wontfix)"
 969                   % (self._num_digits(now), self._num_digits(wontfix)))
 970         self._printer.print_expected(fmtstr %
 971             (len(tests), result_type_str, len(tests & now), len(tests & wontfix)))
 972
 973     def _num_digits(self, num):
 974         """Returns the number of digits needed to represent the length of a
 975         sequence."""
 976         ndigits = 1
 977         if len(num):
 978             ndigits = int(math.log10(len(num))) + 1
 979         return ndigits
 980
 981     def _print_timing_statistics(self, total_time, thread_timings,
 982                                directory_test_timings, individual_test_timings,
 983                                result_summary):
 984         """Record timing-specific information for the test run.
 985
 986         Args:
 987           total_time: total elapsed time (in seconds) for the test run
 988           thread_timings: wall clock time each thread ran for
 989           directory_test_timings: timing by directory
 990           individual_test_timings: timing by file
 991           result_summary: summary object for the test run
 992         """
 993         self._printer.print_timing("Test timing:")
 994         self._printer.print_timing("  %6.2f total testing time" % total_time)
 995         self._printer.print_timing("")
 996         self._printer.print_timing("Thread timing:")
 997         cuml_time = 0
 998         for t in thread_timings:
 999             self._printer.print_timing("    %10s: %5d tests, %6.2f secs" %
1000                   (t['name'], t['num_tests'], t['total_time']))
1001             cuml_time += t['total_time']
1002         self._printer.print_timing("   %6.2f cumulative, %6.2f optimal" %
1003               (cuml_time, cuml_time / int(self._options.child_processes)))
1004         self._printer.print_timing("")
1005
1006         self._print_aggregate_test_statistics(individual_test_timings)
1007         self._print_individual_test_times(individual_test_timings,
1008                                           result_summary)
1009         self._print_directory_timings(directory_test_timings)
1010
1011     def _print_aggregate_test_statistics(self, individual_test_timings):
1012         """Prints aggregate statistics (e.g. median, mean, etc.) for all tests.
1013         Args:
1014           individual_test_timings: List of dump_render_tree_thread.TestStats
1015               for all tests.
1016         """
1017         test_types = []  # Unit tests don't actually produce any timings.
1018         if individual_test_timings:
1019             test_types = individual_test_timings[0].time_for_diffs.keys()
1020         times_for_dump_render_tree = []
1021         times_for_diff_processing = []
1022         times_per_test_type = {}
1023         for test_type in test_types:
1024             times_per_test_type[test_type] = []
1025
1026         for test_stats in individual_test_timings:
1027             times_for_dump_render_tree.append(test_stats.test_run_time)
1028             times_for_diff_processing.append(
1029                 test_stats.total_time_for_all_diffs)
1030             time_for_diffs = test_stats.time_for_diffs
1031             for test_type in test_types:
1032                 times_per_test_type[test_type].append(
1033                     time_for_diffs[test_type])
1034
1035         self._print_statistics_for_test_timings(
1036             "PER TEST TIME IN TESTSHELL (seconds):",
1037             times_for_dump_render_tree)
1038         self._print_statistics_for_test_timings(
1039             "PER TEST DIFF PROCESSING TIMES (seconds):",
1040             times_for_diff_processing)
1041         for test_type in test_types:
1042             self._print_statistics_for_test_timings(
1043                 "PER TEST TIMES BY TEST TYPE: %s" % test_type,
1044                 times_per_test_type[test_type])
1045
1046     def _print_individual_test_times(self, individual_test_timings,
1047                                   result_summary):
1048         """Prints the run times for slow, timeout and crash tests.
1049         Args:
1050           individual_test_timings: List of dump_render_tree_thread.TestStats
1051               for all tests.
1052           result_summary: summary object for test run
1053         """
1054         # Reverse-sort by the time spent in DumpRenderTree.
1055         individual_test_timings.sort(lambda a, b:
1056             cmp(b.test_run_time, a.test_run_time))
1057
1058         num_printed = 0
1059         slow_tests = []
1060         timeout_or_crash_tests = []
1061         unexpected_slow_tests = []
1062         for test_tuple in individual_test_timings:
1063             filename = test_tuple.filename
1064             is_timeout_crash_or_slow = False
1065             if self._test_is_slow(filename):
1066                 is_timeout_crash_or_slow = True
1067                 slow_tests.append(test_tuple)
1068
1069             if filename in result_summary.failures:
1070                 result = result_summary.results[filename].type
1071                 if (result == test_expectations.TIMEOUT or
1072                     result == test_expectations.CRASH):
1073                     is_timeout_crash_or_slow = True
1074                     timeout_or_crash_tests.append(test_tuple)
1075
1076             if (not is_timeout_crash_or_slow and
1077                 num_printed < printing.NUM_SLOW_TESTS_TO_LOG):
1078                 num_printed = num_printed + 1
1079                 unexpected_slow_tests.append(test_tuple)
1080
1081         self._printer.print_timing("")
1082         self._print_test_list_timing("%s slowest tests that are not "
1083             "marked as SLOW and did not timeout/crash:" %
1084             printing.NUM_SLOW_TESTS_TO_LOG, unexpected_slow_tests)
1085         self._printer.print_timing("")
1086         self._print_test_list_timing("Tests marked as SLOW:", slow_tests)
1087         self._printer.print_timing("")
1088         self._print_test_list_timing("Tests that timed out or crashed:",
1089                                      timeout_or_crash_tests)
1090         self._printer.print_timing("")
1091
1092     def _print_test_list_timing(self, title, test_list):
1093         """Print timing info for each test.
1094
1095         Args:
1096           title: section heading
1097           test_list: tests that fall in this section
1098         """
1099         if self._printer.disabled('slowest'):
1100             return
1101
1102         self._printer.print_timing(title)
1103         for test_tuple in test_list:
1104             filename = test_tuple.filename[len(
1105                 self._port.layout_tests_dir()) + 1:]
1106             filename = filename.replace('\\', '/')
1107             test_run_time = round(test_tuple.test_run_time, 1)
1108             self._printer.print_timing("  %s took %s seconds" %
1109                                        (filename, test_run_time))
1110
1111     def _print_directory_timings(self, directory_test_timings):
1112         """Print timing info by directory for any directories that
1113         take > 10 seconds to run.
1114
1115         Args:
1116           directory_test_timing: time info for each directory
1117         """
1118         timings = []
1119         for directory in directory_test_timings:
1120             num_tests, time_for_directory = directory_test_timings[directory]
1121             timings.append((round(time_for_directory, 1), directory,
1122                             num_tests))
1123         timings.sort()
1124
1125         self._printer.print_timing("Time to process slowest subdirectories:")
1126         min_seconds_to_print = 10
1127         for timing in timings:
1128             if timing[0] > min_seconds_to_print:
1129                 self._printer.print_timing(
1130                     "  %s took %s seconds to run %s tests." % (timing[1],
1131                     timing[0], timing[2]))
1132         self._printer.print_timing("")
1133
1134     def _print_statistics_for_test_timings(self, title, timings):
1135         """Prints the median, mean and standard deviation of the values in
1136         timings.
1137
1138         Args:
1139           title: Title for these timings.
1140           timings: A list of floats representing times.
1141         """
1142         self._printer.print_timing(title)
1143         timings.sort()
1144
1145         num_tests = len(timings)
1146         if not num_tests:
1147             return
1148         percentile90 = timings[int(.9 * num_tests)]
1149         percentile99 = timings[int(.99 * num_tests)]
1150
1151         if num_tests % 2 == 1:
1152             median = timings[((num_tests - 1) / 2) - 1]
1153         else:
1154             lower = timings[num_tests / 2 - 1]
1155             upper = timings[num_tests / 2]
1156             median = (float(lower + upper)) / 2
1157
1158         mean = sum(timings) / num_tests
1159
1160         for time in timings:
1161             sum_of_deviations = math.pow(time - mean, 2)
1162
1163         std_deviation = math.sqrt(sum_of_deviations / num_tests)
1164         self._printer.print_timing("  Median:          %6.3f" % median)
1165         self._printer.print_timing("  Mean:            %6.3f" % mean)
1166         self._printer.print_timing("  90th percentile: %6.3f" % percentile90)
1167         self._printer.print_timing("  99th percentile: %6.3f" % percentile99)
1168         self._printer.print_timing("  Standard dev:    %6.3f" % std_deviation)
1169         self._printer.print_timing("")
1170
1171     def _print_result_summary(self, result_summary):
1172         """Print a short summary about how many tests passed.
1173
1174         Args:
1175           result_summary: information to log
1176         """
1177         failed = len(result_summary.failures)
1178         skipped = len(
1179             result_summary.tests_by_expectation[test_expectations.SKIP])
1180         total = result_summary.total
1181         passed = total - failed - skipped
1182         pct_passed = 0.0
1183         if total > 0:
1184             pct_passed = float(passed) * 100 / total
1185
1186         self._printer.print_actual("")
1187         self._printer.print_actual("=> Results: %d/%d tests passed (%.1f%%)" %
1188                      (passed, total, pct_passed))
1189         self._printer.print_actual("")
1190         self._print_result_summary_entry(result_summary,
1191             test_expectations.NOW, "Tests to be fixed")
1192
1193         self._printer.print_actual("")
1194         self._print_result_summary_entry(result_summary,
1195             test_expectations.WONTFIX,
1196             "Tests that will only be fixed if they crash (WONTFIX)")
1197         self._printer.print_actual("")
1198
1199     def _print_result_summary_entry(self, result_summary, timeline,
1200                                     heading):
1201         """Print a summary block of results for a particular timeline of test.
1202
1203         Args:
1204           result_summary: summary to print results for
1205           timeline: the timeline to print results for (NOT, WONTFIX, etc.)
1206           heading: a textual description of the timeline
1207         """
1208         total = len(result_summary.tests_by_timeline[timeline])
1209         not_passing = (total -
1210            len(result_summary.tests_by_expectation[test_expectations.PASS] &
1211                result_summary.tests_by_timeline[timeline]))
1212         self._printer.print_actual("=> %s (%d):" % (heading, not_passing))
1213
1214         for result in TestExpectationsFile.EXPECTATION_ORDER:
1215             if result == test_expectations.PASS:
1216                 continue
1217             results = (result_summary.tests_by_expectation[result] &
1218                        result_summary.tests_by_timeline[timeline])
1219             desc = TestExpectationsFile.EXPECTATION_DESCRIPTIONS[result]
1220             if not_passing and len(results):
1221                 pct = len(results) * 100.0 / not_passing
1222                 self._printer.print_actual("  %5d %-24s (%4.1f%%)" %
1223                     (len(results), desc[len(results) != 1], pct))
1224
1225     def _results_html(self, test_files, failures, title="Test Failures", override_time=None):
1226         """
1227         test_files = a list of file paths
1228         failures = dictionary mapping test paths to failure objects
1229         title = title printed at top of test
1230         override_time = current time (used by unit tests)
1231         """
1232         page = """<html>
1233   <head>
1234     <title>Layout Test Results (%(time)s)</title>
1235   </head>
1236   <body>
1237     <h2>%(title)s (%(time)s)</h2>
1238         """ % {'title': title, 'time': override_time or time.asctime()}
1239
1240         for test_file in sorted(test_files):
1241             test_name = self._port.relative_test_filename(test_file)
1242             test_url = self._port.filename_to_uri(test_file)
1243             page += u"<p><a href='%s'>%s</a><br />\n" % (test_url, test_name)
1244             test_failures = failures.get(test_file, [])
1245             for failure in test_failures:
1246                 page += (u"&nbsp;&nbsp;%s<br/>" %
1247                          failure.result_html_output(test_name))
1248             page += "</p>\n"
1249         page += "</body></html>\n"
1250         return page
1251
1252     def _write_results_html_file(self, result_summary):
1253         """Write results.html which is a summary of tests that failed.
1254
1255         Args:
1256           result_summary: a summary of the results :)
1257
1258         Returns:
1259           True if any results were written (since expected failures may be
1260           omitted)
1261         """
1262         # test failures
1263         if self._options.full_results_html:
1264             results_title = "Test Failures"
1265             test_files = result_summary.failures.keys()
1266         else:
1267             results_title = "Unexpected Test Failures"
1268             unexpected_failures = self._get_failures(result_summary,
1269                 include_crashes=True)
1270             test_files = unexpected_failures.keys()
1271         if not len(test_files):
1272             return False
1273
1274         out_filename = os.path.join(self._options.results_directory,
1275                                     "results.html")
1276         with codecs.open(out_filename, "w", "utf-8") as results_file:
1277             html = self._results_html(test_files, result_summary.failures, results_title)
1278             results_file.write(html)
1279
1280         return True
1281
1282     def _show_results_html_file(self):
1283         """Shows the results.html page."""
1284         results_filename = os.path.join(self._options.results_directory,
1285                                         "results.html")
1286         self._port.show_results_html_file(results_filename)
1287
1288
1289 def read_test_files(files):
1290     tests = []
1291     for file in files:
1292         try:
1293             with codecs.open(file, 'r', 'utf-8') as file_contents:
1294                 # FIXME: This could be cleaner using a list comprehension.
1295                 for line in file_contents:
1296                     line = test_expectations.strip_comments(line)
1297                     if line:
1298                         tests.append(line)
1299         except IOError, e:
1300             if e.errno == errno.ENOENT:
1301                 _log.critical('')
1302                 _log.critical('--test-list file "%s" not found' % file)
1303             raise
1304     return tests
1305
1306
1307 def run(port, options, args, regular_output=sys.stderr,
1308         buildbot_output=sys.stdout):
1309     """Run the tests.
1310
1311     Args:
1312       port: Port object for port-specific behavior
1313       options: a dictionary of command line options
1314       args: a list of sub directories or files to test
1315       regular_output: a stream-like object that we can send logging/debug
1316           output to
1317       buildbot_output: a stream-like object that we can write all output that
1318           is intended to be parsed by the buildbot to
1319     Returns:
1320       the number of unexpected results that occurred, or -1 if there is an
1321           error.
1322
1323     """
1324     _set_up_derived_options(port, options)
1325
1326     printer = printing.Printer(port, options, regular_output, buildbot_output,
1327         int(options.child_processes), options.experimental_fully_parallel)
1328     if options.help_printing:
1329         printer.help_printing()
1330         printer.cleanup()
1331         return 0
1332
1333     last_unexpected_results = _gather_unexpected_results(options)
1334     if options.print_last_failures:
1335         printer.write("\n".join(last_unexpected_results) + "\n")
1336         printer.cleanup()
1337         return 0
1338
1339     broker = message_broker.get(port, options)
1340
1341     # We wrap any parts of the run that are slow or likely to raise exceptions
1342     # in a try/finally to ensure that we clean up the logging configuration.
1343     num_unexpected_results = -1
1344     try:
1345         test_runner = TestRunner(port, options, printer, broker)
1346         test_runner._print_config()
1347
1348         printer.print_update("Collecting tests ...")
1349         try:
1350             test_runner.collect_tests(args, last_unexpected_results)
1351         except IOError, e:
1352             if e.errno == errno.ENOENT:
1353                 return -1
1354             raise
1355
1356         printer.print_update("Parsing expectations ...")
1357         if options.lint_test_files:
1358             return test_runner.lint()
1359         test_runner.parse_expectations(port.test_platform_name(),
1360                                        options.configuration == 'Debug')
1361
1362         printer.print_update("Checking build ...")
1363         if not port.check_build(test_runner.needs_http()):
1364             _log.error("Build check failed")
1365             return -1
1366
1367         result_summary = test_runner.set_up_run()
1368         if result_summary:
1369             num_unexpected_results = test_runner.run(result_summary)
1370             test_runner.clean_up_run()
1371             _log.debug("Testing completed, Exit status: %d" %
1372                        num_unexpected_results)
1373     finally:
1374         broker.cleanup()
1375         printer.cleanup()
1376
1377     return num_unexpected_results
1378
1379
1380 def _set_up_derived_options(port_obj, options):
1381     """Sets the options values that depend on other options values."""
1382
1383     if options.worker_model == 'inline':
1384         if options.child_processes and int(options.child_processes) > 1:
1385             _log.warning("--worker-model=inline overrides --child-processes")
1386         options.child_processes = "1"
1387     if not options.child_processes:
1388         options.child_processes = os.environ.get("WEBKIT_TEST_CHILD_PROCESSES",
1389                                                  str(port_obj.default_child_processes()))
1390
1391     if not options.configuration:
1392         options.configuration = port_obj.default_configuration()
1393
1394     if options.pixel_tests is None:
1395         options.pixel_tests = True
1396
1397     if not options.use_apache:
1398         options.use_apache = sys.platform in ('darwin', 'linux2')
1399
1400     if not os.path.isabs(options.results_directory):
1401         # This normalizes the path to the build dir.
1402         # FIXME: how this happens is not at all obvious; this is a dumb
1403         # interface and should be cleaned up.
1404         options.results_directory = port_obj.results_directory()
1405
1406     if not options.time_out_ms:
1407         if options.configuration == "Debug":
1408             options.time_out_ms = str(2 * TestRunner.DEFAULT_TEST_TIMEOUT_MS)
1409         else:
1410             options.time_out_ms = str(TestRunner.DEFAULT_TEST_TIMEOUT_MS)
1411
1412     options.slow_time_out_ms = str(5 * int(options.time_out_ms))
1413
1414
1415 def _gather_unexpected_results(options):
1416     """Returns the unexpected results from the previous run, if any."""
1417     last_unexpected_results = []
1418     if options.print_last_failures or options.retest_last_failures:
1419         unexpected_results_filename = os.path.join(
1420         options.results_directory, "unexpected_results.json")
1421         with codecs.open(unexpected_results_filename, "r", "utf-8") as file:
1422             results = simplejson.load(file)
1423         last_unexpected_results = results['tests'].keys()
1424     return last_unexpected_results
1425
1426
1427 def _compat_shim_callback(option, opt_str, value, parser):
1428     print "Ignoring unsupported option: %s" % opt_str
1429
1430
1431 def _compat_shim_option(option_name, **kwargs):
1432     return optparse.make_option(option_name, action="callback",
1433         callback=_compat_shim_callback,
1434         help="Ignored, for old-run-webkit-tests compat only.", **kwargs)
1435
1436
1437 def parse_args(args=None):
1438     """Provides a default set of command line args.
1439
1440     Returns a tuple of options, args from optparse"""
1441
1442     # FIXME: All of these options should be stored closer to the code which
1443     # FIXME: actually uses them. configuration_options should move
1444     # FIXME: to WebKitPort and be shared across all scripts.
1445     configuration_options = [
1446         optparse.make_option("-t", "--target", dest="configuration",
1447                              help="(DEPRECATED)"),
1448         # FIXME: --help should display which configuration is default.
1449         optparse.make_option('--debug', action='store_const', const='Debug',
1450                              dest="configuration",
1451                              help='Set the configuration to Debug'),
1452         optparse.make_option('--release', action='store_const',
1453                              const='Release', dest="configuration",
1454                              help='Set the configuration to Release'),
1455         # old-run-webkit-tests also accepts -c, --configuration CONFIGURATION.
1456     ]
1457
1458     print_options = printing.print_options()
1459
1460     # FIXME: These options should move onto the ChromiumPort.
1461     chromium_options = [
1462         optparse.make_option("--chromium", action="store_true", default=False,
1463             help="use the Chromium port"),
1464         optparse.make_option("--startup-dialog", action="store_true",
1465             default=False, help="create a dialog on DumpRenderTree startup"),
1466         optparse.make_option("--gp-fault-error-box", action="store_true",
1467             default=False, help="enable Windows GP fault error box"),
1468         optparse.make_option("--multiple-loads",
1469             type="int", help="turn on multiple loads of each test"),
1470         optparse.make_option("--js-flags",
1471             type="string", help="JavaScript flags to pass to tests"),
1472         optparse.make_option("--nocheck-sys-deps", action="store_true",
1473             default=False,
1474             help="Don't check the system dependencies (themes)"),
1475         optparse.make_option("--use-test-shell", action="store_true",
1476             default=False,
1477             help="Use test_shell instead of DRT"),
1478         optparse.make_option("--accelerated-compositing",
1479             action="store_true",
1480             help="Use hardware-accelated compositing for rendering"),
1481         optparse.make_option("--no-accelerated-compositing",
1482             action="store_false",
1483             dest="accelerated_compositing",
1484             help="Don't use hardware-accelerated compositing for rendering"),
1485         optparse.make_option("--accelerated-2d-canvas",
1486             action="store_true",
1487             help="Use hardware-accelerated 2D Canvas calls"),
1488         optparse.make_option("--no-accelerated-2d-canvas",
1489             action="store_false",
1490             dest="accelerated_2d_canvas",
1491             help="Don't use hardware-accelerated 2D Canvas calls"),
1492     ]
1493
1494     # Missing Mac-specific old-run-webkit-tests options:
1495     # FIXME: Need: -g, --guard for guard malloc support on Mac.
1496     # FIXME: Need: -l --leaks    Enable leaks checking.
1497     # FIXME: Need: --sample-on-timeout Run sample on timeout
1498
1499     old_run_webkit_tests_compat = [
1500         # NRWT doesn't generate results by default anyway.
1501         _compat_shim_option("--no-new-test-results"),
1502         # NRWT doesn't sample on timeout yet anyway.
1503         _compat_shim_option("--no-sample-on-timeout"),
1504         # FIXME: NRWT needs to support remote links eventually.
1505         _compat_shim_option("--use-remote-links-to-tests"),
1506     ]
1507
1508     results_options = [
1509         # NEED for bots: --use-remote-links-to-tests Link to test files
1510         # within the SVN repository in the results.
1511         optparse.make_option("-p", "--pixel-tests", action="store_true",
1512             dest="pixel_tests", help="Enable pixel-to-pixel PNG comparisons"),
1513         optparse.make_option("--no-pixel-tests", action="store_false",
1514             dest="pixel_tests", help="Disable pixel-to-pixel PNG comparisons"),
1515         optparse.make_option("--tolerance",
1516             help="Ignore image differences less than this percentage (some "
1517                 "ports may ignore this option)", type="float"),
1518         optparse.make_option("--results-directory",
1519             default="layout-test-results",
1520             help="Output results directory source dir, relative to Debug or "
1521                  "Release"),
1522         optparse.make_option("--new-baseline", action="store_true",
1523             default=False, help="Save all generated results as new baselines "
1524                  "into the platform directory, overwriting whatever's "
1525                  "already there."),
1526         optparse.make_option("--reset-results", action="store_true",
1527             default=False, help="Reset any existing baselines to the "
1528                  "generated results"),
1529         optparse.make_option("--no-show-results", action="store_false",
1530             default=True, dest="show_results",
1531             help="Don't launch a browser with results after the tests "
1532                  "are done"),
1533         # FIXME: We should have a helper function to do this sort of
1534         # deprectated mapping and automatically log, etc.
1535         optparse.make_option("--noshow-results", action="store_false",
1536             dest="show_results",
1537             help="Deprecated, same as --no-show-results."),
1538         optparse.make_option("--no-launch-safari", action="store_false",
1539             dest="show_results",
1540             help="old-run-webkit-tests compat, same as --noshow-results."),
1541         # old-run-webkit-tests:
1542         # --[no-]launch-safari    Launch (or do not launch) Safari to display
1543         #                         test results (default: launch)
1544         optparse.make_option("--full-results-html", action="store_true",
1545             default=False,
1546             help="Show all failures in results.html, rather than only "
1547                  "regressions"),
1548         optparse.make_option("--clobber-old-results", action="store_true",
1549             default=False, help="Clobbers test results from previous runs."),
1550         optparse.make_option("--platform",
1551             help="Override the platform for expected results"),
1552         optparse.make_option("--no-record-results", action="store_false",
1553             default=True, dest="record_results",
1554             help="Don't record the results."),
1555         # old-run-webkit-tests also has HTTP toggle options:
1556         # --[no-]http                     Run (or do not run) http tests
1557         #                                 (default: run)
1558     ]
1559
1560     test_options = [
1561         optparse.make_option("--build", dest="build",
1562             action="store_true", default=True,
1563             help="Check to ensure the DumpRenderTree build is up-to-date "
1564                  "(default)."),
1565         optparse.make_option("--no-build", dest="build",
1566             action="store_false", help="Don't check to see if the "
1567                                        "DumpRenderTree build is up-to-date."),
1568         optparse.make_option("-n", "--dry-run", action="store_true",
1569             default=False,
1570             help="Do everything but actually run the tests or upload results."),
1571         # old-run-webkit-tests has --valgrind instead of wrapper.
1572         optparse.make_option("--wrapper",
1573             help="wrapper command to insert before invocations of "
1574                  "DumpRenderTree; option is split on whitespace before "
1575                  "running. (Example: --wrapper='valgrind --smc-check=all')"),
1576         # old-run-webkit-tests:
1577         # -i|--ignore-tests               Comma-separated list of directories
1578         #                                 or tests to ignore
1579         optparse.make_option("--test-list", action="append",
1580             help="read list of tests to run from file", metavar="FILE"),
1581         # old-run-webkit-tests uses --skipped==[default|ignore|only]
1582         # instead of --force:
1583         optparse.make_option("--force", action="store_true", default=False,
1584             help="Run all tests, even those marked SKIP in the test list"),
1585         optparse.make_option("--use-apache", action="store_true",
1586             default=False, help="Whether to use apache instead of lighttpd."),
1587         optparse.make_option("--time-out-ms",
1588             help="Set the timeout for each test"),
1589         # old-run-webkit-tests calls --randomize-order --random:
1590         optparse.make_option("--randomize-order", action="store_true",
1591             default=False, help=("Run tests in random order (useful "
1592                                 "for tracking down corruption)")),
1593         optparse.make_option("--run-chunk",
1594             help=("Run a specified chunk (n:l), the nth of len l, "
1595                  "of the layout tests")),
1596         optparse.make_option("--run-part", help=("Run a specified part (n:m), "
1597                   "the nth of m parts, of the layout tests")),
1598         # old-run-webkit-tests calls --batch-size: --nthly n
1599         #   Restart DumpRenderTree every n tests (default: 1000)
1600         optparse.make_option("--batch-size",
1601             help=("Run a the tests in batches (n), after every n tests, "
1602                   "DumpRenderTree is relaunched."), type="int", default=0),
1603         # old-run-webkit-tests calls --run-singly: -1|--singly
1604         # Isolate each test case run (implies --nthly 1 --verbose)
1605         optparse.make_option("--run-singly", action="store_true",
1606             default=False, help="run a separate DumpRenderTree for each test"),
1607         optparse.make_option("--child-processes",
1608             help="Number of DumpRenderTrees to run in parallel."),
1609         # FIXME: Display default number of child processes that will run.
1610         optparse.make_option("--worker-model", action="store",
1611             default="threads", help=("controls worker model. Valid values are "
1612             "'inline' and 'threads' (default).")),
1613         optparse.make_option("--experimental-fully-parallel",
1614             action="store_true", default=False,
1615             help="run all tests in parallel"),
1616         optparse.make_option("--exit-after-n-failures", type="int", nargs=1,
1617             help="Exit after the first N failures instead of running all "
1618             "tests"),
1619         optparse.make_option("--exit-after-n-crashes-or-timeouts", type="int",
1620             nargs=1, help="Exit after the first N crashes instead of running "
1621             "all tests"),
1622         # FIXME: consider: --iterations n
1623         #      Number of times to run the set of tests (e.g. ABCABCABC)
1624         optparse.make_option("--print-last-failures", action="store_true",
1625             default=False, help="Print the tests in the last run that "
1626             "had unexpected failures (or passes) and then exit."),
1627         optparse.make_option("--retest-last-failures", action="store_true",
1628             default=False, help="re-test the tests in the last run that "
1629             "had unexpected failures (or passes)."),
1630         optparse.make_option("--retry-failures", action="store_true",
1631             default=True,
1632             help="Re-try any tests that produce unexpected results (default)"),
1633         optparse.make_option("--no-retry-failures", action="store_false",
1634             dest="retry_failures",
1635             help="Don't re-try any tests that produce unexpected results."),
1636     ]
1637
1638     misc_options = [
1639         optparse.make_option("--lint-test-files", action="store_true",
1640         default=False, help=("Makes sure the test files parse for all "
1641                             "configurations. Does not run any tests.")),
1642     ]
1643
1644     # FIXME: Move these into json_results_generator.py
1645     results_json_options = [
1646         optparse.make_option("--master-name", help="The name of the buildbot master."),
1647         optparse.make_option("--builder-name", default="DUMMY_BUILDER_NAME",
1648             help=("The name of the builder shown on the waterfall running "
1649                   "this script e.g. WebKit.")),
1650         optparse.make_option("--build-name", default="DUMMY_BUILD_NAME",
1651             help=("The name of the builder used in its path, e.g. "
1652                   "webkit-rel.")),
1653         optparse.make_option("--build-number", default="DUMMY_BUILD_NUMBER",
1654             help=("The build number of the builder running this script.")),
1655         optparse.make_option("--test-results-server", default="",
1656             help=("If specified, upload results json files to this appengine "
1657                   "server.")),
1658         optparse.make_option("--upload-full-results",
1659             action="store_true",
1660             default=False,
1661             help="If true, upload full json results to server."),
1662     ]
1663
1664     option_list = (configuration_options + print_options +
1665                    chromium_options + results_options + test_options +
1666                    misc_options + results_json_options +
1667                    old_run_webkit_tests_compat)
1668     option_parser = optparse.OptionParser(option_list=option_list)
1669
1670     return option_parser.parse_args(args)
1671
1672
1673 def main():
1674     options, args = parse_args()
1675     port_obj = port.get(options.platform, options)
1676     return run(port_obj, options, args)
1677
1678
1679 if '__main__' == __name__:
1680     try:
1681         sys.exit(main())
1682     except KeyboardInterrupt:
1683         # this mirrors what the shell normally does
1684         sys.exit(signal.SIGINT + 128)