WebKitTools/Scripts/webkitpy/layout_tests/run_webkit_tests.py

   1 #!/usr/bin/env python
   2 # Copyright (C) 2010 Google Inc. All rights reserved.
   3 # Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Szeged
   4 #
   5 # Redistribution and use in source and binary forms, with or without
   6 # modification, are permitted provided that the following conditions are
   7 # met:
   8 #
   9 #     * Redistributions of source code must retain the above copyright
  10 # notice, this list of conditions and the following disclaimer.
  11 #     * Redistributions in binary form must reproduce the above
  12 # copyright notice, this list of conditions and the following disclaimer
  13 # in the documentation and/or other materials provided with the
  14 # distribution.
  15 #     * Neither the name of Google Inc. nor the names of its
  16 # contributors may be used to endorse or promote products derived from
  17 # this software without specific prior written permission.
  18 #
  19 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  20 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  21 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  22 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  23 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  24 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  25 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  26 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  27 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  28 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  29 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  30
  31 """Run layout tests.
  32
  33 This is a port of the existing webkit test script run-webkit-tests.
  34
  35 The TestRunner class runs a series of tests (TestType interface) against a set
  36 of test files.  If a test file fails a TestType, it returns a list TestFailure
  37 objects to the TestRunner.  The TestRunner then aggregates the TestFailures to
  38 create a final report.
  39
  40 This script reads several files, if they exist in the test_lists subdirectory
  41 next to this script itself.  Each should contain a list of paths to individual
  42 tests or entire subdirectories of tests, relative to the outermost test
  43 directory.  Entire lines starting with '//' (comments) will be ignored.
  44
  45 For details of the files' contents and purposes, see test_lists/README.
  46 """
  47
  48 from __future__ import with_statement
  49
  50 import codecs
  51 import errno
  52 import glob
  53 import logging
  54 import math
  55 import optparse
  56 import os
  57 import platform
  58 import Queue
  59 import random
  60 import re
  61 import shutil
  62 import signal
  63 import sys
  64 import time
  65 import traceback
  66
  67 from layout_package import dump_render_tree_thread
  68 from layout_package import json_layout_results_generator
  69 from layout_package import message_broker
  70 from layout_package import printing
  71 from layout_package import test_expectations
  72 from layout_package import test_failures
  73 from layout_package import test_results
  74 from layout_package import test_results_uploader
  75
  76 from webkitpy.common.system import user
  77 from webkitpy.thirdparty import simplejson
  78
  79 import port
  80
  81 _log = logging.getLogger("webkitpy.layout_tests.run_webkit_tests")
  82
  83 # Builder base URL where we have the archived test results.
  84 BUILDER_BASE_URL = "http://build.chromium.org/buildbot/layout_test_results/"
  85
  86 LAYOUT_TESTS_DIRECTORY = "LayoutTests" + os.sep
  87
  88 TestExpectationsFile = test_expectations.TestExpectationsFile
  89
  90
  91 class TestInput:
  92     """Groups information about a test for easy passing of data."""
  93
  94     def __init__(self, filename, timeout):
  95         """Holds the input parameters for a test.
  96         Args:
  97           filename: Full path to the test.
  98           timeout: Timeout in msecs the driver should use while running the test
  99           """
 100         # FIXME: filename should really be test_name as a relative path.
 101         self.filename = filename
 102         self.timeout = timeout
 103         # The image_hash is used to avoid doing an image dump if the
 104         # checksums match. The image_hash is set later, and only if it is needed
 105         # for the test.
 106         self.image_hash = None
 107
 108
 109 class ResultSummary(object):
 110     """A class for partitioning the test results we get into buckets.
 111
 112     This class is basically a glorified struct and it's private to this file
 113     so we don't bother with any information hiding."""
 114
 115     def __init__(self, expectations, test_files):
 116         self.total = len(test_files)
 117         self.remaining = self.total
 118         self.expectations = expectations
 119         self.expected = 0
 120         self.unexpected = 0
 121         self.tests_by_expectation = {}
 122         self.tests_by_timeline = {}
 123         self.results = {}
 124         self.unexpected_results = {}
 125         self.failures = {}
 126         self.tests_by_expectation[test_expectations.SKIP] = set()
 127         for expectation in TestExpectationsFile.EXPECTATIONS.values():
 128             self.tests_by_expectation[expectation] = set()
 129         for timeline in TestExpectationsFile.TIMELINES.values():
 130             self.tests_by_timeline[timeline] = (
 131                 expectations.get_tests_with_timeline(timeline))
 132
 133     def add(self, result, expected):
 134         """Add a TestResult into the appropriate bin.
 135
 136         Args:
 137           result: TestResult from dump_render_tree_thread.
 138           expected: whether the result was what we expected it to be.
 139         """
 140
 141         self.tests_by_expectation[result.type].add(result.filename)
 142         self.results[result.filename] = result
 143         self.remaining -= 1
 144         if len(result.failures):
 145             self.failures[result.filename] = result.failures
 146         if expected:
 147             self.expected += 1
 148         else:
 149             self.unexpected_results[result.filename] = result.type
 150             self.unexpected += 1
 151
 152
 153 def summarize_unexpected_results(port_obj, expectations, result_summary,
 154                                  retry_summary):
 155     """Summarize any unexpected results as a dict.
 156
 157     FIXME: split this data structure into a separate class?
 158
 159     Args:
 160         port_obj: interface to port-specific hooks
 161         expectations: test_expectations.TestExpectations object
 162         result_summary: summary object from initial test runs
 163         retry_summary: summary object from final test run of retried tests
 164     Returns:
 165         A dictionary containing a summary of the unexpected results from the
 166         run, with the following fields:
 167         'version': a version indicator (1 in this version)
 168         'fixable': # of fixable tests (NOW - PASS)
 169         'skipped': # of skipped tests (NOW & SKIPPED)
 170         'num_regressions': # of non-flaky failures
 171         'num_flaky': # of flaky failures
 172         'num_passes': # of unexpected passes
 173         'tests': a dict of tests -> {'expected': '...', 'actual': '...'}
 174     """
 175     results = {}
 176     results['version'] = 1
 177
 178     tbe = result_summary.tests_by_expectation
 179     tbt = result_summary.tests_by_timeline
 180     results['fixable'] = len(tbt[test_expectations.NOW] -
 181                                 tbe[test_expectations.PASS])
 182     results['skipped'] = len(tbt[test_expectations.NOW] &
 183                                 tbe[test_expectations.SKIP])
 184
 185     num_passes = 0
 186     num_flaky = 0
 187     num_regressions = 0
 188     keywords = {}
 189     for k, v in TestExpectationsFile.EXPECTATIONS.iteritems():
 190         keywords[v] = k.upper()
 191
 192     tests = {}
 193     for filename, result in result_summary.unexpected_results.iteritems():
 194         # Note that if a test crashed in the original run, we ignore
 195         # whether or not it crashed when we retried it (if we retried it),
 196         # and always consider the result not flaky.
 197         test = port_obj.relative_test_filename(filename)
 198         expected = expectations.get_expectations_string(filename)
 199         actual = [keywords[result]]
 200
 201         if result == test_expectations.PASS:
 202             num_passes += 1
 203         elif result == test_expectations.CRASH:
 204             num_regressions += 1
 205         else:
 206             if filename not in retry_summary.unexpected_results:
 207                 actual.extend(expectations.get_expectations_string(
 208                     filename).split(" "))
 209                 num_flaky += 1
 210             else:
 211                 retry_result = retry_summary.unexpected_results[filename]
 212                 if result != retry_result:
 213                     actual.append(keywords[retry_result])
 214                     num_flaky += 1
 215                 else:
 216                     num_regressions += 1
 217
 218         tests[test] = {}
 219         tests[test]['expected'] = expected
 220         tests[test]['actual'] = " ".join(actual)
 221
 222     results['tests'] = tests
 223     results['num_passes'] = num_passes
 224     results['num_flaky'] = num_flaky
 225     results['num_regressions'] = num_regressions
 226
 227     return results
 228
 229
 230 class TestRunner:
 231     """A class for managing running a series of tests on a series of layout
 232     test files."""
 233
 234     HTTP_SUBDIR = os.sep.join(['', 'http', ''])
 235     WEBSOCKET_SUBDIR = os.sep.join(['', 'websocket', ''])
 236
 237     # The per-test timeout in milliseconds, if no --time-out-ms option was
 238     # given to run_webkit_tests. This should correspond to the default timeout
 239     # in DumpRenderTree.
 240     DEFAULT_TEST_TIMEOUT_MS = 6 * 1000
 241
 242     def __init__(self, port, options, printer):
 243         """Initialize test runner data structures.
 244
 245         Args:
 246           port: an object implementing port-specific
 247           options: a dictionary of command line options
 248           printer: a Printer object to record updates to.
 249         """
 250         self._port = port
 251         self._options = options
 252         self._printer = printer
 253
 254         # disable wss server. need to install pyOpenSSL on buildbots.
 255         # self._websocket_secure_server = websocket_server.PyWebSocket(
 256         #        options.results_directory, use_tls=True, port=9323)
 257
 258         # a set of test files, and the same tests as a list
 259         self._test_files = set()
 260         self._test_files_list = None
 261         self._result_queue = Queue.Queue()
 262         self._retrying = False
 263
 264     def collect_tests(self, args, last_unexpected_results):
 265         """Find all the files to test.
 266
 267         Args:
 268           args: list of test arguments from the command line
 269           last_unexpected_results: list of unexpected results to retest, if any
 270
 271         """
 272         paths = [self._strip_test_dir_prefix(arg) for arg in args if arg and arg != '']
 273         paths += last_unexpected_results
 274         if self._options.test_list:
 275             paths += read_test_files(self._options.test_list)
 276         self._test_files = self._port.tests(paths)
 277
 278     def _strip_test_dir_prefix(self, path):
 279         if path.startswith(LAYOUT_TESTS_DIRECTORY):
 280             return path[len(LAYOUT_TESTS_DIRECTORY):]
 281         return path
 282
 283     def lint(self):
 284         # Creating the expecations for each platform/configuration pair does
 285         # all the test list parsing and ensures it's correct syntax (e.g. no
 286         # dupes).
 287         for platform_name in self._port.test_platform_names():
 288             self.parse_expectations(platform_name, is_debug_mode=True)
 289             self.parse_expectations(platform_name, is_debug_mode=False)
 290         self._printer.write("")
 291         _log.info("If there are no fail messages, errors or exceptions, "
 292                   "then the lint succeeded.")
 293         return 0
 294
 295     def parse_expectations(self, test_platform_name, is_debug_mode):
 296         """Parse the expectations from the test_list files and return a data
 297         structure holding them. Throws an error if the test_list files have
 298         invalid syntax."""
 299         if self._options.lint_test_files:
 300             test_files = None
 301         else:
 302             test_files = self._test_files
 303
 304         try:
 305             expectations_str = self._port.test_expectations()
 306             overrides_str = self._port.test_expectations_overrides()
 307             self._expectations = test_expectations.TestExpectations(
 308                 self._port, test_files, expectations_str, test_platform_name,
 309                 is_debug_mode, self._options.lint_test_files,
 310                 overrides=overrides_str)
 311             return self._expectations
 312         except SyntaxError, err:
 313             if self._options.lint_test_files:
 314                 print str(err)
 315             else:
 316                 raise err
 317
 318     def prepare_lists_and_print_output(self):
 319         """Create appropriate subsets of test lists and returns a
 320         ResultSummary object. Also prints expected test counts.
 321         """
 322
 323         # Remove skipped - both fixable and ignored - files from the
 324         # top-level list of files to test.
 325         num_all_test_files = len(self._test_files)
 326         self._printer.print_expected("Found:  %d tests" %
 327                                      (len(self._test_files)))
 328         if not num_all_test_files:
 329             _log.critical('No tests to run.')
 330             return None
 331
 332         skipped = set()
 333         if num_all_test_files > 1 and not self._options.force:
 334             skipped = self._expectations.get_tests_with_result_type(
 335                            test_expectations.SKIP)
 336             self._test_files -= skipped
 337
 338         # Create a sorted list of test files so the subset chunk,
 339         # if used, contains alphabetically consecutive tests.
 340         self._test_files_list = list(self._test_files)
 341         if self._options.randomize_order:
 342             random.shuffle(self._test_files_list)
 343         else:
 344             self._test_files_list.sort()
 345
 346         # If the user specifies they just want to run a subset of the tests,
 347         # just grab a subset of the non-skipped tests.
 348         if self._options.run_chunk or self._options.run_part:
 349             chunk_value = self._options.run_chunk or self._options.run_part
 350             test_files = self._test_files_list
 351             try:
 352                 (chunk_num, chunk_len) = chunk_value.split(":")
 353                 chunk_num = int(chunk_num)
 354                 assert(chunk_num >= 0)
 355                 test_size = int(chunk_len)
 356                 assert(test_size > 0)
 357             except:
 358                 _log.critical("invalid chunk '%s'" % chunk_value)
 359                 return None
 360
 361             # Get the number of tests
 362             num_tests = len(test_files)
 363
 364             # Get the start offset of the slice.
 365             if self._options.run_chunk:
 366                 chunk_len = test_size
 367                 # In this case chunk_num can be really large. We need
 368                 # to make the slave fit in the current number of tests.
 369                 slice_start = (chunk_num * chunk_len) % num_tests
 370             else:
 371                 # Validate the data.
 372                 assert(test_size <= num_tests)
 373                 assert(chunk_num <= test_size)
 374
 375                 # To count the chunk_len, and make sure we don't skip
 376                 # some tests, we round to the next value that fits exactly
 377                 # all the parts.
 378                 rounded_tests = num_tests
 379                 if rounded_tests % test_size != 0:
 380                     rounded_tests = (num_tests + test_size -
 381                                      (num_tests % test_size))
 382
 383                 chunk_len = rounded_tests / test_size
 384                 slice_start = chunk_len * (chunk_num - 1)
 385                 # It does not mind if we go over test_size.
 386
 387             # Get the end offset of the slice.
 388             slice_end = min(num_tests, slice_start + chunk_len)
 389
 390             files = test_files[slice_start:slice_end]
 391
 392             tests_run_msg = 'Running: %d tests (chunk slice [%d:%d] of %d)' % (
 393                 (slice_end - slice_start), slice_start, slice_end, num_tests)
 394             self._printer.print_expected(tests_run_msg)
 395
 396             # If we reached the end and we don't have enough tests, we run some
 397             # from the beginning.
 398             if slice_end - slice_start < chunk_len:
 399                 extra = chunk_len - (slice_end - slice_start)
 400                 extra_msg = ('   last chunk is partial, appending [0:%d]' %
 401                             extra)
 402                 self._printer.print_expected(extra_msg)
 403                 tests_run_msg += "\n" + extra_msg
 404                 files.extend(test_files[0:extra])
 405             tests_run_filename = os.path.join(self._options.results_directory,
 406                                               "tests_run.txt")
 407             with codecs.open(tests_run_filename, "w", "utf-8") as file:
 408                 file.write(tests_run_msg + "\n")
 409
 410             len_skip_chunk = int(len(files) * len(skipped) /
 411                                  float(len(self._test_files)))
 412             skip_chunk_list = list(skipped)[0:len_skip_chunk]
 413             skip_chunk = set(skip_chunk_list)
 414
 415             # Update expectations so that the stats are calculated correctly.
 416             # We need to pass a list that includes the right # of skipped files
 417             # to ParseExpectations so that ResultSummary() will get the correct
 418             # stats. So, we add in the subset of skipped files, and then
 419             # subtract them back out.
 420             self._test_files_list = files + skip_chunk_list
 421             self._test_files = set(self._test_files_list)
 422
 423             self._expectations = self.parse_expectations(
 424                 self._port.test_platform_name(),
 425                 self._options.configuration == 'Debug')
 426
 427             self._test_files = set(files)
 428             self._test_files_list = files
 429         else:
 430             skip_chunk = skipped
 431
 432         result_summary = ResultSummary(self._expectations,
 433             self._test_files | skip_chunk)
 434         self._print_expected_results_of_type(result_summary,
 435             test_expectations.PASS, "passes")
 436         self._print_expected_results_of_type(result_summary,
 437             test_expectations.FAIL, "failures")
 438         self._print_expected_results_of_type(result_summary,
 439             test_expectations.FLAKY, "flaky")
 440         self._print_expected_results_of_type(result_summary,
 441             test_expectations.SKIP, "skipped")
 442
 443         if self._options.force:
 444             self._printer.print_expected('Running all tests, including '
 445                                          'skips (--force)')
 446         else:
 447             # Note that we don't actually run the skipped tests (they were
 448             # subtracted out of self._test_files, above), but we stub out the
 449             # results here so the statistics can remain accurate.
 450             for test in skip_chunk:
 451                 result = test_results.TestResult(test,
 452                     failures=[], test_run_time=0, total_time_for_all_diffs=0,
 453                     time_for_diffs=0)
 454                 result.type = test_expectations.SKIP
 455                 result_summary.add(result, expected=True)
 456         self._printer.print_expected('')
 457
 458         return result_summary
 459
 460     def _get_dir_for_test_file(self, test_file):
 461         """Returns the highest-level directory by which to shard the given
 462         test file."""
 463         index = test_file.rfind(os.sep + LAYOUT_TESTS_DIRECTORY)
 464
 465         test_file = test_file[index + len(LAYOUT_TESTS_DIRECTORY):]
 466         test_file_parts = test_file.split(os.sep, 1)
 467         directory = test_file_parts[0]
 468         test_file = test_file_parts[1]
 469
 470         # The http tests are very stable on mac/linux.
 471         # TODO(ojan): Make the http server on Windows be apache so we can
 472         # turn shard the http tests there as well. Switching to apache is
 473         # what made them stable on linux/mac.
 474         return_value = directory
 475         while ((directory != 'http' or sys.platform in ('darwin', 'linux2'))
 476                 and test_file.find(os.sep) >= 0):
 477             test_file_parts = test_file.split(os.sep, 1)
 478             directory = test_file_parts[0]
 479             return_value = os.path.join(return_value, directory)
 480             test_file = test_file_parts[1]
 481
 482         return return_value
 483
 484     def _get_test_input_for_file(self, test_file):
 485         """Returns the appropriate TestInput object for the file. Mostly this
 486         is used for looking up the timeout value (in ms) to use for the given
 487         test."""
 488         if self._test_is_slow(test_file):
 489             return TestInput(test_file, self._options.slow_time_out_ms)
 490         return TestInput(test_file, self._options.time_out_ms)
 491
 492     def _test_requires_lock(self, test_file):
 493         """Return True if the test needs to be locked when
 494         running multiple copies of NRWTs."""
 495         split_path = test_file.split(os.sep)
 496         return 'http' in split_path or 'websocket' in split_path
 497
 498     def _test_is_slow(self, test_file):
 499         return self._expectations.has_modifier(test_file,
 500                                                test_expectations.SLOW)
 501
 502     def _shard_tests(self, test_files, use_real_shards):
 503         """Groups tests into batches.
 504         This helps ensure that tests that depend on each other (aka bad tests!)
 505         continue to run together as most cross-tests dependencies tend to
 506         occur within the same directory. If use_real_shards is false, we
 507         put each (non-HTTP/websocket) test into its own shard for maximum
 508         concurrency instead of trying to do any sort of real sharding.
 509
 510         Return:
 511             A list of lists of TestInput objects.
 512         """
 513         # FIXME: when we added http locking, we changed how this works such
 514         # that we always lump all of the HTTP threads into a single shard.
 515         # That will slow down experimental-fully-parallel, but it's unclear
 516         # what the best alternative is completely revamping how we track
 517         # when to grab the lock.
 518
 519         test_lists = []
 520         tests_to_http_lock = []
 521         if not use_real_shards:
 522             for test_file in test_files:
 523                 test_input = self._get_test_input_for_file(test_file)
 524                 if self._test_requires_lock(test_file):
 525                     tests_to_http_lock.append(test_input)
 526                 else:
 527                     test_lists.append((".", [test_input]))
 528         else:
 529             tests_by_dir = {}
 530             for test_file in test_files:
 531                 directory = self._get_dir_for_test_file(test_file)
 532                 test_input = self._get_test_input_for_file(test_file)
 533                 if self._test_requires_lock(test_file):
 534                     tests_to_http_lock.append(test_input)
 535                 else:
 536                     tests_by_dir.setdefault(directory, [])
 537                     tests_by_dir[directory].append(test_input)
 538             # Sort by the number of tests in the dir so that the ones with the
 539             # most tests get run first in order to maximize parallelization.
 540             # Number of tests is a good enough, but not perfect, approximation
 541             # of how long that set of tests will take to run. We can't just use
 542             # a PriorityQueue until we move to Python 2.6.
 543             for directory in tests_by_dir:
 544                 test_list = tests_by_dir[directory]
 545                 # Keep the tests in alphabetical order.
 546                 # FIXME: Remove once tests are fixed so they can be run in any
 547                 # order.
 548                 test_list.reverse()
 549                 test_list_tuple = (directory, test_list)
 550                 test_lists.append(test_list_tuple)
 551             test_lists.sort(lambda a, b: cmp(len(b[1]), len(a[1])))
 552
 553         # Put the http tests first. There are only a couple hundred of them,
 554         # but each http test takes a very long time to run, so sorting by the
 555         # number of tests doesn't accurately capture how long they take to run.
 556         if tests_to_http_lock:
 557             tests_to_http_lock.reverse()
 558             test_lists.insert(0, ("tests_to_http_lock", tests_to_http_lock))
 559
 560         return test_lists
 561
 562     def _contains_tests(self, subdir):
 563         for test_file in self._test_files:
 564             if test_file.find(subdir) >= 0:
 565                 return True
 566         return False
 567
 568     def _instantiate_dump_render_tree_threads(self, test_files,
 569                                               result_summary):
 570         """Instantitates and starts the TestShellThread(s).
 571
 572         Return:
 573           The list of threads.
 574         """
 575         num_workers = self._num_workers()
 576         test_lists = self._shard_tests(test_files,
 577             num_workers > 1 and not self._options.experimental_fully_parallel)
 578         filename_queue = Queue.Queue()
 579         for item in test_lists:
 580             filename_queue.put(item)
 581
 582         # Instantiate TestShellThreads and start them.
 583         threads = []
 584         for worker_number in xrange(num_workers):
 585             thread = dump_render_tree_thread.TestShellThread(self._port,
 586                 self._options, worker_number,
 587                 filename_queue, self._result_queue)
 588             if num_workers > 1:
 589                 thread.start()
 590             else:
 591                 thread.run_in_main_thread(self, result_summary)
 592             threads.append(thread)
 593
 594         return threads
 595
 596     def _num_workers(self):
 597         return int(self._options.child_processes)
 598
 599     def _run_tests(self, file_list, result_summary):
 600         """Runs the tests in the file_list.
 601
 602         Return: A tuple (keyboard_interrupted, thread_timings, test_timings,
 603             individual_test_timings)
 604             keyboard_interrupted is whether someone typed Ctrl^C
 605             thread_timings is a list of dicts with the total runtime
 606               of each thread with 'name', 'num_tests', 'total_time' properties
 607             test_timings is a list of timings for each sharded subdirectory
 608               of the form [time, directory_name, num_tests]
 609             individual_test_timings is a list of run times for each test
 610               in the form {filename:filename, test_run_time:test_run_time}
 611             result_summary: summary object to populate with the results
 612         """
 613         plural = ""
 614         if self._num_workers() > 1:
 615             plural = "s"
 616         self._printer.print_update('Starting %s%s ...' %
 617                                    (self._port.driver_name(), plural))
 618         threads = self._instantiate_dump_render_tree_threads(file_list,
 619                                                              result_summary)
 620         self._printer.print_update("Starting testing ...")
 621
 622         keyboard_interrupted = self._wait_for_threads_to_finish(threads,
 623                                                                 result_summary)
 624         (thread_timings, test_timings, individual_test_timings) = \
 625             self._collect_timing_info(threads)
 626
 627         return (keyboard_interrupted, thread_timings, test_timings,
 628                 individual_test_timings)
 629
 630     def _wait_for_threads_to_finish(self, threads, result_summary):
 631         keyboard_interrupted = False
 632         try:
 633             # Loop through all the threads waiting for them to finish.
 634             some_thread_is_alive = True
 635             while some_thread_is_alive:
 636                 some_thread_is_alive = False
 637                 t = time.time()
 638                 for thread in threads:
 639                     exception_info = thread.exception_info()
 640                     if exception_info is not None:
 641                         # Re-raise the thread's exception here to make it
 642                         # clear that testing was aborted. Otherwise,
 643                         # the tests that did not run would be assumed
 644                         # to have passed.
 645                         raise exception_info[0], exception_info[1], exception_info[2]
 646
 647                     if thread.isAlive():
 648                         some_thread_is_alive = True
 649                         next_timeout = thread.next_timeout()
 650                         if (next_timeout and t > next_timeout):
 651                             message_broker.log_wedged_thread(thread.id())
 652                             thread.clear_next_timeout()
 653
 654                 self.update_summary(result_summary)
 655
 656                 if some_thread_is_alive:
 657                     time.sleep(0.01)
 658
 659         except KeyboardInterrupt:
 660             keyboard_interrupted = True
 661             for thread in threads:
 662                 thread.cancel()
 663
 664         return keyboard_interrupted
 665
 666     def _collect_timing_info(self, threads):
 667         test_timings = {}
 668         individual_test_timings = []
 669         thread_timings = []
 670
 671         for thread in threads:
 672             thread_timings.append({'name': thread.getName(),
 673                                    'num_tests': thread.get_num_tests(),
 674                                    'total_time': thread.get_total_time()})
 675             test_timings.update(thread.get_test_group_timing_stats())
 676             individual_test_timings.extend(thread.get_test_results())
 677
 678         return (thread_timings, test_timings, individual_test_timings)
 679
 680     def needs_http(self):
 681         """Returns whether the test runner needs an HTTP server."""
 682         return self._contains_tests(self.HTTP_SUBDIR)
 683
 684     def needs_websocket(self):
 685         """Returns whether the test runner needs a WEBSOCKET server."""
 686         return self._contains_tests(self.WEBSOCKET_SUBDIR)
 687
 688     def set_up_run(self):
 689         """Configures the system to be ready to run tests.
 690
 691         Returns a ResultSummary object if we should continue to run tests,
 692         or None if we should abort.
 693
 694         """
 695         # This must be started before we check the system dependencies,
 696         # since the helper may do things to make the setup correct.
 697         self._printer.print_update("Starting helper ...")
 698         self._port.start_helper()
 699
 700         # Check that the system dependencies (themes, fonts, ...) are correct.
 701         if not self._options.nocheck_sys_deps:
 702             self._printer.print_update("Checking system dependencies ...")
 703             if not self._port.check_sys_deps(self.needs_http()):
 704                 self._port.stop_helper()
 705                 return None
 706
 707         if self._options.clobber_old_results:
 708             self._clobber_old_results()
 709
 710         # Create the output directory if it doesn't already exist.
 711         self._port.maybe_make_directory(self._options.results_directory)
 712
 713         self._port.setup_test_run()
 714
 715         self._printer.print_update("Preparing tests ...")
 716         result_summary = self.prepare_lists_and_print_output()
 717         if not result_summary:
 718             return None
 719
 720         return result_summary
 721
 722     def run(self, result_summary):
 723         """Run all our tests on all our test files.
 724
 725         For each test file, we run each test type. If there are any failures,
 726         we collect them for reporting.
 727
 728         Args:
 729           result_summary: a summary object tracking the test results.
 730
 731         Return:
 732           The number of unexpected results (0 == success)
 733         """
 734         # gather_test_files() must have been called first to initialize us.
 735         # If we didn't find any files to test, we've errored out already in
 736         # prepare_lists_and_print_output().
 737         assert(len(self._test_files))
 738
 739         start_time = time.time()
 740
 741         keyboard_interrupted, thread_timings, test_timings, \
 742             individual_test_timings = (
 743             self._run_tests(self._test_files_list, result_summary))
 744
 745         # We exclude the crashes from the list of results to retry, because
 746         # we want to treat even a potentially flaky crash as an error.
 747         failures = self._get_failures(result_summary, include_crashes=False)
 748         retry_summary = result_summary
 749         while (len(failures) and self._options.retry_failures and
 750             not self._retrying and not keyboard_interrupted):
 751             _log.info('')
 752             _log.info("Retrying %d unexpected failure(s) ..." % len(failures))
 753             _log.info('')
 754             self._retrying = True
 755             retry_summary = ResultSummary(self._expectations, failures.keys())
 756             # Note that we intentionally ignore the return value here.
 757             self._run_tests(failures.keys(), retry_summary)
 758             failures = self._get_failures(retry_summary, include_crashes=True)
 759
 760         end_time = time.time()
 761
 762         self._print_timing_statistics(end_time - start_time,
 763                                       thread_timings, test_timings,
 764                                       individual_test_timings,
 765                                       result_summary)
 766
 767         self._print_result_summary(result_summary)
 768
 769         sys.stdout.flush()
 770         sys.stderr.flush()
 771
 772         self._printer.print_one_line_summary(result_summary.total,
 773                                              result_summary.expected,
 774                                              result_summary.unexpected)
 775
 776         unexpected_results = summarize_unexpected_results(self._port,
 777             self._expectations, result_summary, retry_summary)
 778         self._printer.print_unexpected_results(unexpected_results)
 779
 780         if self._options.record_results:
 781             # Write the same data to log files and upload generated JSON files
 782             # to appengine server.
 783             self._upload_json_files(unexpected_results, result_summary,
 784                                     individual_test_timings)
 785
 786         # Write the summary to disk (results.html) and display it if requested.
 787         wrote_results = self._write_results_html_file(result_summary)
 788         if self._options.show_results and wrote_results:
 789             self._show_results_html_file()
 790
 791         # Now that we've completed all the processing we can, we re-raise
 792         # a KeyboardInterrupt if necessary so the caller can handle it.
 793         if keyboard_interrupted:
 794             raise KeyboardInterrupt
 795
 796         # Ignore flaky failures and unexpected passes so we don't turn the
 797         # bot red for those.
 798         return unexpected_results['num_regressions']
 799
 800     def clean_up_run(self):
 801         """Restores the system after we're done running tests."""
 802
 803         _log.debug("flushing stdout")
 804         sys.stdout.flush()
 805         _log.debug("flushing stderr")
 806         sys.stderr.flush()
 807         _log.debug("stopping helper")
 808         self._port.stop_helper()
 809
 810     def update_summary(self, result_summary):
 811         """Update the summary and print results with any completed tests."""
 812         while True:
 813             try:
 814                 result = test_results.TestResult.loads(self._result_queue.get_nowait())
 815             except Queue.Empty:
 816                 return
 817
 818             expected = self._expectations.matches_an_expected_result(
 819                 result.filename, result.type, self._options.pixel_tests)
 820             result_summary.add(result, expected)
 821             exp_str = self._expectations.get_expectations_string(
 822                 result.filename)
 823             got_str = self._expectations.expectation_to_string(result.type)
 824             self._printer.print_test_result(result, expected, exp_str, got_str)
 825             self._printer.print_progress(result_summary, self._retrying,
 826                                          self._test_files_list)
 827
 828     def _clobber_old_results(self):
 829         # Just clobber the actual test results directories since the other
 830         # files in the results directory are explicitly used for cross-run
 831         # tracking.
 832         self._printer.print_update("Clobbering old results in %s" %
 833                                    self._options.results_directory)
 834         layout_tests_dir = self._port.layout_tests_dir()
 835         possible_dirs = self._port.test_dirs()
 836         for dirname in possible_dirs:
 837             if os.path.isdir(os.path.join(layout_tests_dir, dirname)):
 838                 shutil.rmtree(os.path.join(self._options.results_directory,
 839                                            dirname),
 840                               ignore_errors=True)
 841
 842     def _get_failures(self, result_summary, include_crashes):
 843         """Filters a dict of results and returns only the failures.
 844
 845         Args:
 846           result_summary: the results of the test run
 847           include_crashes: whether crashes are included in the output.
 848             We use False when finding the list of failures to retry
 849             to see if the results were flaky. Although the crashes may also be
 850             flaky, we treat them as if they aren't so that they're not ignored.
 851         Returns:
 852           a dict of files -> results
 853         """
 854         failed_results = {}
 855         for test, result in result_summary.unexpected_results.iteritems():
 856             if (result == test_expectations.PASS or
 857                 result == test_expectations.CRASH and not include_crashes):
 858                 continue
 859             failed_results[test] = result
 860
 861         return failed_results
 862
 863     def _upload_json_files(self, unexpected_results, result_summary,
 864                         individual_test_timings):
 865         """Writes the results of the test run as JSON files into the results
 866         dir and upload the files to the appengine server.
 867
 868         There are three different files written into the results dir:
 869           unexpected_results.json: A short list of any unexpected results.
 870             This is used by the buildbots to display results.
 871           expectations.json: This is used by the flakiness dashboard.
 872           results.json: A full list of the results - used by the flakiness
 873             dashboard and the aggregate results dashboard.
 874
 875         Args:
 876           unexpected_results: dict of unexpected results
 877           result_summary: full summary object
 878           individual_test_timings: list of test times (used by the flakiness
 879             dashboard).
 880         """
 881         results_directory = self._options.results_directory
 882         _log.debug("Writing JSON files in %s." % results_directory)
 883         unexpected_json_path = os.path.join(results_directory, "unexpected_results.json")
 884         with codecs.open(unexpected_json_path, "w", "utf-8") as file:
 885             simplejson.dump(unexpected_results, file, sort_keys=True, indent=2)
 886
 887         # Write a json file of the test_expectations.txt file for the layout
 888         # tests dashboard.
 889         expectations_path = os.path.join(results_directory, "expectations.json")
 890         expectations_json = \
 891             self._expectations.get_expectations_json_for_all_platforms()
 892         with codecs.open(expectations_path, "w", "utf-8") as file:
 893             file.write(u"ADD_EXPECTATIONS(%s);" % expectations_json)
 894
 895         generator = json_layout_results_generator.JSONLayoutResultsGenerator(
 896             self._port, self._options.builder_name, self._options.build_name,
 897             self._options.build_number, self._options.results_directory,
 898             BUILDER_BASE_URL, individual_test_timings,
 899             self._expectations, result_summary, self._test_files_list,
 900             not self._options.upload_full_results,
 901             self._options.test_results_server,
 902             "layout-tests",
 903             self._options.master_name)
 904
 905         _log.debug("Finished writing JSON files.")
 906
 907         json_files = ["expectations.json"]
 908         if self._options.upload_full_results:
 909             json_files.append("results.json")
 910         else:
 911             json_files.append("incremental_results.json")
 912
 913         generator.upload_json_files(json_files)
 914
 915     def _print_config(self):
 916         """Prints the configuration for the test run."""
 917         p = self._printer
 918         p.print_config("Using port '%s'" % self._port.name())
 919         p.print_config("Placing test results in %s" %
 920                        self._options.results_directory)
 921         if self._options.new_baseline:
 922             p.print_config("Placing new baselines in %s" %
 923                            self._port.baseline_path())
 924         p.print_config("Using %s build" % self._options.configuration)
 925         if self._options.pixel_tests:
 926             p.print_config("Pixel tests enabled")
 927         else:
 928             p.print_config("Pixel tests disabled")
 929
 930         p.print_config("Regular timeout: %s, slow test timeout: %s" %
 931                        (self._options.time_out_ms,
 932                         self._options.slow_time_out_ms))
 933
 934         if self._num_workers() == 1:
 935             p.print_config("Running one %s" % self._port.driver_name())
 936         else:
 937             p.print_config("Running %s %ss in parallel" %
 938                            (self._options.child_processes,
 939                             self._port.driver_name()))
 940         p.print_config("Worker model: %s" % self._options.worker_model)
 941         p.print_config("")
 942
 943     def _print_expected_results_of_type(self, result_summary,
 944                                         result_type, result_type_str):
 945         """Print the number of the tests in a given result class.
 946
 947         Args:
 948           result_summary - the object containing all the results to report on
 949           result_type - the particular result type to report in the summary.
 950           result_type_str - a string description of the result_type.
 951         """
 952         tests = self._expectations.get_tests_with_result_type(result_type)
 953         now = result_summary.tests_by_timeline[test_expectations.NOW]
 954         wontfix = result_summary.tests_by_timeline[test_expectations.WONTFIX]
 955
 956         # We use a fancy format string in order to print the data out in a
 957         # nicely-aligned table.
 958         fmtstr = ("Expect: %%5d %%-8s (%%%dd now, %%%dd wontfix)"
 959                   % (self._num_digits(now), self._num_digits(wontfix)))
 960         self._printer.print_expected(fmtstr %
 961             (len(tests), result_type_str, len(tests & now), len(tests & wontfix)))
 962
 963     def _num_digits(self, num):
 964         """Returns the number of digits needed to represent the length of a
 965         sequence."""
 966         ndigits = 1
 967         if len(num):
 968             ndigits = int(math.log10(len(num))) + 1
 969         return ndigits
 970
 971     def _print_timing_statistics(self, total_time, thread_timings,
 972                                directory_test_timings, individual_test_timings,
 973                                result_summary):
 974         """Record timing-specific information for the test run.
 975
 976         Args:
 977           total_time: total elapsed time (in seconds) for the test run
 978           thread_timings: wall clock time each thread ran for
 979           directory_test_timings: timing by directory
 980           individual_test_timings: timing by file
 981           result_summary: summary object for the test run
 982         """
 983         self._printer.print_timing("Test timing:")
 984         self._printer.print_timing("  %6.2f total testing time" % total_time)
 985         self._printer.print_timing("")
 986         self._printer.print_timing("Thread timing:")
 987         cuml_time = 0
 988         for t in thread_timings:
 989             self._printer.print_timing("    %10s: %5d tests, %6.2f secs" %
 990                   (t['name'], t['num_tests'], t['total_time']))
 991             cuml_time += t['total_time']
 992         self._printer.print_timing("   %6.2f cumulative, %6.2f optimal" %
 993               (cuml_time, cuml_time / int(self._options.child_processes)))
 994         self._printer.print_timing("")
 995
 996         self._print_aggregate_test_statistics(individual_test_timings)
 997         self._print_individual_test_times(individual_test_timings,
 998                                           result_summary)
 999         self._print_directory_timings(directory_test_timings)
1000
1001     def _print_aggregate_test_statistics(self, individual_test_timings):
1002         """Prints aggregate statistics (e.g. median, mean, etc.) for all tests.
1003         Args:
1004           individual_test_timings: List of dump_render_tree_thread.TestStats
1005               for all tests.
1006         """
1007         test_types = []  # Unit tests don't actually produce any timings.
1008         if individual_test_timings:
1009             test_types = individual_test_timings[0].time_for_diffs.keys()
1010         times_for_dump_render_tree = []
1011         times_for_diff_processing = []
1012         times_per_test_type = {}
1013         for test_type in test_types:
1014             times_per_test_type[test_type] = []
1015
1016         for test_stats in individual_test_timings:
1017             times_for_dump_render_tree.append(test_stats.test_run_time)
1018             times_for_diff_processing.append(
1019                 test_stats.total_time_for_all_diffs)
1020             time_for_diffs = test_stats.time_for_diffs
1021             for test_type in test_types:
1022                 times_per_test_type[test_type].append(
1023                     time_for_diffs[test_type])
1024
1025         self._print_statistics_for_test_timings(
1026             "PER TEST TIME IN TESTSHELL (seconds):",
1027             times_for_dump_render_tree)
1028         self._print_statistics_for_test_timings(
1029             "PER TEST DIFF PROCESSING TIMES (seconds):",
1030             times_for_diff_processing)
1031         for test_type in test_types:
1032             self._print_statistics_for_test_timings(
1033                 "PER TEST TIMES BY TEST TYPE: %s" % test_type,
1034                 times_per_test_type[test_type])
1035
1036     def _print_individual_test_times(self, individual_test_timings,
1037                                   result_summary):
1038         """Prints the run times for slow, timeout and crash tests.
1039         Args:
1040           individual_test_timings: List of dump_render_tree_thread.TestStats
1041               for all tests.
1042           result_summary: summary object for test run
1043         """
1044         # Reverse-sort by the time spent in DumpRenderTree.
1045         individual_test_timings.sort(lambda a, b:
1046             cmp(b.test_run_time, a.test_run_time))
1047
1048         num_printed = 0
1049         slow_tests = []
1050         timeout_or_crash_tests = []
1051         unexpected_slow_tests = []
1052         for test_tuple in individual_test_timings:
1053             filename = test_tuple.filename
1054             is_timeout_crash_or_slow = False
1055             if self._test_is_slow(filename):
1056                 is_timeout_crash_or_slow = True
1057                 slow_tests.append(test_tuple)
1058
1059             if filename in result_summary.failures:
1060                 result = result_summary.results[filename].type
1061                 if (result == test_expectations.TIMEOUT or
1062                     result == test_expectations.CRASH):
1063                     is_timeout_crash_or_slow = True
1064                     timeout_or_crash_tests.append(test_tuple)
1065
1066             if (not is_timeout_crash_or_slow and
1067                 num_printed < printing.NUM_SLOW_TESTS_TO_LOG):
1068                 num_printed = num_printed + 1
1069                 unexpected_slow_tests.append(test_tuple)
1070
1071         self._printer.print_timing("")
1072         self._print_test_list_timing("%s slowest tests that are not "
1073             "marked as SLOW and did not timeout/crash:" %
1074             printing.NUM_SLOW_TESTS_TO_LOG, unexpected_slow_tests)
1075         self._printer.print_timing("")
1076         self._print_test_list_timing("Tests marked as SLOW:", slow_tests)
1077         self._printer.print_timing("")
1078         self._print_test_list_timing("Tests that timed out or crashed:",
1079                                      timeout_or_crash_tests)
1080         self._printer.print_timing("")
1081
1082     def _print_test_list_timing(self, title, test_list):
1083         """Print timing info for each test.
1084
1085         Args:
1086           title: section heading
1087           test_list: tests that fall in this section
1088         """
1089         if self._printer.disabled('slowest'):
1090             return
1091
1092         self._printer.print_timing(title)
1093         for test_tuple in test_list:
1094             filename = test_tuple.filename[len(
1095                 self._port.layout_tests_dir()) + 1:]
1096             filename = filename.replace('\\', '/')
1097             test_run_time = round(test_tuple.test_run_time, 1)
1098             self._printer.print_timing("  %s took %s seconds" %
1099                                        (filename, test_run_time))
1100
1101     def _print_directory_timings(self, directory_test_timings):
1102         """Print timing info by directory for any directories that
1103         take > 10 seconds to run.
1104
1105         Args:
1106           directory_test_timing: time info for each directory
1107         """
1108         timings = []
1109         for directory in directory_test_timings:
1110             num_tests, time_for_directory = directory_test_timings[directory]
1111             timings.append((round(time_for_directory, 1), directory,
1112                             num_tests))
1113         timings.sort()
1114
1115         self._printer.print_timing("Time to process slowest subdirectories:")
1116         min_seconds_to_print = 10
1117         for timing in timings:
1118             if timing[0] > min_seconds_to_print:
1119                 self._printer.print_timing(
1120                     "  %s took %s seconds to run %s tests." % (timing[1],
1121                     timing[0], timing[2]))
1122         self._printer.print_timing("")
1123
1124     def _print_statistics_for_test_timings(self, title, timings):
1125         """Prints the median, mean and standard deviation of the values in
1126         timings.
1127
1128         Args:
1129           title: Title for these timings.
1130           timings: A list of floats representing times.
1131         """
1132         self._printer.print_timing(title)
1133         timings.sort()
1134
1135         num_tests = len(timings)
1136         if not num_tests:
1137             return
1138         percentile90 = timings[int(.9 * num_tests)]
1139         percentile99 = timings[int(.99 * num_tests)]
1140
1141         if num_tests % 2 == 1:
1142             median = timings[((num_tests - 1) / 2) - 1]
1143         else:
1144             lower = timings[num_tests / 2 - 1]
1145             upper = timings[num_tests / 2]
1146             median = (float(lower + upper)) / 2
1147
1148         mean = sum(timings) / num_tests
1149
1150         for time in timings:
1151             sum_of_deviations = math.pow(time - mean, 2)
1152
1153         std_deviation = math.sqrt(sum_of_deviations / num_tests)
1154         self._printer.print_timing("  Median:          %6.3f" % median)
1155         self._printer.print_timing("  Mean:            %6.3f" % mean)
1156         self._printer.print_timing("  90th percentile: %6.3f" % percentile90)
1157         self._printer.print_timing("  99th percentile: %6.3f" % percentile99)
1158         self._printer.print_timing("  Standard dev:    %6.3f" % std_deviation)
1159         self._printer.print_timing("")
1160
1161     def _print_result_summary(self, result_summary):
1162         """Print a short summary about how many tests passed.
1163
1164         Args:
1165           result_summary: information to log
1166         """
1167         failed = len(result_summary.failures)
1168         skipped = len(
1169             result_summary.tests_by_expectation[test_expectations.SKIP])
1170         total = result_summary.total
1171         passed = total - failed - skipped
1172         pct_passed = 0.0
1173         if total > 0:
1174             pct_passed = float(passed) * 100 / total
1175
1176         self._printer.print_actual("")
1177         self._printer.print_actual("=> Results: %d/%d tests passed (%.1f%%)" %
1178                      (passed, total, pct_passed))
1179         self._printer.print_actual("")
1180         self._print_result_summary_entry(result_summary,
1181             test_expectations.NOW, "Tests to be fixed")
1182
1183         self._printer.print_actual("")
1184         self._print_result_summary_entry(result_summary,
1185             test_expectations.WONTFIX,
1186             "Tests that will only be fixed if they crash (WONTFIX)")
1187         self._printer.print_actual("")
1188
1189     def _print_result_summary_entry(self, result_summary, timeline,
1190                                     heading):
1191         """Print a summary block of results for a particular timeline of test.
1192
1193         Args:
1194           result_summary: summary to print results for
1195           timeline: the timeline to print results for (NOT, WONTFIX, etc.)
1196           heading: a textual description of the timeline
1197         """
1198         total = len(result_summary.tests_by_timeline[timeline])
1199         not_passing = (total -
1200            len(result_summary.tests_by_expectation[test_expectations.PASS] &
1201                result_summary.tests_by_timeline[timeline]))
1202         self._printer.print_actual("=> %s (%d):" % (heading, not_passing))
1203
1204         for result in TestExpectationsFile.EXPECTATION_ORDER:
1205             if result == test_expectations.PASS:
1206                 continue
1207             results = (result_summary.tests_by_expectation[result] &
1208                        result_summary.tests_by_timeline[timeline])
1209             desc = TestExpectationsFile.EXPECTATION_DESCRIPTIONS[result]
1210             if not_passing and len(results):
1211                 pct = len(results) * 100.0 / not_passing
1212                 self._printer.print_actual("  %5d %-24s (%4.1f%%)" %
1213                     (len(results), desc[len(results) != 1], pct))
1214
1215     def _results_html(self, test_files, failures, title="Test Failures", override_time=None):
1216         """
1217         test_files = a list of file paths
1218         failures = dictionary mapping test paths to failure objects
1219         title = title printed at top of test
1220         override_time = current time (used by unit tests)
1221         """
1222         page = """<html>
1223   <head>
1224     <title>Layout Test Results (%(time)s)</title>
1225   </head>
1226   <body>
1227     <h2>%(title)s (%(time)s)</h2>
1228         """ % {'title': title, 'time': override_time or time.asctime()}
1229
1230         for test_file in sorted(test_files):
1231             test_name = self._port.relative_test_filename(test_file)
1232             test_url = self._port.filename_to_uri(test_file)
1233             page += u"<p><a href='%s'>%s</a><br />\n" % (test_url, test_name)
1234             test_failures = failures.get(test_file, [])
1235             for failure in test_failures:
1236                 page += (u"&nbsp;&nbsp;%s<br/>" %
1237                          failure.result_html_output(test_name))
1238             page += "</p>\n"
1239         page += "</body></html>\n"
1240         return page
1241
1242     def _write_results_html_file(self, result_summary):
1243         """Write results.html which is a summary of tests that failed.
1244
1245         Args:
1246           result_summary: a summary of the results :)
1247
1248         Returns:
1249           True if any results were written (since expected failures may be
1250           omitted)
1251         """
1252         # test failures
1253         if self._options.full_results_html:
1254             results_title = "Test Failures"
1255             test_files = result_summary.failures.keys()
1256         else:
1257             results_title = "Unexpected Test Failures"
1258             unexpected_failures = self._get_failures(result_summary,
1259                 include_crashes=True)
1260             test_files = unexpected_failures.keys()
1261         if not len(test_files):
1262             return False
1263
1264         out_filename = os.path.join(self._options.results_directory,
1265                                     "results.html")
1266         with codecs.open(out_filename, "w", "utf-8") as results_file:
1267             html = self._results_html(test_files, result_summary.failures, results_title)
1268             results_file.write(html)
1269
1270         return True
1271
1272     def _show_results_html_file(self):
1273         """Shows the results.html page."""
1274         results_filename = os.path.join(self._options.results_directory,
1275                                         "results.html")
1276         self._port.show_results_html_file(results_filename)
1277
1278
1279 def read_test_files(files):
1280     tests = []
1281     for file in files:
1282         try:
1283             with codecs.open(file, 'r', 'utf-8') as file_contents:
1284                 # FIXME: This could be cleaner using a list comprehension.
1285                 for line in file_contents:
1286                     line = test_expectations.strip_comments(line)
1287                     if line:
1288                         tests.append(line)
1289         except IOError, e:
1290             if e.errno == errno.ENOENT:
1291                 _log.critical('')
1292                 _log.critical('--test-list file "%s" not found' % file)
1293             raise
1294     return tests
1295
1296
1297 def run(port, options, args, regular_output=sys.stderr,
1298         buildbot_output=sys.stdout):
1299     """Run the tests.
1300
1301     Args:
1302       port: Port object for port-specific behavior
1303       options: a dictionary of command line options
1304       args: a list of sub directories or files to test
1305       regular_output: a stream-like object that we can send logging/debug
1306           output to
1307       buildbot_output: a stream-like object that we can write all output that
1308           is intended to be parsed by the buildbot to
1309     Returns:
1310       the number of unexpected results that occurred, or -1 if there is an
1311           error.
1312
1313     """
1314     _set_up_derived_options(port, options)
1315
1316     printer = printing.Printer(port, options, regular_output, buildbot_output,
1317         int(options.child_processes), options.experimental_fully_parallel)
1318     if options.help_printing:
1319         printer.help_printing()
1320         printer.cleanup()
1321         return 0
1322
1323     last_unexpected_results = _gather_unexpected_results(options)
1324     if options.print_last_failures:
1325         printer.write("\n".join(last_unexpected_results) + "\n")
1326         printer.cleanup()
1327         return 0
1328
1329     # We wrap any parts of the run that are slow or likely to raise exceptions
1330     # in a try/finally to ensure that we clean up the logging configuration.
1331     num_unexpected_results = -1
1332     try:
1333         test_runner = TestRunner(port, options, printer)
1334         test_runner._print_config()
1335
1336         printer.print_update("Collecting tests ...")
1337         try:
1338             test_runner.collect_tests(args, last_unexpected_results)
1339         except IOError, e:
1340             if e.errno == errno.ENOENT:
1341                 return -1
1342             raise
1343
1344         printer.print_update("Parsing expectations ...")
1345         if options.lint_test_files:
1346             return test_runner.lint()
1347         test_runner.parse_expectations(port.test_platform_name(),
1348                                        options.configuration == 'Debug')
1349
1350         printer.print_update("Checking build ...")
1351         if not port.check_build(test_runner.needs_http()):
1352             _log.error("Build check failed")
1353             return -1
1354
1355         result_summary = test_runner.set_up_run()
1356         if result_summary:
1357             num_unexpected_results = test_runner.run(result_summary)
1358             test_runner.clean_up_run()
1359             _log.debug("Testing completed, Exit status: %d" %
1360                        num_unexpected_results)
1361     finally:
1362         printer.cleanup()
1363
1364     return num_unexpected_results
1365
1366
1367 def _set_up_derived_options(port_obj, options):
1368     """Sets the options values that depend on other options values."""
1369
1370     if options.worker_model == 'inline':
1371         if options.child_processes and int(options.child_processes) > 1:
1372             _log.warning("--worker-model=inline overrides --child-processes")
1373         options.child_processes = "1"
1374     if not options.child_processes:
1375         options.child_processes = os.environ.get("WEBKIT_TEST_CHILD_PROCESSES",
1376                                                  str(port_obj.default_child_processes()))
1377
1378     if not options.configuration:
1379         options.configuration = port_obj.default_configuration()
1380
1381     if options.pixel_tests is None:
1382         options.pixel_tests = True
1383
1384     if not options.use_apache:
1385         options.use_apache = sys.platform in ('darwin', 'linux2')
1386
1387     if not os.path.isabs(options.results_directory):
1388         # This normalizes the path to the build dir.
1389         # FIXME: how this happens is not at all obvious; this is a dumb
1390         # interface and should be cleaned up.
1391         options.results_directory = port_obj.results_directory()
1392
1393     if not options.time_out_ms:
1394         if options.configuration == "Debug":
1395             options.time_out_ms = str(2 * TestRunner.DEFAULT_TEST_TIMEOUT_MS)
1396         else:
1397             options.time_out_ms = str(TestRunner.DEFAULT_TEST_TIMEOUT_MS)
1398
1399     options.slow_time_out_ms = str(5 * int(options.time_out_ms))
1400
1401
1402 def _gather_unexpected_results(options):
1403     """Returns the unexpected results from the previous run, if any."""
1404     last_unexpected_results = []
1405     if options.print_last_failures or options.retest_last_failures:
1406         unexpected_results_filename = os.path.join(
1407         options.results_directory, "unexpected_results.json")
1408         with codecs.open(unexpected_results_filename, "r", "utf-8") as file:
1409             results = simplejson.load(file)
1410         last_unexpected_results = results['tests'].keys()
1411     return last_unexpected_results
1412
1413
1414 def _compat_shim_callback(option, opt_str, value, parser):
1415     print "Ignoring unsupported option: %s" % opt_str
1416
1417
1418 def _compat_shim_option(option_name, **kwargs):
1419     return optparse.make_option(option_name, action="callback",
1420         callback=_compat_shim_callback,
1421         help="Ignored, for old-run-webkit-tests compat only.", **kwargs)
1422
1423
1424 def parse_args(args=None):
1425     """Provides a default set of command line args.
1426
1427     Returns a tuple of options, args from optparse"""
1428
1429     # FIXME: All of these options should be stored closer to the code which
1430     # FIXME: actually uses them. configuration_options should move
1431     # FIXME: to WebKitPort and be shared across all scripts.
1432     configuration_options = [
1433         optparse.make_option("-t", "--target", dest="configuration",
1434                              help="(DEPRECATED)"),
1435         # FIXME: --help should display which configuration is default.
1436         optparse.make_option('--debug', action='store_const', const='Debug',
1437                              dest="configuration",
1438                              help='Set the configuration to Debug'),
1439         optparse.make_option('--release', action='store_const',
1440                              const='Release', dest="configuration",
1441                              help='Set the configuration to Release'),
1442         # old-run-webkit-tests also accepts -c, --configuration CONFIGURATION.
1443     ]
1444
1445     print_options = printing.print_options()
1446
1447     # FIXME: These options should move onto the ChromiumPort.
1448     chromium_options = [
1449         optparse.make_option("--chromium", action="store_true", default=False,
1450             help="use the Chromium port"),
1451         optparse.make_option("--startup-dialog", action="store_true",
1452             default=False, help="create a dialog on DumpRenderTree startup"),
1453         optparse.make_option("--gp-fault-error-box", action="store_true",
1454             default=False, help="enable Windows GP fault error box"),
1455         optparse.make_option("--multiple-loads",
1456             type="int", help="turn on multiple loads of each test"),
1457         optparse.make_option("--js-flags",
1458             type="string", help="JavaScript flags to pass to tests"),
1459         optparse.make_option("--nocheck-sys-deps", action="store_true",
1460             default=False,
1461             help="Don't check the system dependencies (themes)"),
1462         optparse.make_option("--use-drt", action="store_true",
1463             default=None,
1464             help="Use DumpRenderTree instead of test_shell"),
1465         optparse.make_option("--accelerated-compositing",
1466             action="store_true",
1467             help="Use hardware-accelated compositing for rendering"),
1468         optparse.make_option("--no-accelerated-compositing",
1469             action="store_false",
1470             dest="accelerated_compositing",
1471             help="Don't use hardware-accelerated compositing for rendering"),
1472         optparse.make_option("--accelerated-2d-canvas",
1473             action="store_true",
1474             help="Use hardware-accelerated 2D Canvas calls"),
1475         optparse.make_option("--no-accelerated-2d-canvas",
1476             action="store_false",
1477             dest="accelerated_2d_canvas",
1478             help="Don't use hardware-accelerated 2D Canvas calls"),
1479     ]
1480
1481     # Missing Mac-specific old-run-webkit-tests options:
1482     # FIXME: Need: -g, --guard for guard malloc support on Mac.
1483     # FIXME: Need: -l --leaks    Enable leaks checking.
1484     # FIXME: Need: --sample-on-timeout Run sample on timeout
1485
1486     old_run_webkit_tests_compat = [
1487         # NRWT doesn't generate results by default anyway.
1488         _compat_shim_option("--no-new-test-results"),
1489         # NRWT doesn't sample on timeout yet anyway.
1490         _compat_shim_option("--no-sample-on-timeout"),
1491         # FIXME: NRWT needs to support remote links eventually.
1492         _compat_shim_option("--use-remote-links-to-tests"),
1493         # FIXME: NRWT doesn't need this option as much since failures are
1494         # designed to be cheap.  We eventually plan to add this support.
1495         _compat_shim_option("--exit-after-n-failures", nargs=1, type="int"),
1496     ]
1497
1498     results_options = [
1499         # NEED for bots: --use-remote-links-to-tests Link to test files
1500         # within the SVN repository in the results.
1501         optparse.make_option("-p", "--pixel-tests", action="store_true",
1502             dest="pixel_tests", help="Enable pixel-to-pixel PNG comparisons"),
1503         optparse.make_option("--no-pixel-tests", action="store_false",
1504             dest="pixel_tests", help="Disable pixel-to-pixel PNG comparisons"),
1505         optparse.make_option("--tolerance",
1506             help="Ignore image differences less than this percentage (some "
1507                 "ports may ignore this option)", type="float"),
1508         optparse.make_option("--results-directory",
1509             default="layout-test-results",
1510             help="Output results directory source dir, relative to Debug or "
1511                  "Release"),
1512         optparse.make_option("--new-baseline", action="store_true",
1513             default=False, help="Save all generated results as new baselines "
1514                  "into the platform directory, overwriting whatever's "
1515                  "already there."),
1516         optparse.make_option("--reset-results", action="store_true",
1517             default=False, help="Reset any existing baselines to the "
1518                  "generated results"),
1519         optparse.make_option("--no-show-results", action="store_false",
1520             default=True, dest="show_results",
1521             help="Don't launch a browser with results after the tests "
1522                  "are done"),
1523         # FIXME: We should have a helper function to do this sort of
1524         # deprectated mapping and automatically log, etc.
1525         optparse.make_option("--noshow-results", action="store_false",
1526             dest="show_results",
1527             help="Deprecated, same as --no-show-results."),
1528         optparse.make_option("--no-launch-safari", action="store_false",
1529             dest="show_results",
1530             help="old-run-webkit-tests compat, same as --noshow-results."),
1531         # old-run-webkit-tests:
1532         # --[no-]launch-safari    Launch (or do not launch) Safari to display
1533         #                         test results (default: launch)
1534         optparse.make_option("--full-results-html", action="store_true",
1535             default=False,
1536             help="Show all failures in results.html, rather than only "
1537                  "regressions"),
1538         optparse.make_option("--clobber-old-results", action="store_true",
1539             default=False, help="Clobbers test results from previous runs."),
1540         optparse.make_option("--platform",
1541             help="Override the platform for expected results"),
1542         optparse.make_option("--no-record-results", action="store_false",
1543             default=True, dest="record_results",
1544             help="Don't record the results."),
1545         # old-run-webkit-tests also has HTTP toggle options:
1546         # --[no-]http                     Run (or do not run) http tests
1547         #                                 (default: run)
1548     ]
1549
1550     test_options = [
1551         optparse.make_option("--build", dest="build",
1552             action="store_true", default=True,
1553             help="Check to ensure the DumpRenderTree build is up-to-date "
1554                  "(default)."),
1555         optparse.make_option("--no-build", dest="build",
1556             action="store_false", help="Don't check to see if the "
1557                                        "DumpRenderTree build is up-to-date."),
1558         # old-run-webkit-tests has --valgrind instead of wrapper.
1559         optparse.make_option("--wrapper",
1560             help="wrapper command to insert before invocations of "
1561                  "DumpRenderTree; option is split on whitespace before "
1562                  "running. (Example: --wrapper='valgrind --smc-check=all')"),
1563         # old-run-webkit-tests:
1564         # -i|--ignore-tests               Comma-separated list of directories
1565         #                                 or tests to ignore
1566         optparse.make_option("--test-list", action="append",
1567             help="read list of tests to run from file", metavar="FILE"),
1568         # old-run-webkit-tests uses --skipped==[default|ignore|only]
1569         # instead of --force:
1570         optparse.make_option("--force", action="store_true", default=False,
1571             help="Run all tests, even those marked SKIP in the test list"),
1572         optparse.make_option("--use-apache", action="store_true",
1573             default=False, help="Whether to use apache instead of lighttpd."),
1574         optparse.make_option("--time-out-ms",
1575             help="Set the timeout for each test"),
1576         # old-run-webkit-tests calls --randomize-order --random:
1577         optparse.make_option("--randomize-order", action="store_true",
1578             default=False, help=("Run tests in random order (useful "
1579                                 "for tracking down corruption)")),
1580         optparse.make_option("--run-chunk",
1581             help=("Run a specified chunk (n:l), the nth of len l, "
1582                  "of the layout tests")),
1583         optparse.make_option("--run-part", help=("Run a specified part (n:m), "
1584                   "the nth of m parts, of the layout tests")),
1585         # old-run-webkit-tests calls --batch-size: --nthly n
1586         #   Restart DumpRenderTree every n tests (default: 1000)
1587         optparse.make_option("--batch-size",
1588             help=("Run a the tests in batches (n), after every n tests, "
1589                   "DumpRenderTree is relaunched."), type="int", default=0),
1590         # old-run-webkit-tests calls --run-singly: -1|--singly
1591         # Isolate each test case run (implies --nthly 1 --verbose)
1592         optparse.make_option("--run-singly", action="store_true",
1593             default=False, help="run a separate DumpRenderTree for each test"),
1594         optparse.make_option("--child-processes",
1595             help="Number of DumpRenderTrees to run in parallel."),
1596         # FIXME: Display default number of child processes that will run.
1597         optparse.make_option("--worker-model", action="store",
1598                              default="threads",
1599                              help="controls worker model. Valid values are "
1600                              "'inline' and 'threads' (default)."),
1601         optparse.make_option("--experimental-fully-parallel",
1602             action="store_true", default=False,
1603             help="run all tests in parallel"),
1604         # FIXME: Need --exit-after-n-failures N
1605         #      Exit after the first N failures instead of running all tests
1606         # FIXME: Need --exit-after-n-crashes N
1607         #      Exit after the first N crashes instead of running all tests
1608         # FIXME: consider: --iterations n
1609         #      Number of times to run the set of tests (e.g. ABCABCABC)
1610         optparse.make_option("--print-last-failures", action="store_true",
1611             default=False, help="Print the tests in the last run that "
1612             "had unexpected failures (or passes)."),
1613         optparse.make_option("--retest-last-failures", action="store_true",
1614             default=False, help="re-test the tests in the last run that "
1615             "had unexpected failures (or passes)."),
1616         optparse.make_option("--retry-failures", action="store_true",
1617             default=True,
1618             help="Re-try any tests that produce unexpected results (default)"),
1619         optparse.make_option("--no-retry-failures", action="store_false",
1620             dest="retry_failures",
1621             help="Don't re-try any tests that produce unexpected results."),
1622     ]
1623
1624     misc_options = [
1625         optparse.make_option("--lint-test-files", action="store_true",
1626         default=False, help=("Makes sure the test files parse for all "
1627                             "configurations. Does not run any tests.")),
1628     ]
1629
1630     # FIXME: Move these into json_results_generator.py
1631     results_json_options = [
1632         optparse.make_option("--master-name", help="The name of the buildbot master."),
1633         optparse.make_option("--builder-name", default="DUMMY_BUILDER_NAME",
1634             help=("The name of the builder shown on the waterfall running "
1635                   "this script e.g. WebKit.")),
1636         optparse.make_option("--build-name", default="DUMMY_BUILD_NAME",
1637             help=("The name of the builder used in its path, e.g. "
1638                   "webkit-rel.")),
1639         optparse.make_option("--build-number", default="DUMMY_BUILD_NUMBER",
1640             help=("The build number of the builder running this script.")),
1641         optparse.make_option("--test-results-server", default="",
1642             help=("If specified, upload results json files to this appengine "
1643                   "server.")),
1644         optparse.make_option("--upload-full-results",
1645             action="store_true",
1646             default=False,
1647             help="If true, upload full json results to server."),
1648     ]
1649
1650     option_list = (configuration_options + print_options +
1651                    chromium_options + results_options + test_options +
1652                    misc_options + results_json_options +
1653                    old_run_webkit_tests_compat)
1654     option_parser = optparse.OptionParser(option_list=option_list)
1655
1656     options, args = option_parser.parse_args(args)
1657
1658     if options.worker_model not in ('inline', 'threads'):
1659         option_parser.error("unsupported value for --worker-model: %s" %
1660                             options.worker_model)
1661
1662     return options, args
1663
1664
1665 def main():
1666     options, args = parse_args()
1667     port_obj = port.get(options.platform, options)
1668     return run(port_obj, options, args)
1669
1670
1671 if '__main__' == __name__:
1672     try:
1673         sys.exit(main())
1674     except KeyboardInterrupt:
1675         # this mirrors what the shell normally does
1676         sys.exit(signal.SIGINT + 128)