WebKitTools/Scripts/webkitpy/layout_tests/run_webkit_tests.py

   1 #!/usr/bin/env python
   2 # Copyright (C) 2010 Google Inc. All rights reserved.
   3 # Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Szeged
   4 #
   5 # Redistribution and use in source and binary forms, with or without
   6 # modification, are permitted provided that the following conditions are
   7 # met:
   8 #
   9 #     * Redistributions of source code must retain the above copyright
  10 # notice, this list of conditions and the following disclaimer.
  11 #     * Redistributions in binary form must reproduce the above
  12 # copyright notice, this list of conditions and the following disclaimer
  13 # in the documentation and/or other materials provided with the
  14 # distribution.
  15 #     * Neither the name of Google Inc. nor the names of its
  16 # contributors may be used to endorse or promote products derived from
  17 # this software without specific prior written permission.
  18 #
  19 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  20 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  21 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  22 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  23 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  24 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  25 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  26 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  27 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  28 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  29 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  30
  31 """Run layout tests.
  32
  33 This is a port of the existing webkit test script run-webkit-tests.
  34
  35 The TestRunner class runs a series of tests (TestType interface) against a set
  36 of test files.  If a test file fails a TestType, it returns a list TestFailure
  37 objects to the TestRunner.  The TestRunner then aggregates the TestFailures to
  38 create a final report.
  39
  40 This script reads several files, if they exist in the test_lists subdirectory
  41 next to this script itself.  Each should contain a list of paths to individual
  42 tests or entire subdirectories of tests, relative to the outermost test
  43 directory.  Entire lines starting with '//' (comments) will be ignored.
  44
  45 For details of the files' contents and purposes, see test_lists/README.
  46 """
  47
  48 from __future__ import with_statement
  49
  50 import codecs
  51 import errno
  52 import glob
  53 import logging
  54 import math
  55 import optparse
  56 import os
  57 import platform
  58 import Queue
  59 import random
  60 import re
  61 import shutil
  62 import signal
  63 import sys
  64 import time
  65 import traceback
  66
  67 from layout_package import dump_render_tree_thread
  68 from layout_package import json_layout_results_generator
  69 from layout_package import message_broker
  70 from layout_package import printing
  71 from layout_package import test_expectations
  72 from layout_package import test_failures
  73 from layout_package import test_results
  74 from layout_package import test_results_uploader
  75
  76 from webkitpy.common.system import user
  77 from webkitpy.thirdparty import simplejson
  78 from webkitpy.tool import grammar
  79
  80 import port
  81
  82 _log = logging.getLogger("webkitpy.layout_tests.run_webkit_tests")
  83
  84 # Builder base URL where we have the archived test results.
  85 BUILDER_BASE_URL = "http://build.chromium.org/buildbot/layout_test_results/"
  86
  87 LAYOUT_TESTS_DIRECTORY = "LayoutTests" + os.sep
  88
  89 TestExpectationsFile = test_expectations.TestExpectationsFile
  90
  91
  92 class TestInput:
  93     """Groups information about a test for easy passing of data."""
  94
  95     def __init__(self, filename, timeout):
  96         """Holds the input parameters for a test.
  97         Args:
  98           filename: Full path to the test.
  99           timeout: Timeout in msecs the driver should use while running the test
 100           """
 101         # FIXME: filename should really be test_name as a relative path.
 102         self.filename = filename
 103         self.timeout = timeout
 104         # The image_hash is used to avoid doing an image dump if the
 105         # checksums match. The image_hash is set later, and only if it is needed
 106         # for the test.
 107         self.image_hash = None
 108
 109
 110 class ResultSummary(object):
 111     """A class for partitioning the test results we get into buckets.
 112
 113     This class is basically a glorified struct and it's private to this file
 114     so we don't bother with any information hiding."""
 115
 116     def __init__(self, expectations, test_files):
 117         self.total = len(test_files)
 118         self.remaining = self.total
 119         self.expectations = expectations
 120         self.expected = 0
 121         self.unexpected = 0
 122         self.tests_by_expectation = {}
 123         self.tests_by_timeline = {}
 124         self.results = {}
 125         self.unexpected_results = {}
 126         self.failures = {}
 127         self.tests_by_expectation[test_expectations.SKIP] = set()
 128         for expectation in TestExpectationsFile.EXPECTATIONS.values():
 129             self.tests_by_expectation[expectation] = set()
 130         for timeline in TestExpectationsFile.TIMELINES.values():
 131             self.tests_by_timeline[timeline] = (
 132                 expectations.get_tests_with_timeline(timeline))
 133
 134     def add(self, result, expected):
 135         """Add a TestResult into the appropriate bin.
 136
 137         Args:
 138           result: TestResult from dump_render_tree_thread.
 139           expected: whether the result was what we expected it to be.
 140         """
 141
 142         self.tests_by_expectation[result.type].add(result.filename)
 143         self.results[result.filename] = result
 144         self.remaining -= 1
 145         if len(result.failures):
 146             self.failures[result.filename] = result.failures
 147         if expected:
 148             self.expected += 1
 149         else:
 150             self.unexpected_results[result.filename] = result.type
 151             self.unexpected += 1
 152
 153
 154 def summarize_unexpected_results(port_obj, expectations, result_summary,
 155                                  retry_summary):
 156     """Summarize any unexpected results as a dict.
 157
 158     FIXME: split this data structure into a separate class?
 159
 160     Args:
 161         port_obj: interface to port-specific hooks
 162         expectations: test_expectations.TestExpectations object
 163         result_summary: summary object from initial test runs
 164         retry_summary: summary object from final test run of retried tests
 165     Returns:
 166         A dictionary containing a summary of the unexpected results from the
 167         run, with the following fields:
 168         'version': a version indicator (1 in this version)
 169         'fixable': # of fixable tests (NOW - PASS)
 170         'skipped': # of skipped tests (NOW & SKIPPED)
 171         'num_regressions': # of non-flaky failures
 172         'num_flaky': # of flaky failures
 173         'num_passes': # of unexpected passes
 174         'tests': a dict of tests -> {'expected': '...', 'actual': '...'}
 175     """
 176     results = {}
 177     results['version'] = 1
 178
 179     tbe = result_summary.tests_by_expectation
 180     tbt = result_summary.tests_by_timeline
 181     results['fixable'] = len(tbt[test_expectations.NOW] -
 182                                 tbe[test_expectations.PASS])
 183     results['skipped'] = len(tbt[test_expectations.NOW] &
 184                                 tbe[test_expectations.SKIP])
 185
 186     num_passes = 0
 187     num_flaky = 0
 188     num_regressions = 0
 189     keywords = {}
 190     for k, v in TestExpectationsFile.EXPECTATIONS.iteritems():
 191         keywords[v] = k.upper()
 192
 193     tests = {}
 194     for filename, result in result_summary.unexpected_results.iteritems():
 195         # Note that if a test crashed in the original run, we ignore
 196         # whether or not it crashed when we retried it (if we retried it),
 197         # and always consider the result not flaky.
 198         test = port_obj.relative_test_filename(filename)
 199         expected = expectations.get_expectations_string(filename)
 200         actual = [keywords[result]]
 201
 202         if result == test_expectations.PASS:
 203             num_passes += 1
 204         elif result == test_expectations.CRASH:
 205             num_regressions += 1
 206         else:
 207             if filename not in retry_summary.unexpected_results:
 208                 actual.extend(expectations.get_expectations_string(
 209                     filename).split(" "))
 210                 num_flaky += 1
 211             else:
 212                 retry_result = retry_summary.unexpected_results[filename]
 213                 if result != retry_result:
 214                     actual.append(keywords[retry_result])
 215                     num_flaky += 1
 216                 else:
 217                     num_regressions += 1
 218
 219         tests[test] = {}
 220         tests[test]['expected'] = expected
 221         tests[test]['actual'] = " ".join(actual)
 222
 223     results['tests'] = tests
 224     results['num_passes'] = num_passes
 225     results['num_flaky'] = num_flaky
 226     results['num_regressions'] = num_regressions
 227
 228     return results
 229
 230
 231 class TestRunner:
 232     """A class for managing running a series of tests on a series of layout
 233     test files."""
 234
 235     HTTP_SUBDIR = os.sep.join(['', 'http', ''])
 236     WEBSOCKET_SUBDIR = os.sep.join(['', 'websocket', ''])
 237
 238     # The per-test timeout in milliseconds, if no --time-out-ms option was
 239     # given to run_webkit_tests. This should correspond to the default timeout
 240     # in DumpRenderTree.
 241     DEFAULT_TEST_TIMEOUT_MS = 6 * 1000
 242
 243     def __init__(self, port, options, printer, message_broker):
 244         """Initialize test runner data structures.
 245
 246         Args:
 247           port: an object implementing port-specific
 248           options: a dictionary of command line options
 249           printer: a Printer object to record updates to.
 250           message_broker: object used to communicate with workers.
 251         """
 252         self._port = port
 253         self._options = options
 254         self._printer = printer
 255         self._message_broker = message_broker
 256
 257         # disable wss server. need to install pyOpenSSL on buildbots.
 258         # self._websocket_secure_server = websocket_server.PyWebSocket(
 259         #        options.results_directory, use_tls=True, port=9323)
 260
 261         # a set of test files, and the same tests as a list
 262         self._test_files = set()
 263         self._test_files_list = None
 264         self._result_queue = Queue.Queue()
 265         self._retrying = False
 266
 267     def collect_tests(self, args, last_unexpected_results):
 268         """Find all the files to test.
 269
 270         Args:
 271           args: list of test arguments from the command line
 272           last_unexpected_results: list of unexpected results to retest, if any
 273
 274         """
 275         paths = [self._strip_test_dir_prefix(arg) for arg in args if arg and arg != '']
 276         paths += last_unexpected_results
 277         if self._options.test_list:
 278             paths += read_test_files(self._options.test_list)
 279         self._test_files = self._port.tests(paths)
 280
 281     def _strip_test_dir_prefix(self, path):
 282         if path.startswith(LAYOUT_TESTS_DIRECTORY):
 283             return path[len(LAYOUT_TESTS_DIRECTORY):]
 284         return path
 285
 286     def lint(self):
 287         # Creating the expecations for each platform/configuration pair does
 288         # all the test list parsing and ensures it's correct syntax (e.g. no
 289         # dupes).
 290         for platform_name in self._port.test_platform_names():
 291             self.parse_expectations(platform_name, is_debug_mode=True)
 292             self.parse_expectations(platform_name, is_debug_mode=False)
 293         self._printer.write("")
 294         _log.info("If there are no fail messages, errors or exceptions, "
 295                   "then the lint succeeded.")
 296         return 0
 297
 298     def parse_expectations(self, test_platform_name, is_debug_mode):
 299         """Parse the expectations from the test_list files and return a data
 300         structure holding them. Throws an error if the test_list files have
 301         invalid syntax."""
 302         if self._options.lint_test_files:
 303             test_files = None
 304         else:
 305             test_files = self._test_files
 306
 307         try:
 308             expectations_str = self._port.test_expectations()
 309             overrides_str = self._port.test_expectations_overrides()
 310             self._expectations = test_expectations.TestExpectations(
 311                 self._port, test_files, expectations_str, test_platform_name,
 312                 is_debug_mode, self._options.lint_test_files,
 313                 overrides=overrides_str)
 314             return self._expectations
 315         except SyntaxError, err:
 316             if self._options.lint_test_files:
 317                 print str(err)
 318             else:
 319                 raise err
 320
 321     def prepare_lists_and_print_output(self):
 322         """Create appropriate subsets of test lists and returns a
 323         ResultSummary object. Also prints expected test counts.
 324         """
 325
 326         # Remove skipped - both fixable and ignored - files from the
 327         # top-level list of files to test.
 328         num_all_test_files = len(self._test_files)
 329         self._printer.print_expected("Found:  %d tests" %
 330                                      (len(self._test_files)))
 331         if not num_all_test_files:
 332             _log.critical('No tests to run.')
 333             return None
 334
 335         skipped = set()
 336         if num_all_test_files > 1 and not self._options.force:
 337             skipped = self._expectations.get_tests_with_result_type(
 338                            test_expectations.SKIP)
 339             self._test_files -= skipped
 340
 341         # Create a sorted list of test files so the subset chunk,
 342         # if used, contains alphabetically consecutive tests.
 343         self._test_files_list = list(self._test_files)
 344         if self._options.randomize_order:
 345             random.shuffle(self._test_files_list)
 346         else:
 347             self._test_files_list.sort()
 348
 349         # If the user specifies they just want to run a subset of the tests,
 350         # just grab a subset of the non-skipped tests.
 351         if self._options.run_chunk or self._options.run_part:
 352             chunk_value = self._options.run_chunk or self._options.run_part
 353             test_files = self._test_files_list
 354             try:
 355                 (chunk_num, chunk_len) = chunk_value.split(":")
 356                 chunk_num = int(chunk_num)
 357                 assert(chunk_num >= 0)
 358                 test_size = int(chunk_len)
 359                 assert(test_size > 0)
 360             except:
 361                 _log.critical("invalid chunk '%s'" % chunk_value)
 362                 return None
 363
 364             # Get the number of tests
 365             num_tests = len(test_files)
 366
 367             # Get the start offset of the slice.
 368             if self._options.run_chunk:
 369                 chunk_len = test_size
 370                 # In this case chunk_num can be really large. We need
 371                 # to make the slave fit in the current number of tests.
 372                 slice_start = (chunk_num * chunk_len) % num_tests
 373             else:
 374                 # Validate the data.
 375                 assert(test_size <= num_tests)
 376                 assert(chunk_num <= test_size)
 377
 378                 # To count the chunk_len, and make sure we don't skip
 379                 # some tests, we round to the next value that fits exactly
 380                 # all the parts.
 381                 rounded_tests = num_tests
 382                 if rounded_tests % test_size != 0:
 383                     rounded_tests = (num_tests + test_size -
 384                                      (num_tests % test_size))
 385
 386                 chunk_len = rounded_tests / test_size
 387                 slice_start = chunk_len * (chunk_num - 1)
 388                 # It does not mind if we go over test_size.
 389
 390             # Get the end offset of the slice.
 391             slice_end = min(num_tests, slice_start + chunk_len)
 392
 393             files = test_files[slice_start:slice_end]
 394
 395             tests_run_msg = 'Running: %d tests (chunk slice [%d:%d] of %d)' % (
 396                 (slice_end - slice_start), slice_start, slice_end, num_tests)
 397             self._printer.print_expected(tests_run_msg)
 398
 399             # If we reached the end and we don't have enough tests, we run some
 400             # from the beginning.
 401             if slice_end - slice_start < chunk_len:
 402                 extra = chunk_len - (slice_end - slice_start)
 403                 extra_msg = ('   last chunk is partial, appending [0:%d]' %
 404                             extra)
 405                 self._printer.print_expected(extra_msg)
 406                 tests_run_msg += "\n" + extra_msg
 407                 files.extend(test_files[0:extra])
 408             tests_run_filename = os.path.join(self._options.results_directory,
 409                                               "tests_run.txt")
 410             with codecs.open(tests_run_filename, "w", "utf-8") as file:
 411                 file.write(tests_run_msg + "\n")
 412
 413             len_skip_chunk = int(len(files) * len(skipped) /
 414                                  float(len(self._test_files)))
 415             skip_chunk_list = list(skipped)[0:len_skip_chunk]
 416             skip_chunk = set(skip_chunk_list)
 417
 418             # Update expectations so that the stats are calculated correctly.
 419             # We need to pass a list that includes the right # of skipped files
 420             # to ParseExpectations so that ResultSummary() will get the correct
 421             # stats. So, we add in the subset of skipped files, and then
 422             # subtract them back out.
 423             self._test_files_list = files + skip_chunk_list
 424             self._test_files = set(self._test_files_list)
 425
 426             self._expectations = self.parse_expectations(
 427                 self._port.test_platform_name(),
 428                 self._options.configuration == 'Debug')
 429
 430             self._test_files = set(files)
 431             self._test_files_list = files
 432         else:
 433             skip_chunk = skipped
 434
 435         result_summary = ResultSummary(self._expectations,
 436             self._test_files | skip_chunk)
 437         self._print_expected_results_of_type(result_summary,
 438             test_expectations.PASS, "passes")
 439         self._print_expected_results_of_type(result_summary,
 440             test_expectations.FAIL, "failures")
 441         self._print_expected_results_of_type(result_summary,
 442             test_expectations.FLAKY, "flaky")
 443         self._print_expected_results_of_type(result_summary,
 444             test_expectations.SKIP, "skipped")
 445
 446         if self._options.force:
 447             self._printer.print_expected('Running all tests, including '
 448                                          'skips (--force)')
 449         else:
 450             # Note that we don't actually run the skipped tests (they were
 451             # subtracted out of self._test_files, above), but we stub out the
 452             # results here so the statistics can remain accurate.
 453             for test in skip_chunk:
 454                 result = test_results.TestResult(test,
 455                     failures=[], test_run_time=0, total_time_for_all_diffs=0,
 456                     time_for_diffs=0)
 457                 result.type = test_expectations.SKIP
 458                 result_summary.add(result, expected=True)
 459         self._printer.print_expected('')
 460
 461         return result_summary
 462
 463     def _get_dir_for_test_file(self, test_file):
 464         """Returns the highest-level directory by which to shard the given
 465         test file."""
 466         index = test_file.rfind(os.sep + LAYOUT_TESTS_DIRECTORY)
 467
 468         test_file = test_file[index + len(LAYOUT_TESTS_DIRECTORY):]
 469         test_file_parts = test_file.split(os.sep, 1)
 470         directory = test_file_parts[0]
 471         test_file = test_file_parts[1]
 472
 473         # The http tests are very stable on mac/linux.
 474         # TODO(ojan): Make the http server on Windows be apache so we can
 475         # turn shard the http tests there as well. Switching to apache is
 476         # what made them stable on linux/mac.
 477         return_value = directory
 478         while ((directory != 'http' or sys.platform in ('darwin', 'linux2'))
 479                 and test_file.find(os.sep) >= 0):
 480             test_file_parts = test_file.split(os.sep, 1)
 481             directory = test_file_parts[0]
 482             return_value = os.path.join(return_value, directory)
 483             test_file = test_file_parts[1]
 484
 485         return return_value
 486
 487     def _get_test_input_for_file(self, test_file):
 488         """Returns the appropriate TestInput object for the file. Mostly this
 489         is used for looking up the timeout value (in ms) to use for the given
 490         test."""
 491         if self._test_is_slow(test_file):
 492             return TestInput(test_file, self._options.slow_time_out_ms)
 493         return TestInput(test_file, self._options.time_out_ms)
 494
 495     def _test_requires_lock(self, test_file):
 496         """Return True if the test needs to be locked when
 497         running multiple copies of NRWTs."""
 498         split_path = test_file.split(os.sep)
 499         return 'http' in split_path or 'websocket' in split_path
 500
 501     def _test_is_slow(self, test_file):
 502         return self._expectations.has_modifier(test_file,
 503                                                test_expectations.SLOW)
 504
 505     def _shard_tests(self, test_files, use_real_shards):
 506         """Groups tests into batches.
 507         This helps ensure that tests that depend on each other (aka bad tests!)
 508         continue to run together as most cross-tests dependencies tend to
 509         occur within the same directory. If use_real_shards is false, we
 510         put each (non-HTTP/websocket) test into its own shard for maximum
 511         concurrency instead of trying to do any sort of real sharding.
 512
 513         Return:
 514             A list of lists of TestInput objects.
 515         """
 516         # FIXME: when we added http locking, we changed how this works such
 517         # that we always lump all of the HTTP threads into a single shard.
 518         # That will slow down experimental-fully-parallel, but it's unclear
 519         # what the best alternative is completely revamping how we track
 520         # when to grab the lock.
 521
 522         test_lists = []
 523         tests_to_http_lock = []
 524         if not use_real_shards:
 525             for test_file in test_files:
 526                 test_input = self._get_test_input_for_file(test_file)
 527                 if self._test_requires_lock(test_file):
 528                     tests_to_http_lock.append(test_input)
 529                 else:
 530                     test_lists.append((".", [test_input]))
 531         else:
 532             tests_by_dir = {}
 533             for test_file in test_files:
 534                 directory = self._get_dir_for_test_file(test_file)
 535                 test_input = self._get_test_input_for_file(test_file)
 536                 if self._test_requires_lock(test_file):
 537                     tests_to_http_lock.append(test_input)
 538                 else:
 539                     tests_by_dir.setdefault(directory, [])
 540                     tests_by_dir[directory].append(test_input)
 541             # Sort by the number of tests in the dir so that the ones with the
 542             # most tests get run first in order to maximize parallelization.
 543             # Number of tests is a good enough, but not perfect, approximation
 544             # of how long that set of tests will take to run. We can't just use
 545             # a PriorityQueue until we move to Python 2.6.
 546             for directory in tests_by_dir:
 547                 test_list = tests_by_dir[directory]
 548                 # Keep the tests in alphabetical order.
 549                 # FIXME: Remove once tests are fixed so they can be run in any
 550                 # order.
 551                 test_list.reverse()
 552                 test_list_tuple = (directory, test_list)
 553                 test_lists.append(test_list_tuple)
 554             test_lists.sort(lambda a, b: cmp(len(b[1]), len(a[1])))
 555
 556         # Put the http tests first. There are only a couple hundred of them,
 557         # but each http test takes a very long time to run, so sorting by the
 558         # number of tests doesn't accurately capture how long they take to run.
 559         if tests_to_http_lock:
 560             tests_to_http_lock.reverse()
 561             test_lists.insert(0, ("tests_to_http_lock", tests_to_http_lock))
 562
 563         return test_lists
 564
 565     def _contains_tests(self, subdir):
 566         for test_file in self._test_files:
 567             if test_file.find(subdir) >= 0:
 568                 return True
 569         return False
 570
 571     def _num_workers(self):
 572         return int(self._options.child_processes)
 573
 574     def _run_tests(self, file_list, result_summary):
 575         """Runs the tests in the file_list.
 576
 577         Return: A tuple (keyboard_interrupted, thread_timings, test_timings,
 578             individual_test_timings)
 579             keyboard_interrupted is whether someone typed Ctrl^C
 580             thread_timings is a list of dicts with the total runtime
 581               of each thread with 'name', 'num_tests', 'total_time' properties
 582             test_timings is a list of timings for each sharded subdirectory
 583               of the form [time, directory_name, num_tests]
 584             individual_test_timings is a list of run times for each test
 585               in the form {filename:filename, test_run_time:test_run_time}
 586             result_summary: summary object to populate with the results
 587         """
 588
 589         self._printer.print_update('Sharding tests ...')
 590         num_workers = self._num_workers()
 591         test_lists = self._shard_tests(file_list,
 592             num_workers > 1 and not self._options.experimental_fully_parallel)
 593         filename_queue = Queue.Queue()
 594         for item in test_lists:
 595             filename_queue.put(item)
 596
 597         self._printer.print_update('Starting %s ...' %
 598                                    grammar.pluralize('worker', num_workers))
 599         message_broker = self._message_broker
 600         self._current_filename_queue = filename_queue
 601         self._current_result_summary = result_summary
 602         threads = message_broker.start_workers(self)
 603
 604         self._printer.print_update("Starting testing ...")
 605         keyboard_interrupted = False
 606         try:
 607             message_broker.run_message_loop()
 608         except KeyboardInterrupt:
 609             _log.info("Interrupted, exiting")
 610             message_broker.cancel_workers()
 611             keyboard_interrupted = True
 612         except:
 613             # Unexpected exception; don't try to clean up workers.
 614             _log.info("Exception raised, exiting")
 615             raise
 616
 617         thread_timings, test_timings, individual_test_timings = \
 618             self._collect_timing_info(threads)
 619
 620         return (keyboard_interrupted, thread_timings, test_timings,
 621                 individual_test_timings)
 622
 623     def update(self):
 624         self.update_summary(self._current_result_summary)
 625
 626     def _collect_timing_info(self, threads):
 627         test_timings = {}
 628         individual_test_timings = []
 629         thread_timings = []
 630
 631         for thread in threads:
 632             thread_timings.append({'name': thread.getName(),
 633                                    'num_tests': thread.get_num_tests(),
 634                                    'total_time': thread.get_total_time()})
 635             test_timings.update(thread.get_test_group_timing_stats())
 636             individual_test_timings.extend(thread.get_test_results())
 637
 638         return (thread_timings, test_timings, individual_test_timings)
 639
 640     def needs_http(self):
 641         """Returns whether the test runner needs an HTTP server."""
 642         return self._contains_tests(self.HTTP_SUBDIR)
 643
 644     def needs_websocket(self):
 645         """Returns whether the test runner needs a WEBSOCKET server."""
 646         return self._contains_tests(self.WEBSOCKET_SUBDIR)
 647
 648     def set_up_run(self):
 649         """Configures the system to be ready to run tests.
 650
 651         Returns a ResultSummary object if we should continue to run tests,
 652         or None if we should abort.
 653
 654         """
 655         # This must be started before we check the system dependencies,
 656         # since the helper may do things to make the setup correct.
 657         self._printer.print_update("Starting helper ...")
 658         self._port.start_helper()
 659
 660         # Check that the system dependencies (themes, fonts, ...) are correct.
 661         if not self._options.nocheck_sys_deps:
 662             self._printer.print_update("Checking system dependencies ...")
 663             if not self._port.check_sys_deps(self.needs_http()):
 664                 self._port.stop_helper()
 665                 return None
 666
 667         if self._options.clobber_old_results:
 668             self._clobber_old_results()
 669
 670         # Create the output directory if it doesn't already exist.
 671         self._port.maybe_make_directory(self._options.results_directory)
 672
 673         self._port.setup_test_run()
 674
 675         self._printer.print_update("Preparing tests ...")
 676         result_summary = self.prepare_lists_and_print_output()
 677         if not result_summary:
 678             return None
 679
 680         return result_summary
 681
 682     def run(self, result_summary):
 683         """Run all our tests on all our test files.
 684
 685         For each test file, we run each test type. If there are any failures,
 686         we collect them for reporting.
 687
 688         Args:
 689           result_summary: a summary object tracking the test results.
 690
 691         Return:
 692           The number of unexpected results (0 == success)
 693         """
 694         # gather_test_files() must have been called first to initialize us.
 695         # If we didn't find any files to test, we've errored out already in
 696         # prepare_lists_and_print_output().
 697         assert(len(self._test_files))
 698
 699         start_time = time.time()
 700
 701         keyboard_interrupted, thread_timings, test_timings, \
 702             individual_test_timings = (
 703             self._run_tests(self._test_files_list, result_summary))
 704
 705         # We exclude the crashes from the list of results to retry, because
 706         # we want to treat even a potentially flaky crash as an error.
 707         failures = self._get_failures(result_summary, include_crashes=False)
 708         retry_summary = result_summary
 709         while (len(failures) and self._options.retry_failures and
 710             not self._retrying and not keyboard_interrupted):
 711             _log.info('')
 712             _log.info("Retrying %d unexpected failure(s) ..." % len(failures))
 713             _log.info('')
 714             self._retrying = True
 715             retry_summary = ResultSummary(self._expectations, failures.keys())
 716             # Note that we intentionally ignore the return value here.
 717             self._run_tests(failures.keys(), retry_summary)
 718             failures = self._get_failures(retry_summary, include_crashes=True)
 719
 720         end_time = time.time()
 721
 722         self._print_timing_statistics(end_time - start_time,
 723                                       thread_timings, test_timings,
 724                                       individual_test_timings,
 725                                       result_summary)
 726
 727         self._print_result_summary(result_summary)
 728
 729         sys.stdout.flush()
 730         sys.stderr.flush()
 731
 732         self._printer.print_one_line_summary(result_summary.total,
 733                                              result_summary.expected,
 734                                              result_summary.unexpected)
 735
 736         unexpected_results = summarize_unexpected_results(self._port,
 737             self._expectations, result_summary, retry_summary)
 738         self._printer.print_unexpected_results(unexpected_results)
 739
 740         if self._options.record_results:
 741             # Write the same data to log files and upload generated JSON files
 742             # to appengine server.
 743             self._upload_json_files(unexpected_results, result_summary,
 744                                     individual_test_timings)
 745
 746         # Write the summary to disk (results.html) and display it if requested.
 747         wrote_results = self._write_results_html_file(result_summary)
 748         if self._options.show_results and wrote_results:
 749             self._show_results_html_file()
 750
 751         # Now that we've completed all the processing we can, we re-raise
 752         # a KeyboardInterrupt if necessary so the caller can handle it.
 753         if keyboard_interrupted:
 754             raise KeyboardInterrupt
 755
 756         # Ignore flaky failures and unexpected passes so we don't turn the
 757         # bot red for those.
 758         return unexpected_results['num_regressions']
 759
 760     def clean_up_run(self):
 761         """Restores the system after we're done running tests."""
 762
 763         _log.debug("flushing stdout")
 764         sys.stdout.flush()
 765         _log.debug("flushing stderr")
 766         sys.stderr.flush()
 767         _log.debug("stopping helper")
 768         self._port.stop_helper()
 769
 770     def update_summary(self, result_summary):
 771         """Update the summary and print results with any completed tests."""
 772         while True:
 773             try:
 774                 result = test_results.TestResult.loads(self._result_queue.get_nowait())
 775             except Queue.Empty:
 776                 return
 777
 778             expected = self._expectations.matches_an_expected_result(
 779                 result.filename, result.type, self._options.pixel_tests)
 780             result_summary.add(result, expected)
 781             exp_str = self._expectations.get_expectations_string(
 782                 result.filename)
 783             got_str = self._expectations.expectation_to_string(result.type)
 784             self._printer.print_test_result(result, expected, exp_str, got_str)
 785             self._printer.print_progress(result_summary, self._retrying,
 786                                          self._test_files_list)
 787
 788     def _clobber_old_results(self):
 789         # Just clobber the actual test results directories since the other
 790         # files in the results directory are explicitly used for cross-run
 791         # tracking.
 792         self._printer.print_update("Clobbering old results in %s" %
 793                                    self._options.results_directory)
 794         layout_tests_dir = self._port.layout_tests_dir()
 795         possible_dirs = self._port.test_dirs()
 796         for dirname in possible_dirs:
 797             if os.path.isdir(os.path.join(layout_tests_dir, dirname)):
 798                 shutil.rmtree(os.path.join(self._options.results_directory,
 799                                            dirname),
 800                               ignore_errors=True)
 801
 802     def _get_failures(self, result_summary, include_crashes):
 803         """Filters a dict of results and returns only the failures.
 804
 805         Args:
 806           result_summary: the results of the test run
 807           include_crashes: whether crashes are included in the output.
 808             We use False when finding the list of failures to retry
 809             to see if the results were flaky. Although the crashes may also be
 810             flaky, we treat them as if they aren't so that they're not ignored.
 811         Returns:
 812           a dict of files -> results
 813         """
 814         failed_results = {}
 815         for test, result in result_summary.unexpected_results.iteritems():
 816             if (result == test_expectations.PASS or
 817                 result == test_expectations.CRASH and not include_crashes):
 818                 continue
 819             failed_results[test] = result
 820
 821         return failed_results
 822
 823     def _upload_json_files(self, unexpected_results, result_summary,
 824                         individual_test_timings):
 825         """Writes the results of the test run as JSON files into the results
 826         dir and upload the files to the appengine server.
 827
 828         There are three different files written into the results dir:
 829           unexpected_results.json: A short list of any unexpected results.
 830             This is used by the buildbots to display results.
 831           expectations.json: This is used by the flakiness dashboard.
 832           results.json: A full list of the results - used by the flakiness
 833             dashboard and the aggregate results dashboard.
 834
 835         Args:
 836           unexpected_results: dict of unexpected results
 837           result_summary: full summary object
 838           individual_test_timings: list of test times (used by the flakiness
 839             dashboard).
 840         """
 841         results_directory = self._options.results_directory
 842         _log.debug("Writing JSON files in %s." % results_directory)
 843         unexpected_json_path = os.path.join(results_directory, "unexpected_results.json")
 844         with codecs.open(unexpected_json_path, "w", "utf-8") as file:
 845             simplejson.dump(unexpected_results, file, sort_keys=True, indent=2)
 846
 847         # Write a json file of the test_expectations.txt file for the layout
 848         # tests dashboard.
 849         expectations_path = os.path.join(results_directory, "expectations.json")
 850         expectations_json = \
 851             self._expectations.get_expectations_json_for_all_platforms()
 852         with codecs.open(expectations_path, "w", "utf-8") as file:
 853             file.write(u"ADD_EXPECTATIONS(%s);" % expectations_json)
 854
 855         generator = json_layout_results_generator.JSONLayoutResultsGenerator(
 856             self._port, self._options.builder_name, self._options.build_name,
 857             self._options.build_number, self._options.results_directory,
 858             BUILDER_BASE_URL, individual_test_timings,
 859             self._expectations, result_summary, self._test_files_list,
 860             not self._options.upload_full_results,
 861             self._options.test_results_server,
 862             "layout-tests",
 863             self._options.master_name)
 864
 865         _log.debug("Finished writing JSON files.")
 866
 867         json_files = ["expectations.json"]
 868         if self._options.upload_full_results:
 869             json_files.append("results.json")
 870         else:
 871             json_files.append("incremental_results.json")
 872
 873         generator.upload_json_files(json_files)
 874
 875     def _print_config(self):
 876         """Prints the configuration for the test run."""
 877         p = self._printer
 878         p.print_config("Using port '%s'" % self._port.name())
 879         p.print_config("Placing test results in %s" %
 880                        self._options.results_directory)
 881         if self._options.new_baseline:
 882             p.print_config("Placing new baselines in %s" %
 883                            self._port.baseline_path())
 884         p.print_config("Using %s build" % self._options.configuration)
 885         if self._options.pixel_tests:
 886             p.print_config("Pixel tests enabled")
 887         else:
 888             p.print_config("Pixel tests disabled")
 889
 890         p.print_config("Regular timeout: %s, slow test timeout: %s" %
 891                        (self._options.time_out_ms,
 892                         self._options.slow_time_out_ms))
 893
 894         if self._num_workers() == 1:
 895             p.print_config("Running one %s" % self._port.driver_name())
 896         else:
 897             p.print_config("Running %s %ss in parallel" %
 898                            (self._options.child_processes,
 899                             self._port.driver_name()))
 900         p.print_config('Command line: ' +
 901                        ' '.join(self._port.driver_cmd_line()))
 902         p.print_config("Worker model: %s" % self._options.worker_model)
 903         p.print_config("")
 904
 905     def _print_expected_results_of_type(self, result_summary,
 906                                         result_type, result_type_str):
 907         """Print the number of the tests in a given result class.
 908
 909         Args:
 910           result_summary - the object containing all the results to report on
 911           result_type - the particular result type to report in the summary.
 912           result_type_str - a string description of the result_type.
 913         """
 914         tests = self._expectations.get_tests_with_result_type(result_type)
 915         now = result_summary.tests_by_timeline[test_expectations.NOW]
 916         wontfix = result_summary.tests_by_timeline[test_expectations.WONTFIX]
 917
 918         # We use a fancy format string in order to print the data out in a
 919         # nicely-aligned table.
 920         fmtstr = ("Expect: %%5d %%-8s (%%%dd now, %%%dd wontfix)"
 921                   % (self._num_digits(now), self._num_digits(wontfix)))
 922         self._printer.print_expected(fmtstr %
 923             (len(tests), result_type_str, len(tests & now), len(tests & wontfix)))
 924
 925     def _num_digits(self, num):
 926         """Returns the number of digits needed to represent the length of a
 927         sequence."""
 928         ndigits = 1
 929         if len(num):
 930             ndigits = int(math.log10(len(num))) + 1
 931         return ndigits
 932
 933     def _print_timing_statistics(self, total_time, thread_timings,
 934                                directory_test_timings, individual_test_timings,
 935                                result_summary):
 936         """Record timing-specific information for the test run.
 937
 938         Args:
 939           total_time: total elapsed time (in seconds) for the test run
 940           thread_timings: wall clock time each thread ran for
 941           directory_test_timings: timing by directory
 942           individual_test_timings: timing by file
 943           result_summary: summary object for the test run
 944         """
 945         self._printer.print_timing("Test timing:")
 946         self._printer.print_timing("  %6.2f total testing time" % total_time)
 947         self._printer.print_timing("")
 948         self._printer.print_timing("Thread timing:")
 949         cuml_time = 0
 950         for t in thread_timings:
 951             self._printer.print_timing("    %10s: %5d tests, %6.2f secs" %
 952                   (t['name'], t['num_tests'], t['total_time']))
 953             cuml_time += t['total_time']
 954         self._printer.print_timing("   %6.2f cumulative, %6.2f optimal" %
 955               (cuml_time, cuml_time / int(self._options.child_processes)))
 956         self._printer.print_timing("")
 957
 958         self._print_aggregate_test_statistics(individual_test_timings)
 959         self._print_individual_test_times(individual_test_timings,
 960                                           result_summary)
 961         self._print_directory_timings(directory_test_timings)
 962
 963     def _print_aggregate_test_statistics(self, individual_test_timings):
 964         """Prints aggregate statistics (e.g. median, mean, etc.) for all tests.
 965         Args:
 966           individual_test_timings: List of dump_render_tree_thread.TestStats
 967               for all tests.
 968         """
 969         test_types = []  # Unit tests don't actually produce any timings.
 970         if individual_test_timings:
 971             test_types = individual_test_timings[0].time_for_diffs.keys()
 972         times_for_dump_render_tree = []
 973         times_for_diff_processing = []
 974         times_per_test_type = {}
 975         for test_type in test_types:
 976             times_per_test_type[test_type] = []
 977
 978         for test_stats in individual_test_timings:
 979             times_for_dump_render_tree.append(test_stats.test_run_time)
 980             times_for_diff_processing.append(
 981                 test_stats.total_time_for_all_diffs)
 982             time_for_diffs = test_stats.time_for_diffs
 983             for test_type in test_types:
 984                 times_per_test_type[test_type].append(
 985                     time_for_diffs[test_type])
 986
 987         self._print_statistics_for_test_timings(
 988             "PER TEST TIME IN TESTSHELL (seconds):",
 989             times_for_dump_render_tree)
 990         self._print_statistics_for_test_timings(
 991             "PER TEST DIFF PROCESSING TIMES (seconds):",
 992             times_for_diff_processing)
 993         for test_type in test_types:
 994             self._print_statistics_for_test_timings(
 995                 "PER TEST TIMES BY TEST TYPE: %s" % test_type,
 996                 times_per_test_type[test_type])
 997
 998     def _print_individual_test_times(self, individual_test_timings,
 999                                   result_summary):
1000         """Prints the run times for slow, timeout and crash tests.
1001         Args:
1002           individual_test_timings: List of dump_render_tree_thread.TestStats
1003               for all tests.
1004           result_summary: summary object for test run
1005         """
1006         # Reverse-sort by the time spent in DumpRenderTree.
1007         individual_test_timings.sort(lambda a, b:
1008             cmp(b.test_run_time, a.test_run_time))
1009
1010         num_printed = 0
1011         slow_tests = []
1012         timeout_or_crash_tests = []
1013         unexpected_slow_tests = []
1014         for test_tuple in individual_test_timings:
1015             filename = test_tuple.filename
1016             is_timeout_crash_or_slow = False
1017             if self._test_is_slow(filename):
1018                 is_timeout_crash_or_slow = True
1019                 slow_tests.append(test_tuple)
1020
1021             if filename in result_summary.failures:
1022                 result = result_summary.results[filename].type
1023                 if (result == test_expectations.TIMEOUT or
1024                     result == test_expectations.CRASH):
1025                     is_timeout_crash_or_slow = True
1026                     timeout_or_crash_tests.append(test_tuple)
1027
1028             if (not is_timeout_crash_or_slow and
1029                 num_printed < printing.NUM_SLOW_TESTS_TO_LOG):
1030                 num_printed = num_printed + 1
1031                 unexpected_slow_tests.append(test_tuple)
1032
1033         self._printer.print_timing("")
1034         self._print_test_list_timing("%s slowest tests that are not "
1035             "marked as SLOW and did not timeout/crash:" %
1036             printing.NUM_SLOW_TESTS_TO_LOG, unexpected_slow_tests)
1037         self._printer.print_timing("")
1038         self._print_test_list_timing("Tests marked as SLOW:", slow_tests)
1039         self._printer.print_timing("")
1040         self._print_test_list_timing("Tests that timed out or crashed:",
1041                                      timeout_or_crash_tests)
1042         self._printer.print_timing("")
1043
1044     def _print_test_list_timing(self, title, test_list):
1045         """Print timing info for each test.
1046
1047         Args:
1048           title: section heading
1049           test_list: tests that fall in this section
1050         """
1051         if self._printer.disabled('slowest'):
1052             return
1053
1054         self._printer.print_timing(title)
1055         for test_tuple in test_list:
1056             filename = test_tuple.filename[len(
1057                 self._port.layout_tests_dir()) + 1:]
1058             filename = filename.replace('\\', '/')
1059             test_run_time = round(test_tuple.test_run_time, 1)
1060             self._printer.print_timing("  %s took %s seconds" %
1061                                        (filename, test_run_time))
1062
1063     def _print_directory_timings(self, directory_test_timings):
1064         """Print timing info by directory for any directories that
1065         take > 10 seconds to run.
1066
1067         Args:
1068           directory_test_timing: time info for each directory
1069         """
1070         timings = []
1071         for directory in directory_test_timings:
1072             num_tests, time_for_directory = directory_test_timings[directory]
1073             timings.append((round(time_for_directory, 1), directory,
1074                             num_tests))
1075         timings.sort()
1076
1077         self._printer.print_timing("Time to process slowest subdirectories:")
1078         min_seconds_to_print = 10
1079         for timing in timings:
1080             if timing[0] > min_seconds_to_print:
1081                 self._printer.print_timing(
1082                     "  %s took %s seconds to run %s tests." % (timing[1],
1083                     timing[0], timing[2]))
1084         self._printer.print_timing("")
1085
1086     def _print_statistics_for_test_timings(self, title, timings):
1087         """Prints the median, mean and standard deviation of the values in
1088         timings.
1089
1090         Args:
1091           title: Title for these timings.
1092           timings: A list of floats representing times.
1093         """
1094         self._printer.print_timing(title)
1095         timings.sort()
1096
1097         num_tests = len(timings)
1098         if not num_tests:
1099             return
1100         percentile90 = timings[int(.9 * num_tests)]
1101         percentile99 = timings[int(.99 * num_tests)]
1102
1103         if num_tests % 2 == 1:
1104             median = timings[((num_tests - 1) / 2) - 1]
1105         else:
1106             lower = timings[num_tests / 2 - 1]
1107             upper = timings[num_tests / 2]
1108             median = (float(lower + upper)) / 2
1109
1110         mean = sum(timings) / num_tests
1111
1112         for time in timings:
1113             sum_of_deviations = math.pow(time - mean, 2)
1114
1115         std_deviation = math.sqrt(sum_of_deviations / num_tests)
1116         self._printer.print_timing("  Median:          %6.3f" % median)
1117         self._printer.print_timing("  Mean:            %6.3f" % mean)
1118         self._printer.print_timing("  90th percentile: %6.3f" % percentile90)
1119         self._printer.print_timing("  99th percentile: %6.3f" % percentile99)
1120         self._printer.print_timing("  Standard dev:    %6.3f" % std_deviation)
1121         self._printer.print_timing("")
1122
1123     def _print_result_summary(self, result_summary):
1124         """Print a short summary about how many tests passed.
1125
1126         Args:
1127           result_summary: information to log
1128         """
1129         failed = len(result_summary.failures)
1130         skipped = len(
1131             result_summary.tests_by_expectation[test_expectations.SKIP])
1132         total = result_summary.total
1133         passed = total - failed - skipped
1134         pct_passed = 0.0
1135         if total > 0:
1136             pct_passed = float(passed) * 100 / total
1137
1138         self._printer.print_actual("")
1139         self._printer.print_actual("=> Results: %d/%d tests passed (%.1f%%)" %
1140                      (passed, total, pct_passed))
1141         self._printer.print_actual("")
1142         self._print_result_summary_entry(result_summary,
1143             test_expectations.NOW, "Tests to be fixed")
1144
1145         self._printer.print_actual("")
1146         self._print_result_summary_entry(result_summary,
1147             test_expectations.WONTFIX,
1148             "Tests that will only be fixed if they crash (WONTFIX)")
1149         self._printer.print_actual("")
1150
1151     def _print_result_summary_entry(self, result_summary, timeline,
1152                                     heading):
1153         """Print a summary block of results for a particular timeline of test.
1154
1155         Args:
1156           result_summary: summary to print results for
1157           timeline: the timeline to print results for (NOT, WONTFIX, etc.)
1158           heading: a textual description of the timeline
1159         """
1160         total = len(result_summary.tests_by_timeline[timeline])
1161         not_passing = (total -
1162            len(result_summary.tests_by_expectation[test_expectations.PASS] &
1163                result_summary.tests_by_timeline[timeline]))
1164         self._printer.print_actual("=> %s (%d):" % (heading, not_passing))
1165
1166         for result in TestExpectationsFile.EXPECTATION_ORDER:
1167             if result == test_expectations.PASS:
1168                 continue
1169             results = (result_summary.tests_by_expectation[result] &
1170                        result_summary.tests_by_timeline[timeline])
1171             desc = TestExpectationsFile.EXPECTATION_DESCRIPTIONS[result]
1172             if not_passing and len(results):
1173                 pct = len(results) * 100.0 / not_passing
1174                 self._printer.print_actual("  %5d %-24s (%4.1f%%)" %
1175                     (len(results), desc[len(results) != 1], pct))
1176
1177     def _results_html(self, test_files, failures, title="Test Failures", override_time=None):
1178         """
1179         test_files = a list of file paths
1180         failures = dictionary mapping test paths to failure objects
1181         title = title printed at top of test
1182         override_time = current time (used by unit tests)
1183         """
1184         page = """<html>
1185   <head>
1186     <title>Layout Test Results (%(time)s)</title>
1187   </head>
1188   <body>
1189     <h2>%(title)s (%(time)s)</h2>
1190         """ % {'title': title, 'time': override_time or time.asctime()}
1191
1192         for test_file in sorted(test_files):
1193             test_name = self._port.relative_test_filename(test_file)
1194             test_url = self._port.filename_to_uri(test_file)
1195             page += u"<p><a href='%s'>%s</a><br />\n" % (test_url, test_name)
1196             test_failures = failures.get(test_file, [])
1197             for failure in test_failures:
1198                 page += (u"&nbsp;&nbsp;%s<br/>" %
1199                          failure.result_html_output(test_name))
1200             page += "</p>\n"
1201         page += "</body></html>\n"
1202         return page
1203
1204     def _write_results_html_file(self, result_summary):
1205         """Write results.html which is a summary of tests that failed.
1206
1207         Args:
1208           result_summary: a summary of the results :)
1209
1210         Returns:
1211           True if any results were written (since expected failures may be
1212           omitted)
1213         """
1214         # test failures
1215         if self._options.full_results_html:
1216             results_title = "Test Failures"
1217             test_files = result_summary.failures.keys()
1218         else:
1219             results_title = "Unexpected Test Failures"
1220             unexpected_failures = self._get_failures(result_summary,
1221                 include_crashes=True)
1222             test_files = unexpected_failures.keys()
1223         if not len(test_files):
1224             return False
1225
1226         out_filename = os.path.join(self._options.results_directory,
1227                                     "results.html")
1228         with codecs.open(out_filename, "w", "utf-8") as results_file:
1229             html = self._results_html(test_files, result_summary.failures, results_title)
1230             results_file.write(html)
1231
1232         return True
1233
1234     def _show_results_html_file(self):
1235         """Shows the results.html page."""
1236         results_filename = os.path.join(self._options.results_directory,
1237                                         "results.html")
1238         self._port.show_results_html_file(results_filename)
1239
1240
1241 def read_test_files(files):
1242     tests = []
1243     for file in files:
1244         try:
1245             with codecs.open(file, 'r', 'utf-8') as file_contents:
1246                 # FIXME: This could be cleaner using a list comprehension.
1247                 for line in file_contents:
1248                     line = test_expectations.strip_comments(line)
1249                     if line:
1250                         tests.append(line)
1251         except IOError, e:
1252             if e.errno == errno.ENOENT:
1253                 _log.critical('')
1254                 _log.critical('--test-list file "%s" not found' % file)
1255             raise
1256     return tests
1257
1258
1259 def run(port, options, args, regular_output=sys.stderr,
1260         buildbot_output=sys.stdout):
1261     """Run the tests.
1262
1263     Args:
1264       port: Port object for port-specific behavior
1265       options: a dictionary of command line options
1266       args: a list of sub directories or files to test
1267       regular_output: a stream-like object that we can send logging/debug
1268           output to
1269       buildbot_output: a stream-like object that we can write all output that
1270           is intended to be parsed by the buildbot to
1271     Returns:
1272       the number of unexpected results that occurred, or -1 if there is an
1273           error.
1274
1275     """
1276     _set_up_derived_options(port, options)
1277
1278     printer = printing.Printer(port, options, regular_output, buildbot_output,
1279         int(options.child_processes), options.experimental_fully_parallel)
1280     if options.help_printing:
1281         printer.help_printing()
1282         printer.cleanup()
1283         return 0
1284
1285     last_unexpected_results = _gather_unexpected_results(options)
1286     if options.print_last_failures:
1287         printer.write("\n".join(last_unexpected_results) + "\n")
1288         printer.cleanup()
1289         return 0
1290
1291     broker = message_broker.get(port, options)
1292
1293     # We wrap any parts of the run that are slow or likely to raise exceptions
1294     # in a try/finally to ensure that we clean up the logging configuration.
1295     num_unexpected_results = -1
1296     try:
1297         test_runner = TestRunner(port, options, printer, broker)
1298         test_runner._print_config()
1299
1300         printer.print_update("Collecting tests ...")
1301         try:
1302             test_runner.collect_tests(args, last_unexpected_results)
1303         except IOError, e:
1304             if e.errno == errno.ENOENT:
1305                 return -1
1306             raise
1307
1308         printer.print_update("Parsing expectations ...")
1309         if options.lint_test_files:
1310             return test_runner.lint()
1311         test_runner.parse_expectations(port.test_platform_name(),
1312                                        options.configuration == 'Debug')
1313
1314         printer.print_update("Checking build ...")
1315         if not port.check_build(test_runner.needs_http()):
1316             _log.error("Build check failed")
1317             return -1
1318
1319         result_summary = test_runner.set_up_run()
1320         if result_summary:
1321             num_unexpected_results = test_runner.run(result_summary)
1322             test_runner.clean_up_run()
1323             _log.debug("Testing completed, Exit status: %d" %
1324                        num_unexpected_results)
1325     finally:
1326         broker.cleanup()
1327         printer.cleanup()
1328
1329     return num_unexpected_results
1330
1331
1332 def _set_up_derived_options(port_obj, options):
1333     """Sets the options values that depend on other options values."""
1334
1335     if options.worker_model == 'inline':
1336         if options.child_processes and int(options.child_processes) > 1:
1337             _log.warning("--worker-model=inline overrides --child-processes")
1338         options.child_processes = "1"
1339     if not options.child_processes:
1340         options.child_processes = os.environ.get("WEBKIT_TEST_CHILD_PROCESSES",
1341                                                  str(port_obj.default_child_processes()))
1342
1343     if not options.configuration:
1344         options.configuration = port_obj.default_configuration()
1345
1346     if options.pixel_tests is None:
1347         options.pixel_tests = True
1348
1349     if not options.use_apache:
1350         options.use_apache = sys.platform in ('darwin', 'linux2')
1351
1352     if not os.path.isabs(options.results_directory):
1353         # This normalizes the path to the build dir.
1354         # FIXME: how this happens is not at all obvious; this is a dumb
1355         # interface and should be cleaned up.
1356         options.results_directory = port_obj.results_directory()
1357
1358     if not options.time_out_ms:
1359         if options.configuration == "Debug":
1360             options.time_out_ms = str(2 * TestRunner.DEFAULT_TEST_TIMEOUT_MS)
1361         else:
1362             options.time_out_ms = str(TestRunner.DEFAULT_TEST_TIMEOUT_MS)
1363
1364     options.slow_time_out_ms = str(5 * int(options.time_out_ms))
1365
1366
1367 def _gather_unexpected_results(options):
1368     """Returns the unexpected results from the previous run, if any."""
1369     last_unexpected_results = []
1370     if options.print_last_failures or options.retest_last_failures:
1371         unexpected_results_filename = os.path.join(
1372         options.results_directory, "unexpected_results.json")
1373         with codecs.open(unexpected_results_filename, "r", "utf-8") as file:
1374             results = simplejson.load(file)
1375         last_unexpected_results = results['tests'].keys()
1376     return last_unexpected_results
1377
1378
1379 def _compat_shim_callback(option, opt_str, value, parser):
1380     print "Ignoring unsupported option: %s" % opt_str
1381
1382
1383 def _compat_shim_option(option_name, **kwargs):
1384     return optparse.make_option(option_name, action="callback",
1385         callback=_compat_shim_callback,
1386         help="Ignored, for old-run-webkit-tests compat only.", **kwargs)
1387
1388
1389 def parse_args(args=None):
1390     """Provides a default set of command line args.
1391
1392     Returns a tuple of options, args from optparse"""
1393
1394     # FIXME: All of these options should be stored closer to the code which
1395     # FIXME: actually uses them. configuration_options should move
1396     # FIXME: to WebKitPort and be shared across all scripts.
1397     configuration_options = [
1398         optparse.make_option("-t", "--target", dest="configuration",
1399                              help="(DEPRECATED)"),
1400         # FIXME: --help should display which configuration is default.
1401         optparse.make_option('--debug', action='store_const', const='Debug',
1402                              dest="configuration",
1403                              help='Set the configuration to Debug'),
1404         optparse.make_option('--release', action='store_const',
1405                              const='Release', dest="configuration",
1406                              help='Set the configuration to Release'),
1407         # old-run-webkit-tests also accepts -c, --configuration CONFIGURATION.
1408     ]
1409
1410     print_options = printing.print_options()
1411
1412     # FIXME: These options should move onto the ChromiumPort.
1413     chromium_options = [
1414         optparse.make_option("--chromium", action="store_true", default=False,
1415             help="use the Chromium port"),
1416         optparse.make_option("--startup-dialog", action="store_true",
1417             default=False, help="create a dialog on DumpRenderTree startup"),
1418         optparse.make_option("--gp-fault-error-box", action="store_true",
1419             default=False, help="enable Windows GP fault error box"),
1420         optparse.make_option("--multiple-loads",
1421             type="int", help="turn on multiple loads of each test"),
1422         optparse.make_option("--js-flags",
1423             type="string", help="JavaScript flags to pass to tests"),
1424         optparse.make_option("--nocheck-sys-deps", action="store_true",
1425             default=False,
1426             help="Don't check the system dependencies (themes)"),
1427         optparse.make_option("--use-drt", action="store_true",
1428             default=None,
1429             help="Use DumpRenderTree instead of test_shell"),
1430         optparse.make_option("--accelerated-compositing",
1431             action="store_true",
1432             help="Use hardware-accelated compositing for rendering"),
1433         optparse.make_option("--no-accelerated-compositing",
1434             action="store_false",
1435             dest="accelerated_compositing",
1436             help="Don't use hardware-accelerated compositing for rendering"),
1437         optparse.make_option("--accelerated-2d-canvas",
1438             action="store_true",
1439             help="Use hardware-accelerated 2D Canvas calls"),
1440         optparse.make_option("--no-accelerated-2d-canvas",
1441             action="store_false",
1442             dest="accelerated_2d_canvas",
1443             help="Don't use hardware-accelerated 2D Canvas calls"),
1444     ]
1445
1446     # Missing Mac-specific old-run-webkit-tests options:
1447     # FIXME: Need: -g, --guard for guard malloc support on Mac.
1448     # FIXME: Need: -l --leaks    Enable leaks checking.
1449     # FIXME: Need: --sample-on-timeout Run sample on timeout
1450
1451     old_run_webkit_tests_compat = [
1452         # NRWT doesn't generate results by default anyway.
1453         _compat_shim_option("--no-new-test-results"),
1454         # NRWT doesn't sample on timeout yet anyway.
1455         _compat_shim_option("--no-sample-on-timeout"),
1456         # FIXME: NRWT needs to support remote links eventually.
1457         _compat_shim_option("--use-remote-links-to-tests"),
1458         # FIXME: NRWT doesn't need this option as much since failures are
1459         # designed to be cheap.  We eventually plan to add this support.
1460         _compat_shim_option("--exit-after-n-failures", nargs=1, type="int"),
1461     ]
1462
1463     results_options = [
1464         # NEED for bots: --use-remote-links-to-tests Link to test files
1465         # within the SVN repository in the results.
1466         optparse.make_option("-p", "--pixel-tests", action="store_true",
1467             dest="pixel_tests", help="Enable pixel-to-pixel PNG comparisons"),
1468         optparse.make_option("--no-pixel-tests", action="store_false",
1469             dest="pixel_tests", help="Disable pixel-to-pixel PNG comparisons"),
1470         optparse.make_option("--tolerance",
1471             help="Ignore image differences less than this percentage (some "
1472                 "ports may ignore this option)", type="float"),
1473         optparse.make_option("--results-directory",
1474             default="layout-test-results",
1475             help="Output results directory source dir, relative to Debug or "
1476                  "Release"),
1477         optparse.make_option("--new-baseline", action="store_true",
1478             default=False, help="Save all generated results as new baselines "
1479                  "into the platform directory, overwriting whatever's "
1480                  "already there."),
1481         optparse.make_option("--reset-results", action="store_true",
1482             default=False, help="Reset any existing baselines to the "
1483                  "generated results"),
1484         optparse.make_option("--no-show-results", action="store_false",
1485             default=True, dest="show_results",
1486             help="Don't launch a browser with results after the tests "
1487                  "are done"),
1488         # FIXME: We should have a helper function to do this sort of
1489         # deprectated mapping and automatically log, etc.
1490         optparse.make_option("--noshow-results", action="store_false",
1491             dest="show_results",
1492             help="Deprecated, same as --no-show-results."),
1493         optparse.make_option("--no-launch-safari", action="store_false",
1494             dest="show_results",
1495             help="old-run-webkit-tests compat, same as --noshow-results."),
1496         # old-run-webkit-tests:
1497         # --[no-]launch-safari    Launch (or do not launch) Safari to display
1498         #                         test results (default: launch)
1499         optparse.make_option("--full-results-html", action="store_true",
1500             default=False,
1501             help="Show all failures in results.html, rather than only "
1502                  "regressions"),
1503         optparse.make_option("--clobber-old-results", action="store_true",
1504             default=False, help="Clobbers test results from previous runs."),
1505         optparse.make_option("--platform",
1506             help="Override the platform for expected results"),
1507         optparse.make_option("--no-record-results", action="store_false",
1508             default=True, dest="record_results",
1509             help="Don't record the results."),
1510         # old-run-webkit-tests also has HTTP toggle options:
1511         # --[no-]http                     Run (or do not run) http tests
1512         #                                 (default: run)
1513     ]
1514
1515     test_options = [
1516         optparse.make_option("--build", dest="build",
1517             action="store_true", default=True,
1518             help="Check to ensure the DumpRenderTree build is up-to-date "
1519                  "(default)."),
1520         optparse.make_option("--no-build", dest="build",
1521             action="store_false", help="Don't check to see if the "
1522                                        "DumpRenderTree build is up-to-date."),
1523         # old-run-webkit-tests has --valgrind instead of wrapper.
1524         optparse.make_option("--wrapper",
1525             help="wrapper command to insert before invocations of "
1526                  "DumpRenderTree; option is split on whitespace before "
1527                  "running. (Example: --wrapper='valgrind --smc-check=all')"),
1528         # old-run-webkit-tests:
1529         # -i|--ignore-tests               Comma-separated list of directories
1530         #                                 or tests to ignore
1531         optparse.make_option("--test-list", action="append",
1532             help="read list of tests to run from file", metavar="FILE"),
1533         # old-run-webkit-tests uses --skipped==[default|ignore|only]
1534         # instead of --force:
1535         optparse.make_option("--force", action="store_true", default=False,
1536             help="Run all tests, even those marked SKIP in the test list"),
1537         optparse.make_option("--use-apache", action="store_true",
1538             default=False, help="Whether to use apache instead of lighttpd."),
1539         optparse.make_option("--time-out-ms",
1540             help="Set the timeout for each test"),
1541         # old-run-webkit-tests calls --randomize-order --random:
1542         optparse.make_option("--randomize-order", action="store_true",
1543             default=False, help=("Run tests in random order (useful "
1544                                 "for tracking down corruption)")),
1545         optparse.make_option("--run-chunk",
1546             help=("Run a specified chunk (n:l), the nth of len l, "
1547                  "of the layout tests")),
1548         optparse.make_option("--run-part", help=("Run a specified part (n:m), "
1549                   "the nth of m parts, of the layout tests")),
1550         # old-run-webkit-tests calls --batch-size: --nthly n
1551         #   Restart DumpRenderTree every n tests (default: 1000)
1552         optparse.make_option("--batch-size",
1553             help=("Run a the tests in batches (n), after every n tests, "
1554                   "DumpRenderTree is relaunched."), type="int", default=0),
1555         # old-run-webkit-tests calls --run-singly: -1|--singly
1556         # Isolate each test case run (implies --nthly 1 --verbose)
1557         optparse.make_option("--run-singly", action="store_true",
1558             default=False, help="run a separate DumpRenderTree for each test"),
1559         optparse.make_option("--child-processes",
1560             help="Number of DumpRenderTrees to run in parallel."),
1561         # FIXME: Display default number of child processes that will run.
1562         optparse.make_option("--worker-model", action="store",
1563             default="threads", help=("controls worker model. Valid values are "
1564             "'inline' and 'threads' (default).")),
1565         optparse.make_option("--experimental-fully-parallel",
1566             action="store_true", default=False,
1567             help="run all tests in parallel"),
1568         # FIXME: Need --exit-after-n-failures N
1569         #      Exit after the first N failures instead of running all tests
1570         # FIXME: Need --exit-after-n-crashes N
1571         #      Exit after the first N crashes instead of running all tests
1572         # FIXME: consider: --iterations n
1573         #      Number of times to run the set of tests (e.g. ABCABCABC)
1574         optparse.make_option("--print-last-failures", action="store_true",
1575             default=False, help="Print the tests in the last run that "
1576             "had unexpected failures (or passes) and then exit."),
1577         optparse.make_option("--retest-last-failures", action="store_true",
1578             default=False, help="re-test the tests in the last run that "
1579             "had unexpected failures (or passes)."),
1580         optparse.make_option("--retry-failures", action="store_true",
1581             default=True,
1582             help="Re-try any tests that produce unexpected results (default)"),
1583         optparse.make_option("--no-retry-failures", action="store_false",
1584             dest="retry_failures",
1585             help="Don't re-try any tests that produce unexpected results."),
1586     ]
1587
1588     misc_options = [
1589         optparse.make_option("--lint-test-files", action="store_true",
1590         default=False, help=("Makes sure the test files parse for all "
1591                             "configurations. Does not run any tests.")),
1592     ]
1593
1594     # FIXME: Move these into json_results_generator.py
1595     results_json_options = [
1596         optparse.make_option("--master-name", help="The name of the buildbot master."),
1597         optparse.make_option("--builder-name", default="DUMMY_BUILDER_NAME",
1598             help=("The name of the builder shown on the waterfall running "
1599                   "this script e.g. WebKit.")),
1600         optparse.make_option("--build-name", default="DUMMY_BUILD_NAME",
1601             help=("The name of the builder used in its path, e.g. "
1602                   "webkit-rel.")),
1603         optparse.make_option("--build-number", default="DUMMY_BUILD_NUMBER",
1604             help=("The build number of the builder running this script.")),
1605         optparse.make_option("--test-results-server", default="",
1606             help=("If specified, upload results json files to this appengine "
1607                   "server.")),
1608         optparse.make_option("--upload-full-results",
1609             action="store_true",
1610             default=False,
1611             help="If true, upload full json results to server."),
1612     ]
1613
1614     option_list = (configuration_options + print_options +
1615                    chromium_options + results_options + test_options +
1616                    misc_options + results_json_options +
1617                    old_run_webkit_tests_compat)
1618     option_parser = optparse.OptionParser(option_list=option_list)
1619
1620     return option_parser.parse_args(args)
1621
1622
1623 def main():
1624     options, args = parse_args()
1625     port_obj = port.get(options.platform, options)
1626     return run(port_obj, options, args)
1627
1628
1629 if '__main__' == __name__:
1630     try:
1631         sys.exit(main())
1632     except KeyboardInterrupt:
1633         # this mirrors what the shell normally does
1634         sys.exit(signal.SIGINT + 128)