Tools/Scripts/webkitpy/layout_tests/run_webkit_tests.py

   1 #!/usr/bin/env python
   2 # Copyright (C) 2010 Google Inc. All rights reserved.
   3 # Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Szeged
   4 #
   5 # Redistribution and use in source and binary forms, with or without
   6 # modification, are permitted provided that the following conditions are
   7 # met:
   8 #
   9 #     * Redistributions of source code must retain the above copyright
  10 # notice, this list of conditions and the following disclaimer.
  11 #     * Redistributions in binary form must reproduce the above
  12 # copyright notice, this list of conditions and the following disclaimer
  13 # in the documentation and/or other materials provided with the
  14 # distribution.
  15 #     * Neither the name of Google Inc. nor the names of its
  16 # contributors may be used to endorse or promote products derived from
  17 # this software without specific prior written permission.
  18 #
  19 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  20 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  21 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  22 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  23 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  24 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  25 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  26 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  27 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  28 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  29 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  30
  31 """Run layout tests."""
  32
  33 from __future__ import with_statement
  34
  35 import codecs
  36 import errno
  37 import logging
  38 import optparse
  39 import os
  40 import signal
  41 import sys
  42
  43 from layout_package import printing
  44 from layout_package import test_runner
  45
  46 from webkitpy.common.system import user
  47 from webkitpy.thirdparty import simplejson
  48
  49 import port
  50
  51 _log = logging.getLogger(__name__)
  52
  53
  54 def run(port, options, args, regular_output=sys.stderr,
  55         buildbot_output=sys.stdout):
  56     """Run the tests.
  57
  58     Args:
  59       port: Port object for port-specific behavior
  60       options: a dictionary of command line options
  61       args: a list of sub directories or files to test
  62       regular_output: a stream-like object that we can send logging/debug
  63           output to
  64       buildbot_output: a stream-like object that we can write all output that
  65           is intended to be parsed by the buildbot to
  66     Returns:
  67       the number of unexpected results that occurred, or -1 if there is an
  68           error.
  69
  70     """
  71     warnings = _set_up_derived_options(port, options)
  72
  73     printer = printing.Printer(port, options, regular_output, buildbot_output,
  74         int(options.child_processes), options.experimental_fully_parallel)
  75     for w in warnings:
  76         _log.warning(w)
  77
  78     if options.help_printing:
  79         printer.help_printing()
  80         printer.cleanup()
  81         return 0
  82
  83     last_unexpected_results = _gather_unexpected_results(options)
  84     if options.print_last_failures:
  85         printer.write("\n".join(last_unexpected_results) + "\n")
  86         printer.cleanup()
  87         return 0
  88
  89     # We wrap any parts of the run that are slow or likely to raise exceptions
  90     # in a try/finally to ensure that we clean up the logging configuration.
  91     num_unexpected_results = -1
  92     try:
  93         runner = test_runner.TestRunner(port, options, printer)
  94         runner._print_config()
  95
  96         printer.print_update("Collecting tests ...")
  97         try:
  98             runner.collect_tests(args, last_unexpected_results)
  99         except IOError, e:
 100             if e.errno == errno.ENOENT:
 101                 return -1
 102             raise
 103
 104         printer.print_update("Parsing expectations ...")
 105         if options.lint_test_files:
 106             return runner.lint()
 107         runner.parse_expectations(port.test_platform_name(),
 108                                   options.configuration == 'Debug')
 109
 110         printer.print_update("Checking build ...")
 111         if not port.check_build(runner.needs_http()):
 112             _log.error("Build check failed")
 113             return -1
 114
 115         result_summary = runner.set_up_run()
 116         if result_summary:
 117             num_unexpected_results = runner.run(result_summary)
 118             runner.clean_up_run()
 119             _log.debug("Testing completed, Exit status: %d" %
 120                        num_unexpected_results)
 121     finally:
 122         printer.cleanup()
 123
 124     return num_unexpected_results
 125
 126
 127 def _set_up_derived_options(port_obj, options):
 128     """Sets the options values that depend on other options values."""
 129     # We return a list of warnings to print after the printer is initialized.
 130     warnings = []
 131
 132     if options.worker_model == 'old-inline':
 133         if options.child_processes and int(options.child_processes) > 1:
 134             warnings.append("--worker-model=old-inline overrides --child-processes")
 135         options.child_processes = "1"
 136     if not options.child_processes:
 137         options.child_processes = os.environ.get("WEBKIT_TEST_CHILD_PROCESSES",
 138                                                  str(port_obj.default_child_processes()))
 139
 140     if not options.configuration:
 141         options.configuration = port_obj.default_configuration()
 142
 143     if options.pixel_tests is None:
 144         options.pixel_tests = True
 145
 146     if not options.use_apache:
 147         options.use_apache = sys.platform in ('darwin', 'linux2')
 148
 149     if not os.path.isabs(options.results_directory):
 150         # This normalizes the path to the build dir.
 151         # FIXME: how this happens is not at all obvious; this is a dumb
 152         # interface and should be cleaned up.
 153         options.results_directory = port_obj.results_directory()
 154
 155     if not options.time_out_ms:
 156         if options.configuration == "Debug":
 157             options.time_out_ms = str(2 * test_runner.TestRunner.DEFAULT_TEST_TIMEOUT_MS)
 158         else:
 159             options.time_out_ms = str(test_runner.TestRunner.DEFAULT_TEST_TIMEOUT_MS)
 160
 161     options.slow_time_out_ms = str(5 * int(options.time_out_ms))
 162     return warnings
 163
 164
 165 def _gather_unexpected_results(options):
 166     """Returns the unexpected results from the previous run, if any."""
 167     last_unexpected_results = []
 168     if options.print_last_failures or options.retest_last_failures:
 169         unexpected_results_filename = os.path.join(
 170         options.results_directory, "unexpected_results.json")
 171         with codecs.open(unexpected_results_filename, "r", "utf-8") as file:
 172             results = simplejson.load(file)
 173         last_unexpected_results = results['tests'].keys()
 174     return last_unexpected_results
 175
 176
 177 def _compat_shim_callback(option, opt_str, value, parser):
 178     print "Ignoring unsupported option: %s" % opt_str
 179
 180
 181 def _compat_shim_option(option_name, **kwargs):
 182     return optparse.make_option(option_name, action="callback",
 183         callback=_compat_shim_callback,
 184         help="Ignored, for old-run-webkit-tests compat only.", **kwargs)
 185
 186
 187 def parse_args(args=None):
 188     """Provides a default set of command line args.
 189
 190     Returns a tuple of options, args from optparse"""
 191
 192     # FIXME: All of these options should be stored closer to the code which
 193     # FIXME: actually uses them. configuration_options should move
 194     # FIXME: to WebKitPort and be shared across all scripts.
 195     configuration_options = [
 196         optparse.make_option("-t", "--target", dest="configuration",
 197                              help="(DEPRECATED)"),
 198         # FIXME: --help should display which configuration is default.
 199         optparse.make_option('--debug', action='store_const', const='Debug',
 200                              dest="configuration",
 201                              help='Set the configuration to Debug'),
 202         optparse.make_option('--release', action='store_const',
 203                              const='Release', dest="configuration",
 204                              help='Set the configuration to Release'),
 205         # old-run-webkit-tests also accepts -c, --configuration CONFIGURATION.
 206     ]
 207
 208     print_options = printing.print_options()
 209
 210     # FIXME: These options should move onto the ChromiumPort.
 211     chromium_options = [
 212         optparse.make_option("--chromium", action="store_true", default=False,
 213             help="use the Chromium port"),
 214         optparse.make_option("--startup-dialog", action="store_true",
 215             default=False, help="create a dialog on DumpRenderTree startup"),
 216         optparse.make_option("--gp-fault-error-box", action="store_true",
 217             default=False, help="enable Windows GP fault error box"),
 218         optparse.make_option("--multiple-loads",
 219             type="int", help="turn on multiple loads of each test"),
 220         optparse.make_option("--js-flags",
 221             type="string", help="JavaScript flags to pass to tests"),
 222         optparse.make_option("--nocheck-sys-deps", action="store_true",
 223             default=False,
 224             help="Don't check the system dependencies (themes)"),
 225         optparse.make_option("--use-test-shell", action="store_true",
 226             default=False,
 227             help="Use test_shell instead of DRT"),
 228         optparse.make_option("--accelerated-compositing",
 229             action="store_true",
 230             help="Use hardware-accelated compositing for rendering"),
 231         optparse.make_option("--no-accelerated-compositing",
 232             action="store_false",
 233             dest="accelerated_compositing",
 234             help="Don't use hardware-accelerated compositing for rendering"),
 235         optparse.make_option("--accelerated-2d-canvas",
 236             action="store_true",
 237             help="Use hardware-accelerated 2D Canvas calls"),
 238         optparse.make_option("--no-accelerated-2d-canvas",
 239             action="store_false",
 240             dest="accelerated_2d_canvas",
 241             help="Don't use hardware-accelerated 2D Canvas calls"),
 242     ]
 243
 244     # Missing Mac-specific old-run-webkit-tests options:
 245     # FIXME: Need: -g, --guard for guard malloc support on Mac.
 246     # FIXME: Need: -l --leaks    Enable leaks checking.
 247     # FIXME: Need: --sample-on-timeout Run sample on timeout
 248
 249     old_run_webkit_tests_compat = [
 250         # NRWT doesn't generate results by default anyway.
 251         _compat_shim_option("--no-new-test-results"),
 252         # NRWT doesn't sample on timeout yet anyway.
 253         _compat_shim_option("--no-sample-on-timeout"),
 254         # FIXME: NRWT needs to support remote links eventually.
 255         _compat_shim_option("--use-remote-links-to-tests"),
 256     ]
 257
 258     results_options = [
 259         # NEED for bots: --use-remote-links-to-tests Link to test files
 260         # within the SVN repository in the results.
 261         optparse.make_option("-p", "--pixel-tests", action="store_true",
 262             dest="pixel_tests", help="Enable pixel-to-pixel PNG comparisons"),
 263         optparse.make_option("--no-pixel-tests", action="store_false",
 264             dest="pixel_tests", help="Disable pixel-to-pixel PNG comparisons"),
 265         optparse.make_option("--tolerance",
 266             help="Ignore image differences less than this percentage (some "
 267                 "ports may ignore this option)", type="float"),
 268         optparse.make_option("--results-directory",
 269             default="layout-test-results",
 270             help="Output results directory source dir, relative to Debug or "
 271                  "Release"),
 272         optparse.make_option("--new-baseline", action="store_true",
 273             default=False, help="Save all generated results as new baselines "
 274                  "into the platform directory, overwriting whatever's "
 275                  "already there."),
 276         optparse.make_option("--reset-results", action="store_true",
 277             default=False, help="Reset any existing baselines to the "
 278                  "generated results"),
 279         optparse.make_option("--no-show-results", action="store_false",
 280             default=True, dest="show_results",
 281             help="Don't launch a browser with results after the tests "
 282                  "are done"),
 283         # FIXME: We should have a helper function to do this sort of
 284         # deprectated mapping and automatically log, etc.
 285         optparse.make_option("--noshow-results", action="store_false",
 286             dest="show_results",
 287             help="Deprecated, same as --no-show-results."),
 288         optparse.make_option("--no-launch-safari", action="store_false",
 289             dest="show_results",
 290             help="old-run-webkit-tests compat, same as --noshow-results."),
 291         # old-run-webkit-tests:
 292         # --[no-]launch-safari    Launch (or do not launch) Safari to display
 293         #                         test results (default: launch)
 294         optparse.make_option("--full-results-html", action="store_true",
 295             default=False,
 296             help="Show all failures in results.html, rather than only "
 297                  "regressions"),
 298         optparse.make_option("--clobber-old-results", action="store_true",
 299             default=False, help="Clobbers test results from previous runs."),
 300         optparse.make_option("--platform",
 301             help="Override the platform for expected results"),
 302         optparse.make_option("--no-record-results", action="store_false",
 303             default=True, dest="record_results",
 304             help="Don't record the results."),
 305         # old-run-webkit-tests also has HTTP toggle options:
 306         # --[no-]http                     Run (or do not run) http tests
 307         #                                 (default: run)
 308     ]
 309
 310     test_options = [
 311         optparse.make_option("--build", dest="build",
 312             action="store_true", default=True,
 313             help="Check to ensure the DumpRenderTree build is up-to-date "
 314                  "(default)."),
 315         optparse.make_option("--no-build", dest="build",
 316             action="store_false", help="Don't check to see if the "
 317                                        "DumpRenderTree build is up-to-date."),
 318         optparse.make_option("-n", "--dry-run", action="store_true",
 319             default=False,
 320             help="Do everything but actually run the tests or upload results."),
 321         # old-run-webkit-tests has --valgrind instead of wrapper.
 322         optparse.make_option("--wrapper",
 323             help="wrapper command to insert before invocations of "
 324                  "DumpRenderTree; option is split on whitespace before "
 325                  "running. (Example: --wrapper='valgrind --smc-check=all')"),
 326         # old-run-webkit-tests:
 327         # -i|--ignore-tests               Comma-separated list of directories
 328         #                                 or tests to ignore
 329         optparse.make_option("--test-list", action="append",
 330             help="read list of tests to run from file", metavar="FILE"),
 331         # old-run-webkit-tests uses --skipped==[default|ignore|only]
 332         # instead of --force:
 333         optparse.make_option("--force", action="store_true", default=False,
 334             help="Run all tests, even those marked SKIP in the test list"),
 335         optparse.make_option("--use-apache", action="store_true",
 336             default=False, help="Whether to use apache instead of lighttpd."),
 337         optparse.make_option("--time-out-ms",
 338             help="Set the timeout for each test"),
 339         # old-run-webkit-tests calls --randomize-order --random:
 340         optparse.make_option("--randomize-order", action="store_true",
 341             default=False, help=("Run tests in random order (useful "
 342                                 "for tracking down corruption)")),
 343         optparse.make_option("--run-chunk",
 344             help=("Run a specified chunk (n:l), the nth of len l, "
 345                  "of the layout tests")),
 346         optparse.make_option("--run-part", help=("Run a specified part (n:m), "
 347                   "the nth of m parts, of the layout tests")),
 348         # old-run-webkit-tests calls --batch-size: --nthly n
 349         #   Restart DumpRenderTree every n tests (default: 1000)
 350         optparse.make_option("--batch-size",
 351             help=("Run a the tests in batches (n), after every n tests, "
 352                   "DumpRenderTree is relaunched."), type="int", default=0),
 353         # old-run-webkit-tests calls --run-singly: -1|--singly
 354         # Isolate each test case run (implies --nthly 1 --verbose)
 355         optparse.make_option("--run-singly", action="store_true",
 356             default=False, help="run a separate DumpRenderTree for each test"),
 357         optparse.make_option("--child-processes",
 358             help="Number of DumpRenderTrees to run in parallel."),
 359         # FIXME: Display default number of child processes that will run.
 360         optparse.make_option("--worker-model", action="store",
 361             default="old-threads", help=("controls worker model. Valid values "
 362             "are 'old-inline', 'old-threads'.")),
 363         optparse.make_option("--experimental-fully-parallel",
 364             action="store_true", default=False,
 365             help="run all tests in parallel"),
 366         optparse.make_option("--exit-after-n-failures", type="int", nargs=1,
 367             help="Exit after the first N failures instead of running all "
 368             "tests"),
 369         optparse.make_option("--exit-after-n-crashes-or-timeouts", type="int",
 370             nargs=1, help="Exit after the first N crashes instead of running "
 371             "all tests"),
 372         # FIXME: consider: --iterations n
 373         #      Number of times to run the set of tests (e.g. ABCABCABC)
 374         optparse.make_option("--print-last-failures", action="store_true",
 375             default=False, help="Print the tests in the last run that "
 376             "had unexpected failures (or passes) and then exit."),
 377         optparse.make_option("--retest-last-failures", action="store_true",
 378             default=False, help="re-test the tests in the last run that "
 379             "had unexpected failures (or passes)."),
 380         optparse.make_option("--retry-failures", action="store_true",
 381             default=True,
 382             help="Re-try any tests that produce unexpected results (default)"),
 383         optparse.make_option("--no-retry-failures", action="store_false",
 384             dest="retry_failures",
 385             help="Don't re-try any tests that produce unexpected results."),
 386     ]
 387
 388     misc_options = [
 389         optparse.make_option("--lint-test-files", action="store_true",
 390         default=False, help=("Makes sure the test files parse for all "
 391                             "configurations. Does not run any tests.")),
 392     ]
 393
 394     # FIXME: Move these into json_results_generator.py
 395     results_json_options = [
 396         optparse.make_option("--master-name", help="The name of the buildbot master."),
 397         optparse.make_option("--builder-name", default="DUMMY_BUILDER_NAME",
 398             help=("The name of the builder shown on the waterfall running "
 399                   "this script e.g. WebKit.")),
 400         optparse.make_option("--build-name", default="DUMMY_BUILD_NAME",
 401             help=("The name of the builder used in its path, e.g. "
 402                   "webkit-rel.")),
 403         optparse.make_option("--build-number", default="DUMMY_BUILD_NUMBER",
 404             help=("The build number of the builder running this script.")),
 405         optparse.make_option("--test-results-server", default="",
 406             help=("If specified, upload results json files to this appengine "
 407                   "server.")),
 408         optparse.make_option("--upload-full-results",
 409             action="store_true",
 410             default=False,
 411             help="If true, upload full json results to server."),
 412     ]
 413
 414     option_list = (configuration_options + print_options +
 415                    chromium_options + results_options + test_options +
 416                    misc_options + results_json_options +
 417                    old_run_webkit_tests_compat)
 418     option_parser = optparse.OptionParser(option_list=option_list)
 419
 420     return option_parser.parse_args(args)
 421
 422
 423 def main():
 424     options, args = parse_args()
 425     port_obj = port.get(options.platform, options)
 426     return run(port_obj, options, args)
 427
 428
 429 if '__main__' == __name__:
 430     try:
 431         sys.exit(main())
 432     except KeyboardInterrupt:
 433         # this mirrors what the shell normally does
 434         sys.exit(signal.SIGINT + 128)