Tools/Scripts/webkitpy/layout_tests/run_webkit_tests.py

   1 #!/usr/bin/env python
   2 # Copyright (C) 2010 Google Inc. All rights reserved.
   3 # Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Szeged
   4 #
   5 # Redistribution and use in source and binary forms, with or without
   6 # modification, are permitted provided that the following conditions are
   7 # met:
   8 #
   9 #     * Redistributions of source code must retain the above copyright
  10 # notice, this list of conditions and the following disclaimer.
  11 #     * Redistributions in binary form must reproduce the above
  12 # copyright notice, this list of conditions and the following disclaimer
  13 # in the documentation and/or other materials provided with the
  14 # distribution.
  15 #     * Neither the name of Google Inc. nor the names of its
  16 # contributors may be used to endorse or promote products derived from
  17 # this software without specific prior written permission.
  18 #
  19 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  20 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  21 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  22 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  23 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  24 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  25 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  26 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  27 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  28 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  29 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  30
  31 """Run layout tests."""
  32
  33 from __future__ import with_statement
  34
  35 import codecs
  36 import errno
  37 import logging
  38 import optparse
  39 import os
  40 import signal
  41 import sys
  42
  43 from layout_package import printing
  44 from layout_package import test_runner
  45
  46 from webkitpy.common.system import user
  47 from webkitpy.thirdparty import simplejson
  48
  49 import port
  50
  51 _log = logging.getLogger(__name__)
  52
  53
  54 def run(port, options, args, regular_output=sys.stderr,
  55         buildbot_output=sys.stdout):
  56     """Run the tests.
  57
  58     Args:
  59       port: Port object for port-specific behavior
  60       options: a dictionary of command line options
  61       args: a list of sub directories or files to test
  62       regular_output: a stream-like object that we can send logging/debug
  63           output to
  64       buildbot_output: a stream-like object that we can write all output that
  65           is intended to be parsed by the buildbot to
  66     Returns:
  67       the number of unexpected results that occurred, or -1 if there is an
  68           error.
  69
  70     """
  71     warnings = _set_up_derived_options(port, options)
  72
  73     printer = printing.Printer(port, options, regular_output, buildbot_output,
  74         int(options.child_processes), options.experimental_fully_parallel)
  75     for w in warnings:
  76         _log.warning(w)
  77
  78     if options.help_printing:
  79         printer.help_printing()
  80         printer.cleanup()
  81         return 0
  82
  83     last_unexpected_results = _gather_unexpected_results(options)
  84     if options.print_last_failures:
  85         printer.write("\n".join(last_unexpected_results) + "\n")
  86         printer.cleanup()
  87         return 0
  88
  89     # We wrap any parts of the run that are slow or likely to raise exceptions
  90     # in a try/finally to ensure that we clean up the logging configuration.
  91     num_unexpected_results = -1
  92     try:
  93         runner = test_runner.TestRunner(port, options, printer)
  94         runner._print_config()
  95
  96         printer.print_update("Collecting tests ...")
  97         try:
  98             runner.collect_tests(args, last_unexpected_results)
  99         except IOError, e:
 100             if e.errno == errno.ENOENT:
 101                 return -1
 102             raise
 103
 104         printer.print_update("Parsing expectations ...")
 105         if options.lint_test_files:
 106             return runner.lint()
 107         runner.parse_expectations(port.test_platform_name(),
 108                                   options.configuration == 'Debug')
 109
 110         printer.print_update("Checking build ...")
 111         if not port.check_build(runner.needs_http()):
 112             _log.error("Build check failed")
 113             return -1
 114
 115         result_summary = runner.set_up_run()
 116         if result_summary:
 117             num_unexpected_results = runner.run(result_summary)
 118             runner.clean_up_run()
 119             _log.debug("Testing completed, Exit status: %d" %
 120                        num_unexpected_results)
 121     finally:
 122         printer.cleanup()
 123
 124     return num_unexpected_results
 125
 126
 127 def _set_up_derived_options(port_obj, options):
 128     """Sets the options values that depend on other options values."""
 129     # We return a list of warnings to print after the printer is initialized.
 130     warnings = []
 131
 132     if options.worker_model == 'old-inline':
 133         if options.child_processes and int(options.child_processes) > 1:
 134             warnings.append("--worker-model=old-inline overrides --child-processes")
 135         options.child_processes = "1"
 136     if not options.child_processes:
 137         options.child_processes = os.environ.get("WEBKIT_TEST_CHILD_PROCESSES",
 138                                                  str(port_obj.default_child_processes()))
 139
 140     if not options.configuration:
 141         options.configuration = port_obj.default_configuration()
 142
 143     if options.pixel_tests is None:
 144         options.pixel_tests = True
 145
 146     if not options.use_apache:
 147         options.use_apache = sys.platform in ('darwin', 'linux2')
 148
 149     if not os.path.isabs(options.results_directory):
 150         # This normalizes the path to the build dir.
 151         # FIXME: how this happens is not at all obvious; this is a dumb
 152         # interface and should be cleaned up.
 153         options.results_directory = port_obj.results_directory()
 154
 155     if not options.time_out_ms:
 156         if options.configuration == "Debug":
 157             options.time_out_ms = str(2 * test_runner.TestRunner.DEFAULT_TEST_TIMEOUT_MS)
 158         else:
 159             options.time_out_ms = str(test_runner.TestRunner.DEFAULT_TEST_TIMEOUT_MS)
 160
 161     options.slow_time_out_ms = str(5 * int(options.time_out_ms))
 162     return warnings
 163
 164
 165 def _gather_unexpected_results(options):
 166     """Returns the unexpected results from the previous run, if any."""
 167     last_unexpected_results = []
 168     if options.print_last_failures or options.retest_last_failures:
 169         unexpected_results_filename = os.path.join(
 170         options.results_directory, "unexpected_results.json")
 171         with codecs.open(unexpected_results_filename, "r", "utf-8") as file:
 172             results = simplejson.load(file)
 173         last_unexpected_results = results['tests'].keys()
 174     return last_unexpected_results
 175
 176
 177 def _compat_shim_callback(option, opt_str, value, parser):
 178     print "Ignoring unsupported option: %s" % opt_str
 179
 180
 181 def _compat_shim_option(option_name, **kwargs):
 182     return optparse.make_option(option_name, action="callback",
 183         callback=_compat_shim_callback,
 184         help="Ignored, for old-run-webkit-tests compat only.", **kwargs)
 185
 186
 187 def parse_args(args=None):
 188     """Provides a default set of command line args.
 189
 190     Returns a tuple of options, args from optparse"""
 191
 192     # FIXME: All of these options should be stored closer to the code which
 193     # FIXME: actually uses them. configuration_options should move
 194     # FIXME: to WebKitPort and be shared across all scripts.
 195     configuration_options = [
 196         optparse.make_option("-t", "--target", dest="configuration",
 197                              help="(DEPRECATED)"),
 198         # FIXME: --help should display which configuration is default.
 199         optparse.make_option('--debug', action='store_const', const='Debug',
 200                              dest="configuration",
 201                              help='Set the configuration to Debug'),
 202         optparse.make_option('--release', action='store_const',
 203                              const='Release', dest="configuration",
 204                              help='Set the configuration to Release'),
 205         # old-run-webkit-tests also accepts -c, --configuration CONFIGURATION.
 206     ]
 207
 208     print_options = printing.print_options()
 209
 210     # FIXME: These options should move onto the ChromiumPort.
 211     chromium_options = [
 212         optparse.make_option("--chromium", action="store_true", default=False,
 213             help="use the Chromium port"),
 214         optparse.make_option("--startup-dialog", action="store_true",
 215             default=False, help="create a dialog on DumpRenderTree startup"),
 216         optparse.make_option("--gp-fault-error-box", action="store_true",
 217             default=False, help="enable Windows GP fault error box"),
 218         optparse.make_option("--js-flags",
 219             type="string", help="JavaScript flags to pass to tests"),
 220         optparse.make_option("--stress-opt", action="store_true",
 221             default=False,
 222             help="Enable additional stress test to JavaScript optimization"),
 223         optparse.make_option("--stress-deopt", action="store_true",
 224             default=False,
 225             help="Enable additional stress test to JavaScript optimization"),
 226         optparse.make_option("--nocheck-sys-deps", action="store_true",
 227             default=False,
 228             help="Don't check the system dependencies (themes)"),
 229         optparse.make_option("--use-test-shell", action="store_true",
 230             default=False,
 231             help="Use test_shell instead of DRT"),
 232         optparse.make_option("--accelerated-compositing",
 233             action="store_true",
 234             help="Use hardware-accelated compositing for rendering"),
 235         optparse.make_option("--no-accelerated-compositing",
 236             action="store_false",
 237             dest="accelerated_compositing",
 238             help="Don't use hardware-accelerated compositing for rendering"),
 239         optparse.make_option("--accelerated-2d-canvas",
 240             action="store_true",
 241             help="Use hardware-accelerated 2D Canvas calls"),
 242         optparse.make_option("--no-accelerated-2d-canvas",
 243             action="store_false",
 244             dest="accelerated_2d_canvas",
 245             help="Don't use hardware-accelerated 2D Canvas calls"),
 246         optparse.make_option("--enable-hardware-gpu",
 247             action="store_true",
 248             default=False,
 249             help="Run graphics tests on real GPU hardware vs software"),
 250     ]
 251
 252     # Missing Mac-specific old-run-webkit-tests options:
 253     # FIXME: Need: -g, --guard for guard malloc support on Mac.
 254     # FIXME: Need: -l --leaks    Enable leaks checking.
 255     # FIXME: Need: --sample-on-timeout Run sample on timeout
 256
 257     old_run_webkit_tests_compat = [
 258         # NRWT doesn't generate results by default anyway.
 259         _compat_shim_option("--no-new-test-results"),
 260         # NRWT doesn't sample on timeout yet anyway.
 261         _compat_shim_option("--no-sample-on-timeout"),
 262         # FIXME: NRWT needs to support remote links eventually.
 263         _compat_shim_option("--use-remote-links-to-tests"),
 264     ]
 265
 266     results_options = [
 267         # NEED for bots: --use-remote-links-to-tests Link to test files
 268         # within the SVN repository in the results.
 269         optparse.make_option("-p", "--pixel-tests", action="store_true",
 270             dest="pixel_tests", help="Enable pixel-to-pixel PNG comparisons"),
 271         optparse.make_option("--no-pixel-tests", action="store_false",
 272             dest="pixel_tests", help="Disable pixel-to-pixel PNG comparisons"),
 273         optparse.make_option("--tolerance",
 274             help="Ignore image differences less than this percentage (some "
 275                 "ports may ignore this option)", type="float"),
 276         optparse.make_option("--results-directory",
 277             default="layout-test-results",
 278             help="Output results directory source dir, relative to Debug or "
 279                  "Release"),
 280         optparse.make_option("--build-directory",
 281             help="Path to the directory under which build files are kept (should not include configuration)"),
 282         optparse.make_option("--new-baseline", action="store_true",
 283             default=False, help="Save all generated results as new baselines "
 284                  "into the platform directory, overwriting whatever's "
 285                  "already there."),
 286         optparse.make_option("--reset-results", action="store_true",
 287             default=False, help="Reset any existing baselines to the "
 288                  "generated results"),
 289         optparse.make_option("--no-show-results", action="store_false",
 290             default=True, dest="show_results",
 291             help="Don't launch a browser with results after the tests "
 292                  "are done"),
 293         # FIXME: We should have a helper function to do this sort of
 294         # deprectated mapping and automatically log, etc.
 295         optparse.make_option("--noshow-results", action="store_false",
 296             dest="show_results",
 297             help="Deprecated, same as --no-show-results."),
 298         optparse.make_option("--no-launch-safari", action="store_false",
 299             dest="show_results",
 300             help="old-run-webkit-tests compat, same as --noshow-results."),
 301         # old-run-webkit-tests:
 302         # --[no-]launch-safari    Launch (or do not launch) Safari to display
 303         #                         test results (default: launch)
 304         optparse.make_option("--full-results-html", action="store_true",
 305             default=False,
 306             help="Show all failures in results.html, rather than only "
 307                  "regressions"),
 308         optparse.make_option("--clobber-old-results", action="store_true",
 309             default=False, help="Clobbers test results from previous runs."),
 310         optparse.make_option("--platform",
 311             help="Override the platform for expected results"),
 312         optparse.make_option("--no-record-results", action="store_false",
 313             default=True, dest="record_results",
 314             help="Don't record the results."),
 315         # old-run-webkit-tests also has HTTP toggle options:
 316         # --[no-]http                     Run (or do not run) http tests
 317         #                                 (default: run)
 318     ]
 319
 320     test_options = [
 321         optparse.make_option("--build", dest="build",
 322             action="store_true", default=True,
 323             help="Check to ensure the DumpRenderTree build is up-to-date "
 324                  "(default)."),
 325         optparse.make_option("--no-build", dest="build",
 326             action="store_false", help="Don't check to see if the "
 327                                        "DumpRenderTree build is up-to-date."),
 328         optparse.make_option("-n", "--dry-run", action="store_true",
 329             default=False,
 330             help="Do everything but actually run the tests or upload results."),
 331         # old-run-webkit-tests has --valgrind instead of wrapper.
 332         optparse.make_option("--wrapper",
 333             help="wrapper command to insert before invocations of "
 334                  "DumpRenderTree; option is split on whitespace before "
 335                  "running. (Example: --wrapper='valgrind --smc-check=all')"),
 336         # old-run-webkit-tests:
 337         # -i|--ignore-tests               Comma-separated list of directories
 338         #                                 or tests to ignore
 339         optparse.make_option("--test-list", action="append",
 340             help="read list of tests to run from file", metavar="FILE"),
 341         # old-run-webkit-tests uses --skipped==[default|ignore|only]
 342         # instead of --force:
 343         optparse.make_option("--force", action="store_true", default=False,
 344             help="Run all tests, even those marked SKIP in the test list"),
 345         optparse.make_option("--use-apache", action="store_true",
 346             default=False, help="Whether to use apache instead of lighttpd."),
 347         optparse.make_option("--time-out-ms",
 348             help="Set the timeout for each test"),
 349         # old-run-webkit-tests calls --randomize-order --random:
 350         optparse.make_option("--randomize-order", action="store_true",
 351             default=False, help=("Run tests in random order (useful "
 352                                 "for tracking down corruption)")),
 353         optparse.make_option("--run-chunk",
 354             help=("Run a specified chunk (n:l), the nth of len l, "
 355                  "of the layout tests")),
 356         optparse.make_option("--run-part", help=("Run a specified part (n:m), "
 357                   "the nth of m parts, of the layout tests")),
 358         # old-run-webkit-tests calls --batch-size: --nthly n
 359         #   Restart DumpRenderTree every n tests (default: 1000)
 360         optparse.make_option("--batch-size",
 361             help=("Run a the tests in batches (n), after every n tests, "
 362                   "DumpRenderTree is relaunched."), type="int", default=0),
 363         # old-run-webkit-tests calls --run-singly: -1|--singly
 364         # Isolate each test case run (implies --nthly 1 --verbose)
 365         optparse.make_option("--run-singly", action="store_true",
 366             default=False, help="run a separate DumpRenderTree for each test"),
 367         optparse.make_option("--child-processes",
 368             help="Number of DumpRenderTrees to run in parallel."),
 369         # FIXME: Display default number of child processes that will run.
 370         optparse.make_option("--worker-model", action="store",
 371             default="old-threads", help=("controls worker model. Valid values "
 372             "are 'old-inline', 'old-threads'.")),
 373         optparse.make_option("--experimental-fully-parallel",
 374             action="store_true", default=False,
 375             help="run all tests in parallel"),
 376         optparse.make_option("--exit-after-n-failures", type="int", nargs=1,
 377             help="Exit after the first N failures instead of running all "
 378             "tests"),
 379         optparse.make_option("--exit-after-n-crashes-or-timeouts", type="int",
 380             nargs=1, help="Exit after the first N crashes instead of running "
 381             "all tests"),
 382         # FIXME: consider: --iterations n
 383         #      Number of times to run the set of tests (e.g. ABCABCABC)
 384         optparse.make_option("--print-last-failures", action="store_true",
 385             default=False, help="Print the tests in the last run that "
 386             "had unexpected failures (or passes) and then exit."),
 387         optparse.make_option("--retest-last-failures", action="store_true",
 388             default=False, help="re-test the tests in the last run that "
 389             "had unexpected failures (or passes)."),
 390         optparse.make_option("--retry-failures", action="store_true",
 391             default=True,
 392             help="Re-try any tests that produce unexpected results (default)"),
 393         optparse.make_option("--no-retry-failures", action="store_false",
 394             dest="retry_failures",
 395             help="Don't re-try any tests that produce unexpected results."),
 396     ]
 397
 398     misc_options = [
 399         optparse.make_option("--lint-test-files", action="store_true",
 400         default=False, help=("Makes sure the test files parse for all "
 401                             "configurations. Does not run any tests.")),
 402     ]
 403
 404     # FIXME: Move these into json_results_generator.py
 405     results_json_options = [
 406         optparse.make_option("--master-name", help="The name of the buildbot master."),
 407         optparse.make_option("--builder-name", default="DUMMY_BUILDER_NAME",
 408             help=("The name of the builder shown on the waterfall running "
 409                   "this script e.g. WebKit.")),
 410         optparse.make_option("--build-name", default="DUMMY_BUILD_NAME",
 411             help=("The name of the builder used in its path, e.g. "
 412                   "webkit-rel.")),
 413         optparse.make_option("--build-number", default="DUMMY_BUILD_NUMBER",
 414             help=("The build number of the builder running this script.")),
 415         optparse.make_option("--test-results-server", default="",
 416             help=("If specified, upload results json files to this appengine "
 417                   "server.")),
 418         optparse.make_option("--upload-full-results",
 419             action="store_true",
 420             default=False,
 421             help="If true, upload full json results to server."),
 422     ]
 423
 424     option_list = (configuration_options + print_options +
 425                    chromium_options + results_options + test_options +
 426                    misc_options + results_json_options +
 427                    old_run_webkit_tests_compat)
 428     option_parser = optparse.OptionParser(option_list=option_list)
 429
 430     return option_parser.parse_args(args)
 431
 432
 433 def main():
 434     options, args = parse_args()
 435     port_obj = port.get(options.platform, options)
 436     return run(port_obj, options, args)
 437
 438
 439 if '__main__' == __name__:
 440     try:
 441         sys.exit(main())
 442     except KeyboardInterrupt:
 443         # this mirrors what the shell normally does
 444         sys.exit(signal.SIGINT + 128)