Tools/Scripts/webkitpy/layout_tests/run_webkit_tests.py

   1 #!/usr/bin/env python
   2 # Copyright (C) 2010 Google Inc. All rights reserved.
   3 # Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Szeged
   4 # Copyright (C) 2011 Apple Inc. All rights reserved.
   5 #
   6 # Redistribution and use in source and binary forms, with or without
   7 # modification, are permitted provided that the following conditions are
   8 # met:
   9 #
  10 #     * Redistributions of source code must retain the above copyright
  11 # notice, this list of conditions and the following disclaimer.
  12 #     * Redistributions in binary form must reproduce the above
  13 # copyright notice, this list of conditions and the following disclaimer
  14 # in the documentation and/or other materials provided with the
  15 # distribution.
  16 #     * Neither the name of Google Inc. nor the names of its
  17 # contributors may be used to endorse or promote products derived from
  18 # this software without specific prior written permission.
  19 #
  20 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  21 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  22 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  23 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  24 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  25 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  26 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  27 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  28 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  29 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  30 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  31
  32 import errno
  33 import logging
  34 import optparse
  35 import os
  36 import signal
  37 import sys
  38
  39 from webkitpy.common.host import Host
  40 from webkitpy.layout_tests.controllers.manager import Manager, WorkerException
  41 from webkitpy.layout_tests.models import test_expectations
  42 from webkitpy.layout_tests.views import printing
  43
  44
  45 _log = logging.getLogger(__name__)
  46
  47
  48 def lint(port, options, expectations_class):
  49     host = port.host
  50     if options.platform:
  51         ports_to_lint = [port]
  52     else:
  53         ports_to_lint = [host.port_factory.get(name) for name in host.port_factory.all_port_names()]
  54
  55     files_linted = set()
  56     lint_failed = False
  57
  58     for port_to_lint in ports_to_lint:
  59         expectations_file = port_to_lint.path_to_test_expectations_file()
  60         if expectations_file in files_linted:
  61             continue
  62
  63         try:
  64             expectations_class(port_to_lint,
  65                 tests=None,
  66                 expectations=port_to_lint.test_expectations(),
  67                 test_config=port_to_lint.test_configuration(),
  68                 is_lint_mode=True,
  69                 overrides=port_to_lint.test_expectations_overrides())
  70         except test_expectations.ParseError, e:
  71             lint_failed = True
  72             _log.error('')
  73             for warning in e.warnings:
  74                 _log.error(warning)
  75             _log.error('')
  76         files_linted.add(expectations_file)
  77
  78     if lint_failed:
  79         _log.error('Lint failed.')
  80         return -1
  81     _log.info('Lint succeeded.')
  82     return 0
  83
  84
  85 def run(port, options, args, regular_output=sys.stderr, buildbot_output=sys.stdout):
  86     warnings = _set_up_derived_options(port, options)
  87
  88     printer = printing.Printer(port, options, regular_output, buildbot_output, logger=logging.getLogger())
  89
  90     for warning in warnings:
  91         _log.warning(warning)
  92
  93     if options.help_printing:
  94         printer.help_printing()
  95         printer.cleanup()
  96         return 0
  97
  98     if options.lint_test_files:
  99         return lint(port, options, test_expectations.TestExpectations)
 100
 101     # We wrap any parts of the run that are slow or likely to raise exceptions
 102     # in a try/finally to ensure that we clean up the logging configuration.
 103     unexpected_result_count = -1
 104     try:
 105         manager = Manager(port, options, printer)
 106         manager.print_config()
 107
 108         printer.print_update("Collecting tests ...")
 109         try:
 110             manager.collect_tests(args)
 111         except IOError, e:
 112             if e.errno == errno.ENOENT:
 113                 return -1
 114             raise
 115
 116         printer.print_update("Checking build ...")
 117         if not port.check_build(manager.needs_servers()):
 118             _log.error("Build check failed")
 119             return -1
 120
 121         printer.print_update("Parsing expectations ...")
 122         manager.parse_expectations()
 123
 124         unexpected_result_count = manager.run()
 125         _log.debug("Testing completed, Exit status: %d" % unexpected_result_count)
 126     finally:
 127         printer.cleanup()
 128
 129     return unexpected_result_count
 130
 131
 132 def _set_up_derived_options(port, options):
 133     """Sets the options values that depend on other options values."""
 134     # We return a list of warnings to print after the printer is initialized.
 135     warnings = []
 136
 137     if options.worker_model is None:
 138         options.worker_model = port.default_worker_model()
 139
 140     if options.worker_model == 'inline':
 141         if options.child_processes and int(options.child_processes) > 1:
 142             warnings.append("--worker-model=inline overrides --child-processes")
 143         options.child_processes = "1"
 144     if not options.child_processes:
 145         options.child_processes = os.environ.get("WEBKIT_TEST_CHILD_PROCESSES",
 146                                                  str(port.default_child_processes()))
 147
 148     if not options.configuration:
 149         options.configuration = port.default_configuration()
 150
 151     if options.pixel_tests is None:
 152         options.pixel_tests = True
 153
 154     if not options.time_out_ms:
 155         if options.configuration == "Debug":
 156             options.time_out_ms = str(2 * port.default_test_timeout_ms())
 157         else:
 158             options.time_out_ms = str(port.default_test_timeout_ms())
 159
 160     options.slow_time_out_ms = str(5 * int(options.time_out_ms))
 161
 162     if options.additional_platform_directory:
 163         normalized_platform_directories = []
 164         for path in options.additional_platform_directory:
 165             if not port.host.filesystem.isabs(path):
 166                 warnings.append("--additional-platform-directory=%s is ignored since it is not absolute" % path)
 167                 continue
 168             normalized_platform_directories.append(port.host.filesystem.normpath(path))
 169         options.additional_platform_directory = normalized_platform_directories
 170
 171     if not options.http and options.force:
 172         warnings.append("--no-http is ignored since --force is also provided")
 173         options.http = True
 174
 175     if options.skip_pixel_test_if_no_baseline and not options.pixel_tests:
 176         warnings.append("--skip-pixel-test-if-no-baseline is only supported with -p (--pixel-tests)")
 177
 178     if options.ignore_metrics and (options.new_baseline or options.reset_results):
 179         warnings.append("--ignore-metrics has no effect with --new-baselines or with --reset-results")
 180
 181     return warnings
 182
 183
 184 def _compat_shim_callback(option, opt_str, value, parser):
 185     print "Ignoring unsupported option: %s" % opt_str
 186
 187
 188 def _compat_shim_option(option_name, **kwargs):
 189     return optparse.make_option(option_name, action="callback",
 190         callback=_compat_shim_callback,
 191         help="Ignored, for old-run-webkit-tests compat only.", **kwargs)
 192
 193
 194 def parse_args(args=None):
 195     """Provides a default set of command line args.
 196
 197     Returns a tuple of options, args from optparse"""
 198
 199     option_group_definitions = []
 200
 201     # FIXME: All of these options should be stored closer to the code which
 202     # FIXME: actually uses them. configuration_options should move
 203     # FIXME: to WebKitPort and be shared across all scripts.
 204     option_group_definitions.append(("Configuration Options", [
 205         optparse.make_option("-t", "--target", dest="configuration",
 206                              help="(DEPRECATED)"),
 207         # FIXME: --help should display which configuration is default.
 208         optparse.make_option('--debug', action='store_const', const='Debug',
 209                              dest="configuration",
 210                              help='Set the configuration to Debug'),
 211         optparse.make_option('--release', action='store_const',
 212                              const='Release', dest="configuration",
 213                              help='Set the configuration to Release'),
 214         # old-run-webkit-tests also accepts -c, --configuration CONFIGURATION.
 215         optparse.make_option("--platform", help="Override port/platform being tested (i.e. chromium-mac)"),
 216         optparse.make_option("--chromium", action="store_const", const='chromium', dest='platform', help='Alias for --platform=chromium'),
 217         optparse.make_option('--efl', action='store_const', const='efl', dest="platform", help='Alias for --platform=efl'),
 218         optparse.make_option('--gtk', action='store_const', const='gtk', dest="platform", help='Alias for --platform=gtk'),
 219         optparse.make_option('--qt', action='store_const', const='qt', dest="platform", help='Alias for --platform=qt'),
 220     ]))
 221
 222     option_group_definitions.append(("Printing Options", printing.print_options()))
 223
 224     # FIXME: These options should move onto the ChromiumPort.
 225     option_group_definitions.append(("Chromium-specific Options", [
 226         optparse.make_option("--startup-dialog", action="store_true",
 227             default=False, help="create a dialog on DumpRenderTree startup"),
 228         optparse.make_option("--gp-fault-error-box", action="store_true",
 229             default=False, help="enable Windows GP fault error box"),
 230         optparse.make_option("--js-flags",
 231             type="string", help="JavaScript flags to pass to tests"),
 232         optparse.make_option("--stress-opt", action="store_true",
 233             default=False,
 234             help="Enable additional stress test to JavaScript optimization"),
 235         optparse.make_option("--stress-deopt", action="store_true",
 236             default=False,
 237             help="Enable additional stress test to JavaScript optimization"),
 238         optparse.make_option("--nocheck-sys-deps", action="store_true",
 239             default=False,
 240             help="Don't check the system dependencies (themes)"),
 241         optparse.make_option("--accelerated-video",
 242             action="store_true",
 243             help="Use hardware-accelerated compositing for video"),
 244         optparse.make_option("--no-accelerated-video",
 245             action="store_false",
 246             dest="accelerated_video",
 247             help="Don't use hardware-accelerated compositing for video"),
 248         optparse.make_option("--threaded-compositing",
 249             action="store_true",
 250             help="Use threaded compositing for rendering"),
 251         optparse.make_option("--accelerated-2d-canvas",
 252             action="store_true",
 253             help="Use hardware-accelerated 2D Canvas calls"),
 254         optparse.make_option("--no-accelerated-2d-canvas",
 255             action="store_false",
 256             dest="accelerated_2d_canvas",
 257             help="Don't use hardware-accelerated 2D Canvas calls"),
 258         optparse.make_option("--accelerated-painting",
 259             action="store_true",
 260             default=False,
 261             help="Use hardware accelerated painting of composited pages"),
 262         optparse.make_option("--enable-hardware-gpu",
 263             action="store_true",
 264             default=False,
 265             help="Run graphics tests on real GPU hardware vs software"),
 266         optparse.make_option("--per-tile-painting",
 267             action="store_true",
 268             help="Use per-tile painting of composited pages"),
 269         optparse.make_option("--adb-args", type="string",
 270             help="Arguments parsed to Android adb, to select device, etc."),
 271     ]))
 272
 273     option_group_definitions.append(("WebKit Options", [
 274         optparse.make_option("--gc-between-tests", action="store_true", default=False,
 275             help="Force garbage collection between each test"),
 276         optparse.make_option("--complex-text", action="store_true", default=False,
 277             help="Use the complex text code path for all text (Mac OS X and Windows only)"),
 278         optparse.make_option("-l", "--leaks", action="store_true", default=False,
 279             help="Enable leaks checking (Mac OS X only)"),
 280         optparse.make_option("-g", "--guard-malloc", action="store_true", default=False,
 281             help="Enable malloc guard (Mac OS X only)"),
 282         optparse.make_option("--threaded", action="store_true", default=False,
 283             help="Run a concurrent JavaScript thread with each test"),
 284         optparse.make_option("--webkit-test-runner", "-2", action="store_true",
 285             help="Use WebKitTestRunner rather than DumpRenderTree."),
 286         optparse.make_option("--root", action="store",
 287             help="Path to a pre-built root of WebKit (for running tests using a nightly build of WebKit)"),
 288     ]))
 289
 290     option_group_definitions.append(("ORWT Compatibility Options", [
 291         # FIXME: Remove this option once the bots don't refer to it.
 292         # results.html is smart enough to figure this out itself.
 293         _compat_shim_option("--use-remote-links-to-tests"),
 294     ]))
 295
 296     option_group_definitions.append(("Results Options", [
 297         optparse.make_option("-p", "--pixel-tests", action="store_true",
 298             dest="pixel_tests", help="Enable pixel-to-pixel PNG comparisons"),
 299         optparse.make_option("--no-pixel-tests", action="store_false",
 300             dest="pixel_tests", help="Disable pixel-to-pixel PNG comparisons"),
 301         optparse.make_option("--no-sample-on-timeout", action="store_false",
 302             dest="sample_on_timeout", help="Don't run sample on timeout (Mac OS X only)"),
 303         optparse.make_option("--no-ref-tests", action="store_true",
 304             dest="no_ref_tests", help="Skip all ref tests"),
 305         optparse.make_option("--tolerance",
 306             help="Ignore image differences less than this percentage (some "
 307                 "ports may ignore this option)", type="float"),
 308         optparse.make_option("--results-directory", help="Location of test results"),
 309         optparse.make_option("--build-directory",
 310             help="Path to the directory under which build files are kept (should not include configuration)"),
 311         optparse.make_option("--new-baseline", action="store_true",
 312             default=False, help="Save generated results as new baselines "
 313                  "into the *platform* directory, overwriting whatever's "
 314                  "already there."),
 315         optparse.make_option("--reset-results", action="store_true",
 316             default=False, help="Reset expectations to the "
 317                  "generated results in their existing location."),
 318         optparse.make_option("--no-new-test-results", action="store_false",
 319             dest="new_test_results", default=True,
 320             help="Don't create new baselines when no expected results exist"),
 321         optparse.make_option("--skip-pixel-test-if-no-baseline", action="store_true",
 322             dest="skip_pixel_test_if_no_baseline", help="Do not generate and check pixel result in the case when "
 323                  "no image baseline is available for the test."),
 324         optparse.make_option("--skip-failing-tests", action="store_true",
 325             default=False, help="Skip tests that are expected to fail. "
 326                  "Note: When using this option, you might miss new crashes "
 327                  "in these tests."),
 328         optparse.make_option("--additional-drt-flag", action="append",
 329             default=[], help="Additional command line flag to pass to DumpRenderTree "
 330                  "Specify multiple times to add multiple flags."),
 331         optparse.make_option("--additional-platform-directory", action="append",
 332             default=[], help="Additional directory where to look for test "
 333                  "baselines (will take precendence over platform baselines). "
 334                  "Specify multiple times to add multiple search path entries."),
 335         optparse.make_option("--no-show-results", action="store_false",
 336             default=True, dest="show_results",
 337             help="Don't launch a browser with results after the tests "
 338                  "are done"),
 339         # FIXME: We should have a helper function to do this sort of
 340         # deprectated mapping and automatically log, etc.
 341         optparse.make_option("--noshow-results", action="store_false", dest="show_results", help="Deprecated, same as --no-show-results."),
 342         optparse.make_option("--no-launch-safari", action="store_false", dest="show_results", help="Deprecated, same as --no-show-results."),
 343         optparse.make_option("--full-results-html", action="store_true",
 344             default=False,
 345             help="Show all failures in results.html, rather than only regressions"),
 346         optparse.make_option("--clobber-old-results", action="store_true",
 347             default=False, help="Clobbers test results from previous runs."),
 348         optparse.make_option("--no-record-results", action="store_false",
 349             default=True, dest="record_results",
 350             help="Don't record the results."),
 351         optparse.make_option("--http", action="store_true", dest="http",
 352             default=True, help="Run HTTP and WebSocket tests (default)"),
 353         optparse.make_option("--no-http", action="store_false", dest="http",
 354             help="Don't run HTTP and WebSocket tests"),
 355         optparse.make_option("--ignore-metrics", action="store_true", dest="ignore_metrics",
 356             default=False, help="Ignore rendering metrics related information from test "
 357             "output, only compare the structure of the rendertree."),
 358     ]))
 359
 360     option_group_definitions.append(("Testing Options", [
 361         optparse.make_option("--build", dest="build",
 362             action="store_true", default=True,
 363             help="Check to ensure the DumpRenderTree build is up-to-date "
 364                  "(default)."),
 365         optparse.make_option("--no-build", dest="build",
 366             action="store_false", help="Don't check to see if the "
 367                                        "DumpRenderTree build is up-to-date."),
 368         optparse.make_option("-n", "--dry-run", action="store_true",
 369             default=False,
 370             help="Do everything but actually run the tests or upload results."),
 371         # old-run-webkit-tests has --valgrind instead of wrapper.
 372         optparse.make_option("--wrapper",
 373             help="wrapper command to insert before invocations of "
 374                  "DumpRenderTree; option is split on whitespace before "
 375                  "running. (Example: --wrapper='valgrind --smc-check=all')"),
 376         # old-run-webkit-tests:
 377         # -i|--ignore-tests               Comma-separated list of directories
 378         #                                 or tests to ignore
 379         optparse.make_option("-i", "--ignore-tests", action="append", default=[],
 380             help="directories or test to ignore (may specify multiple times)"),
 381         optparse.make_option("--test-list", action="append",
 382             help="read list of tests to run from file", metavar="FILE"),
 383         # old-run-webkit-tests uses --skipped==[default|ignore|only]
 384         # instead of --force:
 385         optparse.make_option("--force", action="store_true", default=False,
 386             help="Run all tests, even those marked SKIP in the test list"),
 387         optparse.make_option("--time-out-ms",
 388             help="Set the timeout for each test"),
 389         # old-run-webkit-tests calls --randomize-order --random:
 390         optparse.make_option("--randomize-order", action="store_true",
 391             default=False, help=("Run tests in random order (useful "
 392                                 "for tracking down corruption)")),
 393         optparse.make_option("--run-chunk",
 394             help=("Run a specified chunk (n:l), the nth of len l, "
 395                  "of the layout tests")),
 396         optparse.make_option("--run-part", help=("Run a specified part (n:m), "
 397                   "the nth of m parts, of the layout tests")),
 398         # old-run-webkit-tests calls --batch-size: --nthly n
 399         #   Restart DumpRenderTree every n tests (default: 1000)
 400         optparse.make_option("--batch-size",
 401             help=("Run a the tests in batches (n), after every n tests, "
 402                   "DumpRenderTree is relaunched."), type="int", default=None),
 403         # old-run-webkit-tests calls --run-singly: -1|--singly
 404         # Isolate each test case run (implies --nthly 1 --verbose)
 405         optparse.make_option("--run-singly", action="store_true",
 406             default=False, help="run a separate DumpRenderTree for each test"),
 407         optparse.make_option("--child-processes",
 408             help="Number of DumpRenderTrees to run in parallel."),
 409         # FIXME: Display default number of child processes that will run.
 410         optparse.make_option("--worker-model", action="store",
 411             default=None, help=("controls worker model. Valid values are "
 412                                 "'inline' and 'processes'.")),
 413         optparse.make_option("-f", "--experimental-fully-parallel",
 414             action="store_true",
 415             help="run all tests in parallel"),
 416         optparse.make_option("--no-experimental-fully-parallel",
 417             action="store_false",
 418             dest="experimental_fully_parallel",
 419             help="do not run all tests in parallel"),
 420         optparse.make_option("--exit-after-n-failures", type="int", default=500,
 421             help="Exit after the first N failures instead of running all "
 422             "tests"),
 423         optparse.make_option("--exit-after-n-crashes-or-timeouts", type="int",
 424             default=20, help="Exit after the first N crashes instead of "
 425             "running all tests"),
 426         optparse.make_option("--iterations", type="int", help="Number of times to run the set of tests (e.g. ABCABCABC)"),
 427         optparse.make_option("--repeat-each", type="int", help="Number of times to run each test (e.g. AAABBBCCC)"),
 428         optparse.make_option("--retry-failures", action="store_true",
 429             default=True,
 430             help="Re-try any tests that produce unexpected results (default)"),
 431         optparse.make_option("--no-retry-failures", action="store_false",
 432             dest="retry_failures",
 433             help="Don't re-try any tests that produce unexpected results."),
 434         optparse.make_option("--max-locked-shards", type="int",
 435             help="Set the maximum number of locked shards"),
 436     ]))
 437
 438     option_group_definitions.append(("Miscellaneous Options", [
 439         optparse.make_option("--lint-test-files", action="store_true",
 440         default=False, help=("Makes sure the test files parse for all "
 441                             "configurations. Does not run any tests.")),
 442     ]))
 443
 444     # FIXME: Move these into json_results_generator.py
 445     option_group_definitions.append(("Result JSON Options", [
 446         optparse.make_option("--master-name", help="The name of the buildbot master."),
 447         optparse.make_option("--builder-name", default="",
 448             help=("The name of the builder shown on the waterfall running "
 449                   "this script e.g. WebKit.")),
 450         optparse.make_option("--build-name", default="DUMMY_BUILD_NAME",
 451             help=("The name of the builder used in its path, e.g. "
 452                   "webkit-rel.")),
 453         optparse.make_option("--build-number", default="DUMMY_BUILD_NUMBER",
 454             help=("The build number of the builder running this script.")),
 455         optparse.make_option("--test-results-server", default="",
 456             help=("If specified, upload results json files to this appengine "
 457                   "server.")),
 458     ]))
 459
 460     option_parser = optparse.OptionParser()
 461
 462     for group_name, group_options in option_group_definitions:
 463         option_group = optparse.OptionGroup(option_parser, group_name)
 464         option_group.add_options(group_options)
 465         option_parser.add_option_group(option_group)
 466
 467     return option_parser.parse_args(args)
 468
 469
 470 def main():
 471     options, args = parse_args()
 472     if options.platform and 'test' in options.platform:
 473         # It's a bit lame to import mocks into real code, but this allows the user
 474         # to run tests against the test platform interactively, which is useful for
 475         # debugging test failures.
 476         from webkitpy.common.host_mock import MockHost
 477         host = MockHost()
 478     else:
 479         host = Host()
 480     host._initialize_scm()
 481     port = host.port_factory.get(options.platform, options)
 482     logging.getLogger().setLevel(logging.DEBUG if options.verbose else logging.INFO)
 483     return run(port, options, args)
 484
 485
 486 if '__main__' == __name__:
 487     try:
 488         sys.exit(main())
 489     except KeyboardInterrupt:
 490         # This mirrors what the shell normally does.
 491         INTERRUPTED_EXIT_STATUS = signal.SIGINT + 128
 492         sys.exit(INTERRUPTED_EXIT_STATUS)
 493     except WorkerException:
 494         # This is a randomly chosen exit code that can be tested against to
 495         # indicate that an unexpected exception occurred.
 496         EXCEPTIONAL_EXIT_STATUS = 254
 497         sys.exit(EXCEPTIONAL_EXIT_STATUS)