2010-12-22 Dirk Pranke <dpranke@chromium.org>
[WebKit.git] / Tools / Scripts / webkitpy / layout_tests / run_webkit_tests.py
1 #!/usr/bin/env python
2 # Copyright (C) 2010 Google Inc. All rights reserved.
3 # Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Szeged
4 #
5 # Redistribution and use in source and binary forms, with or without
6 # modification, are permitted provided that the following conditions are
7 # met:
8 #
9 #     * Redistributions of source code must retain the above copyright
10 # notice, this list of conditions and the following disclaimer.
11 #     * Redistributions in binary form must reproduce the above
12 # copyright notice, this list of conditions and the following disclaimer
13 # in the documentation and/or other materials provided with the
14 # distribution.
15 #     * Neither the name of Google Inc. nor the names of its
16 # contributors may be used to endorse or promote products derived from
17 # this software without specific prior written permission.
18 #
19 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30
31 """Run layout tests."""
32
33 from __future__ import with_statement
34
35 import codecs
36 import errno
37 import logging
38 import optparse
39 import os
40 import signal
41 import sys
42
43 from layout_package import printing
44 from layout_package import test_runner
45
46 from webkitpy.common.system import user
47 from webkitpy.thirdparty import simplejson
48
49 import port
50
51 _log = logging.getLogger(__name__)
52
53
54 def run(port, options, args, regular_output=sys.stderr,
55         buildbot_output=sys.stdout):
56     """Run the tests.
57
58     Args:
59       port: Port object for port-specific behavior
60       options: a dictionary of command line options
61       args: a list of sub directories or files to test
62       regular_output: a stream-like object that we can send logging/debug
63           output to
64       buildbot_output: a stream-like object that we can write all output that
65           is intended to be parsed by the buildbot to
66     Returns:
67       the number of unexpected results that occurred, or -1 if there is an
68           error.
69
70     """
71     warnings = _set_up_derived_options(port, options)
72
73     printer = printing.Printer(port, options, regular_output, buildbot_output,
74         int(options.child_processes), options.experimental_fully_parallel)
75     for w in warnings:
76         _log.warning(w)
77
78     if options.help_printing:
79         printer.help_printing()
80         printer.cleanup()
81         return 0
82
83     last_unexpected_results = _gather_unexpected_results(options)
84     if options.print_last_failures:
85         printer.write("\n".join(last_unexpected_results) + "\n")
86         printer.cleanup()
87         return 0
88
89     # We wrap any parts of the run that are slow or likely to raise exceptions
90     # in a try/finally to ensure that we clean up the logging configuration.
91     num_unexpected_results = -1
92     try:
93         runner = test_runner.TestRunner(port, options, printer)
94         runner._print_config()
95
96         printer.print_update("Collecting tests ...")
97         try:
98             runner.collect_tests(args, last_unexpected_results)
99         except IOError, e:
100             if e.errno == errno.ENOENT:
101                 return -1
102             raise
103
104         printer.print_update("Parsing expectations ...")
105         if options.lint_test_files:
106             return runner.lint()
107         runner.parse_expectations(port.test_platform_name(),
108                                   options.configuration == 'Debug')
109
110         printer.print_update("Checking build ...")
111         if not port.check_build(runner.needs_http()):
112             _log.error("Build check failed")
113             return -1
114
115         result_summary = runner.set_up_run()
116         if result_summary:
117             num_unexpected_results = runner.run(result_summary)
118             runner.clean_up_run()
119             _log.debug("Testing completed, Exit status: %d" %
120                        num_unexpected_results)
121     finally:
122         printer.cleanup()
123
124     return num_unexpected_results
125
126
127 def _set_up_derived_options(port_obj, options):
128     """Sets the options values that depend on other options values."""
129     # We return a list of warnings to print after the printer is initialized.
130     warnings = []
131
132     if options.worker_model == 'old-inline':
133         if options.child_processes and int(options.child_processes) > 1:
134             warnings.append("--worker-model=old-inline overrides --child-processes")
135         options.child_processes = "1"
136     if not options.child_processes:
137         options.child_processes = os.environ.get("WEBKIT_TEST_CHILD_PROCESSES",
138                                                  str(port_obj.default_child_processes()))
139
140     if not options.configuration:
141         options.configuration = port_obj.default_configuration()
142
143     if options.pixel_tests is None:
144         options.pixel_tests = True
145
146     if not options.use_apache:
147         options.use_apache = sys.platform in ('darwin', 'linux2')
148
149     if not os.path.isabs(options.results_directory):
150         # This normalizes the path to the build dir.
151         # FIXME: how this happens is not at all obvious; this is a dumb
152         # interface and should be cleaned up.
153         options.results_directory = port_obj.results_directory()
154
155     if not options.time_out_ms:
156         if options.configuration == "Debug":
157             options.time_out_ms = str(2 * test_runner.TestRunner.DEFAULT_TEST_TIMEOUT_MS)
158         else:
159             options.time_out_ms = str(test_runner.TestRunner.DEFAULT_TEST_TIMEOUT_MS)
160
161     options.slow_time_out_ms = str(5 * int(options.time_out_ms))
162     return warnings
163
164
165 def _gather_unexpected_results(options):
166     """Returns the unexpected results from the previous run, if any."""
167     last_unexpected_results = []
168     if options.print_last_failures or options.retest_last_failures:
169         unexpected_results_filename = os.path.join(
170         options.results_directory, "unexpected_results.json")
171         with codecs.open(unexpected_results_filename, "r", "utf-8") as file:
172             results = simplejson.load(file)
173         last_unexpected_results = results['tests'].keys()
174     return last_unexpected_results
175
176
177 def _compat_shim_callback(option, opt_str, value, parser):
178     print "Ignoring unsupported option: %s" % opt_str
179
180
181 def _compat_shim_option(option_name, **kwargs):
182     return optparse.make_option(option_name, action="callback",
183         callback=_compat_shim_callback,
184         help="Ignored, for old-run-webkit-tests compat only.", **kwargs)
185
186
187 def parse_args(args=None):
188     """Provides a default set of command line args.
189
190     Returns a tuple of options, args from optparse"""
191
192     # FIXME: All of these options should be stored closer to the code which
193     # FIXME: actually uses them. configuration_options should move
194     # FIXME: to WebKitPort and be shared across all scripts.
195     configuration_options = [
196         optparse.make_option("-t", "--target", dest="configuration",
197                              help="(DEPRECATED)"),
198         # FIXME: --help should display which configuration is default.
199         optparse.make_option('--debug', action='store_const', const='Debug',
200                              dest="configuration",
201                              help='Set the configuration to Debug'),
202         optparse.make_option('--release', action='store_const',
203                              const='Release', dest="configuration",
204                              help='Set the configuration to Release'),
205         # old-run-webkit-tests also accepts -c, --configuration CONFIGURATION.
206     ]
207
208     print_options = printing.print_options()
209
210     # FIXME: These options should move onto the ChromiumPort.
211     chromium_options = [
212         optparse.make_option("--chromium", action="store_true", default=False,
213             help="use the Chromium port"),
214         optparse.make_option("--startup-dialog", action="store_true",
215             default=False, help="create a dialog on DumpRenderTree startup"),
216         optparse.make_option("--gp-fault-error-box", action="store_true",
217             default=False, help="enable Windows GP fault error box"),
218         optparse.make_option("--multiple-loads",
219             type="int", help="turn on multiple loads of each test"),
220         optparse.make_option("--js-flags",
221             type="string", help="JavaScript flags to pass to tests"),
222         optparse.make_option("--nocheck-sys-deps", action="store_true",
223             default=False,
224             help="Don't check the system dependencies (themes)"),
225         optparse.make_option("--use-test-shell", action="store_true",
226             default=False,
227             help="Use test_shell instead of DRT"),
228         optparse.make_option("--accelerated-compositing",
229             action="store_true",
230             help="Use hardware-accelated compositing for rendering"),
231         optparse.make_option("--no-accelerated-compositing",
232             action="store_false",
233             dest="accelerated_compositing",
234             help="Don't use hardware-accelerated compositing for rendering"),
235         optparse.make_option("--accelerated-2d-canvas",
236             action="store_true",
237             help="Use hardware-accelerated 2D Canvas calls"),
238         optparse.make_option("--no-accelerated-2d-canvas",
239             action="store_false",
240             dest="accelerated_2d_canvas",
241             help="Don't use hardware-accelerated 2D Canvas calls"),
242     ]
243
244     # Missing Mac-specific old-run-webkit-tests options:
245     # FIXME: Need: -g, --guard for guard malloc support on Mac.
246     # FIXME: Need: -l --leaks    Enable leaks checking.
247     # FIXME: Need: --sample-on-timeout Run sample on timeout
248
249     old_run_webkit_tests_compat = [
250         # NRWT doesn't generate results by default anyway.
251         _compat_shim_option("--no-new-test-results"),
252         # NRWT doesn't sample on timeout yet anyway.
253         _compat_shim_option("--no-sample-on-timeout"),
254         # FIXME: NRWT needs to support remote links eventually.
255         _compat_shim_option("--use-remote-links-to-tests"),
256     ]
257
258     results_options = [
259         # NEED for bots: --use-remote-links-to-tests Link to test files
260         # within the SVN repository in the results.
261         optparse.make_option("-p", "--pixel-tests", action="store_true",
262             dest="pixel_tests", help="Enable pixel-to-pixel PNG comparisons"),
263         optparse.make_option("--no-pixel-tests", action="store_false",
264             dest="pixel_tests", help="Disable pixel-to-pixel PNG comparisons"),
265         optparse.make_option("--tolerance",
266             help="Ignore image differences less than this percentage (some "
267                 "ports may ignore this option)", type="float"),
268         optparse.make_option("--results-directory",
269             default="layout-test-results",
270             help="Output results directory source dir, relative to Debug or "
271                  "Release"),
272         optparse.make_option("--new-baseline", action="store_true",
273             default=False, help="Save all generated results as new baselines "
274                  "into the platform directory, overwriting whatever's "
275                  "already there."),
276         optparse.make_option("--reset-results", action="store_true",
277             default=False, help="Reset any existing baselines to the "
278                  "generated results"),
279         optparse.make_option("--no-show-results", action="store_false",
280             default=True, dest="show_results",
281             help="Don't launch a browser with results after the tests "
282                  "are done"),
283         # FIXME: We should have a helper function to do this sort of
284         # deprectated mapping and automatically log, etc.
285         optparse.make_option("--noshow-results", action="store_false",
286             dest="show_results",
287             help="Deprecated, same as --no-show-results."),
288         optparse.make_option("--no-launch-safari", action="store_false",
289             dest="show_results",
290             help="old-run-webkit-tests compat, same as --noshow-results."),
291         # old-run-webkit-tests:
292         # --[no-]launch-safari    Launch (or do not launch) Safari to display
293         #                         test results (default: launch)
294         optparse.make_option("--full-results-html", action="store_true",
295             default=False,
296             help="Show all failures in results.html, rather than only "
297                  "regressions"),
298         optparse.make_option("--clobber-old-results", action="store_true",
299             default=False, help="Clobbers test results from previous runs."),
300         optparse.make_option("--platform",
301             help="Override the platform for expected results"),
302         optparse.make_option("--no-record-results", action="store_false",
303             default=True, dest="record_results",
304             help="Don't record the results."),
305         # old-run-webkit-tests also has HTTP toggle options:
306         # --[no-]http                     Run (or do not run) http tests
307         #                                 (default: run)
308     ]
309
310     test_options = [
311         optparse.make_option("--build", dest="build",
312             action="store_true", default=True,
313             help="Check to ensure the DumpRenderTree build is up-to-date "
314                  "(default)."),
315         optparse.make_option("--no-build", dest="build",
316             action="store_false", help="Don't check to see if the "
317                                        "DumpRenderTree build is up-to-date."),
318         optparse.make_option("-n", "--dry-run", action="store_true",
319             default=False,
320             help="Do everything but actually run the tests or upload results."),
321         # old-run-webkit-tests has --valgrind instead of wrapper.
322         optparse.make_option("--wrapper",
323             help="wrapper command to insert before invocations of "
324                  "DumpRenderTree; option is split on whitespace before "
325                  "running. (Example: --wrapper='valgrind --smc-check=all')"),
326         # old-run-webkit-tests:
327         # -i|--ignore-tests               Comma-separated list of directories
328         #                                 or tests to ignore
329         optparse.make_option("--test-list", action="append",
330             help="read list of tests to run from file", metavar="FILE"),
331         # old-run-webkit-tests uses --skipped==[default|ignore|only]
332         # instead of --force:
333         optparse.make_option("--force", action="store_true", default=False,
334             help="Run all tests, even those marked SKIP in the test list"),
335         optparse.make_option("--use-apache", action="store_true",
336             default=False, help="Whether to use apache instead of lighttpd."),
337         optparse.make_option("--time-out-ms",
338             help="Set the timeout for each test"),
339         # old-run-webkit-tests calls --randomize-order --random:
340         optparse.make_option("--randomize-order", action="store_true",
341             default=False, help=("Run tests in random order (useful "
342                                 "for tracking down corruption)")),
343         optparse.make_option("--run-chunk",
344             help=("Run a specified chunk (n:l), the nth of len l, "
345                  "of the layout tests")),
346         optparse.make_option("--run-part", help=("Run a specified part (n:m), "
347                   "the nth of m parts, of the layout tests")),
348         # old-run-webkit-tests calls --batch-size: --nthly n
349         #   Restart DumpRenderTree every n tests (default: 1000)
350         optparse.make_option("--batch-size",
351             help=("Run a the tests in batches (n), after every n tests, "
352                   "DumpRenderTree is relaunched."), type="int", default=0),
353         # old-run-webkit-tests calls --run-singly: -1|--singly
354         # Isolate each test case run (implies --nthly 1 --verbose)
355         optparse.make_option("--run-singly", action="store_true",
356             default=False, help="run a separate DumpRenderTree for each test"),
357         optparse.make_option("--child-processes",
358             help="Number of DumpRenderTrees to run in parallel."),
359         # FIXME: Display default number of child processes that will run.
360         optparse.make_option("--worker-model", action="store",
361             default="old-threads", help=("controls worker model. Valid values "
362             "are 'old-inline', 'old-threads'.")),
363         optparse.make_option("--experimental-fully-parallel",
364             action="store_true", default=False,
365             help="run all tests in parallel"),
366         optparse.make_option("--exit-after-n-failures", type="int", nargs=1,
367             help="Exit after the first N failures instead of running all "
368             "tests"),
369         optparse.make_option("--exit-after-n-crashes-or-timeouts", type="int",
370             nargs=1, help="Exit after the first N crashes instead of running "
371             "all tests"),
372         # FIXME: consider: --iterations n
373         #      Number of times to run the set of tests (e.g. ABCABCABC)
374         optparse.make_option("--print-last-failures", action="store_true",
375             default=False, help="Print the tests in the last run that "
376             "had unexpected failures (or passes) and then exit."),
377         optparse.make_option("--retest-last-failures", action="store_true",
378             default=False, help="re-test the tests in the last run that "
379             "had unexpected failures (or passes)."),
380         optparse.make_option("--retry-failures", action="store_true",
381             default=True,
382             help="Re-try any tests that produce unexpected results (default)"),
383         optparse.make_option("--no-retry-failures", action="store_false",
384             dest="retry_failures",
385             help="Don't re-try any tests that produce unexpected results."),
386     ]
387
388     misc_options = [
389         optparse.make_option("--lint-test-files", action="store_true",
390         default=False, help=("Makes sure the test files parse for all "
391                             "configurations. Does not run any tests.")),
392     ]
393
394     # FIXME: Move these into json_results_generator.py
395     results_json_options = [
396         optparse.make_option("--master-name", help="The name of the buildbot master."),
397         optparse.make_option("--builder-name", default="DUMMY_BUILDER_NAME",
398             help=("The name of the builder shown on the waterfall running "
399                   "this script e.g. WebKit.")),
400         optparse.make_option("--build-name", default="DUMMY_BUILD_NAME",
401             help=("The name of the builder used in its path, e.g. "
402                   "webkit-rel.")),
403         optparse.make_option("--build-number", default="DUMMY_BUILD_NUMBER",
404             help=("The build number of the builder running this script.")),
405         optparse.make_option("--test-results-server", default="",
406             help=("If specified, upload results json files to this appengine "
407                   "server.")),
408         optparse.make_option("--upload-full-results",
409             action="store_true",
410             default=False,
411             help="If true, upload full json results to server."),
412     ]
413
414     option_list = (configuration_options + print_options +
415                    chromium_options + results_options + test_options +
416                    misc_options + results_json_options +
417                    old_run_webkit_tests_compat)
418     option_parser = optparse.OptionParser(option_list=option_list)
419
420     return option_parser.parse_args(args)
421
422
423 def main():
424     options, args = parse_args()
425     port_obj = port.get(options.platform, options)
426     return run(port_obj, options, args)
427
428
429 if '__main__' == __name__:
430     try:
431         sys.exit(main())
432     except KeyboardInterrupt:
433         # this mirrors what the shell normally does
434         sys.exit(signal.SIGINT + 128)