2011-01-19 Dirk Pranke <dpranke@chromium.org>
[WebKit.git] / Tools / Scripts / webkitpy / layout_tests / run_webkit_tests.py
1 #!/usr/bin/env python
2 # Copyright (C) 2010 Google Inc. All rights reserved.
3 # Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Szeged
4 #
5 # Redistribution and use in source and binary forms, with or without
6 # modification, are permitted provided that the following conditions are
7 # met:
8 #
9 #     * Redistributions of source code must retain the above copyright
10 # notice, this list of conditions and the following disclaimer.
11 #     * Redistributions in binary form must reproduce the above
12 # copyright notice, this list of conditions and the following disclaimer
13 # in the documentation and/or other materials provided with the
14 # distribution.
15 #     * Neither the name of Google Inc. nor the names of its
16 # contributors may be used to endorse or promote products derived from
17 # this software without specific prior written permission.
18 #
19 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30
31 """Run layout tests."""
32
33 from __future__ import with_statement
34
35 import codecs
36 import errno
37 import logging
38 import optparse
39 import os
40 import signal
41 import sys
42
43 from layout_package import printing
44 from layout_package import test_runner
45
46 from webkitpy.common.system import user
47 from webkitpy.thirdparty import simplejson
48
49 import port
50
51 _log = logging.getLogger(__name__)
52
53
54 def run(port, options, args, regular_output=sys.stderr,
55         buildbot_output=sys.stdout):
56     """Run the tests.
57
58     Args:
59       port: Port object for port-specific behavior
60       options: a dictionary of command line options
61       args: a list of sub directories or files to test
62       regular_output: a stream-like object that we can send logging/debug
63           output to
64       buildbot_output: a stream-like object that we can write all output that
65           is intended to be parsed by the buildbot to
66     Returns:
67       the number of unexpected results that occurred, or -1 if there is an
68           error.
69
70     """
71     warnings = _set_up_derived_options(port, options)
72
73     printer = printing.Printer(port, options, regular_output, buildbot_output,
74         int(options.child_processes), options.experimental_fully_parallel)
75     for w in warnings:
76         _log.warning(w)
77
78     if options.help_printing:
79         printer.help_printing()
80         printer.cleanup()
81         return 0
82
83     last_unexpected_results = _gather_unexpected_results(port, options)
84     if options.print_last_failures:
85         printer.write("\n".join(last_unexpected_results) + "\n")
86         printer.cleanup()
87         return 0
88
89     # We wrap any parts of the run that are slow or likely to raise exceptions
90     # in a try/finally to ensure that we clean up the logging configuration.
91     num_unexpected_results = -1
92     try:
93         runner = test_runner.TestRunner(port, options, printer)
94         runner._print_config()
95
96         printer.print_update("Collecting tests ...")
97         try:
98             runner.collect_tests(args, last_unexpected_results)
99         except IOError, e:
100             if e.errno == errno.ENOENT:
101                 return -1
102             raise
103
104         printer.print_update("Parsing expectations ...")
105         if options.lint_test_files:
106             return runner.lint()
107         runner.parse_expectations(port.test_platform_name(),
108                                   options.configuration == 'Debug')
109
110         printer.print_update("Checking build ...")
111         if not port.check_build(runner.needs_http()):
112             _log.error("Build check failed")
113             return -1
114
115         result_summary = runner.set_up_run()
116         if result_summary:
117             num_unexpected_results = runner.run(result_summary)
118             runner.clean_up_run()
119             _log.debug("Testing completed, Exit status: %d" %
120                        num_unexpected_results)
121     finally:
122         printer.cleanup()
123
124     return num_unexpected_results
125
126
127 def _set_up_derived_options(port_obj, options):
128     """Sets the options values that depend on other options values."""
129     # We return a list of warnings to print after the printer is initialized.
130     warnings = []
131
132     if options.worker_model == 'old-inline':
133         if options.child_processes and int(options.child_processes) > 1:
134             warnings.append("--worker-model=old-inline overrides --child-processes")
135         options.child_processes = "1"
136     if not options.child_processes:
137         options.child_processes = os.environ.get("WEBKIT_TEST_CHILD_PROCESSES",
138                                                  str(port_obj.default_child_processes()))
139
140     if not options.configuration:
141         options.configuration = port_obj.default_configuration()
142
143     if options.pixel_tests is None:
144         options.pixel_tests = True
145
146     if not options.use_apache:
147         options.use_apache = sys.platform in ('darwin', 'linux2')
148
149     if not os.path.isabs(options.results_directory):
150         # This normalizes the path to the build dir.
151         # FIXME: how this happens is not at all obvious; this is a dumb
152         # interface and should be cleaned up.
153         options.results_directory = port_obj.results_directory()
154
155     if not options.time_out_ms:
156         if options.configuration == "Debug":
157             options.time_out_ms = str(2 * test_runner.TestRunner.DEFAULT_TEST_TIMEOUT_MS)
158         else:
159             options.time_out_ms = str(test_runner.TestRunner.DEFAULT_TEST_TIMEOUT_MS)
160
161     options.slow_time_out_ms = str(5 * int(options.time_out_ms))
162     return warnings
163
164
165 def _gather_unexpected_results(port, options):
166     """Returns the unexpected results from the previous run, if any."""
167     last_unexpected_results = []
168     if options.print_last_failures or options.retest_last_failures:
169         unexpected_results_filename = os.path.join(
170         options.results_directory, "unexpected_results.json")
171         if port._filesystem.exists(unexpected_results_filename):
172             file = port._filesystem.read_text_file(unexpected_results.json)
173             results = simplejson.load(file)
174             last_unexpected_results = results['tests'].keys()
175     return last_unexpected_results
176
177
178 def _compat_shim_callback(option, opt_str, value, parser):
179     print "Ignoring unsupported option: %s" % opt_str
180
181
182 def _compat_shim_option(option_name, **kwargs):
183     return optparse.make_option(option_name, action="callback",
184         callback=_compat_shim_callback,
185         help="Ignored, for old-run-webkit-tests compat only.", **kwargs)
186
187
188 def parse_args(args=None):
189     """Provides a default set of command line args.
190
191     Returns a tuple of options, args from optparse"""
192
193     # FIXME: All of these options should be stored closer to the code which
194     # FIXME: actually uses them. configuration_options should move
195     # FIXME: to WebKitPort and be shared across all scripts.
196     configuration_options = [
197         optparse.make_option("-t", "--target", dest="configuration",
198                              help="(DEPRECATED)"),
199         # FIXME: --help should display which configuration is default.
200         optparse.make_option('--debug', action='store_const', const='Debug',
201                              dest="configuration",
202                              help='Set the configuration to Debug'),
203         optparse.make_option('--release', action='store_const',
204                              const='Release', dest="configuration",
205                              help='Set the configuration to Release'),
206         # old-run-webkit-tests also accepts -c, --configuration CONFIGURATION.
207     ]
208
209     print_options = printing.print_options()
210
211     # FIXME: These options should move onto the ChromiumPort.
212     chromium_options = [
213         optparse.make_option("--chromium", action="store_true", default=False,
214             help="use the Chromium port"),
215         optparse.make_option("--startup-dialog", action="store_true",
216             default=False, help="create a dialog on DumpRenderTree startup"),
217         optparse.make_option("--gp-fault-error-box", action="store_true",
218             default=False, help="enable Windows GP fault error box"),
219         optparse.make_option("--js-flags",
220             type="string", help="JavaScript flags to pass to tests"),
221         optparse.make_option("--stress-opt", action="store_true",
222             default=False,
223             help="Enable additional stress test to JavaScript optimization"),
224         optparse.make_option("--stress-deopt", action="store_true",
225             default=False,
226             help="Enable additional stress test to JavaScript optimization"),
227         optparse.make_option("--nocheck-sys-deps", action="store_true",
228             default=False,
229             help="Don't check the system dependencies (themes)"),
230         optparse.make_option("--use-test-shell", action="store_true",
231             default=False,
232             help="Use test_shell instead of DRT"),
233         optparse.make_option("--accelerated-compositing",
234             action="store_true",
235             help="Use hardware-accelated compositing for rendering"),
236         optparse.make_option("--no-accelerated-compositing",
237             action="store_false",
238             dest="accelerated_compositing",
239             help="Don't use hardware-accelerated compositing for rendering"),
240         optparse.make_option("--accelerated-2d-canvas",
241             action="store_true",
242             help="Use hardware-accelerated 2D Canvas calls"),
243         optparse.make_option("--no-accelerated-2d-canvas",
244             action="store_false",
245             dest="accelerated_2d_canvas",
246             help="Don't use hardware-accelerated 2D Canvas calls"),
247         optparse.make_option("--enable-hardware-gpu",
248             action="store_true",
249             default=False,
250             help="Run graphics tests on real GPU hardware vs software"),
251     ]
252
253     # Missing Mac-specific old-run-webkit-tests options:
254     # FIXME: Need: -g, --guard for guard malloc support on Mac.
255     # FIXME: Need: -l --leaks    Enable leaks checking.
256     # FIXME: Need: --sample-on-timeout Run sample on timeout
257
258     old_run_webkit_tests_compat = [
259         # NRWT doesn't generate results by default anyway.
260         _compat_shim_option("--no-new-test-results"),
261         # NRWT doesn't sample on timeout yet anyway.
262         _compat_shim_option("--no-sample-on-timeout"),
263         # FIXME: NRWT needs to support remote links eventually.
264         _compat_shim_option("--use-remote-links-to-tests"),
265     ]
266
267     results_options = [
268         # NEED for bots: --use-remote-links-to-tests Link to test files
269         # within the SVN repository in the results.
270         optparse.make_option("-p", "--pixel-tests", action="store_true",
271             dest="pixel_tests", help="Enable pixel-to-pixel PNG comparisons"),
272         optparse.make_option("--no-pixel-tests", action="store_false",
273             dest="pixel_tests", help="Disable pixel-to-pixel PNG comparisons"),
274         optparse.make_option("--tolerance",
275             help="Ignore image differences less than this percentage (some "
276                 "ports may ignore this option)", type="float"),
277         optparse.make_option("--results-directory",
278             default="layout-test-results",
279             help="Output results directory source dir, relative to Debug or "
280                  "Release"),
281         optparse.make_option("--build-directory",
282             help="Path to the directory under which build files are kept (should not include configuration)"),
283         optparse.make_option("--new-baseline", action="store_true",
284             default=False, help="Save all generated results as new baselines "
285                  "into the platform directory, overwriting whatever's "
286                  "already there."),
287         optparse.make_option("--reset-results", action="store_true",
288             default=False, help="Reset any existing baselines to the "
289                  "generated results"),
290         optparse.make_option("--no-show-results", action="store_false",
291             default=True, dest="show_results",
292             help="Don't launch a browser with results after the tests "
293                  "are done"),
294         # FIXME: We should have a helper function to do this sort of
295         # deprectated mapping and automatically log, etc.
296         optparse.make_option("--noshow-results", action="store_false",
297             dest="show_results",
298             help="Deprecated, same as --no-show-results."),
299         optparse.make_option("--no-launch-safari", action="store_false",
300             dest="show_results",
301             help="old-run-webkit-tests compat, same as --noshow-results."),
302         # old-run-webkit-tests:
303         # --[no-]launch-safari    Launch (or do not launch) Safari to display
304         #                         test results (default: launch)
305         optparse.make_option("--full-results-html", action="store_true",
306             default=False,
307             help="Show all failures in results.html, rather than only "
308                  "regressions"),
309         optparse.make_option("--clobber-old-results", action="store_true",
310             default=False, help="Clobbers test results from previous runs."),
311         optparse.make_option("--platform",
312             help="Override the platform for expected results"),
313         optparse.make_option("--no-record-results", action="store_false",
314             default=True, dest="record_results",
315             help="Don't record the results."),
316         # old-run-webkit-tests also has HTTP toggle options:
317         # --[no-]http                     Run (or do not run) http tests
318         #                                 (default: run)
319     ]
320
321     test_options = [
322         optparse.make_option("--build", dest="build",
323             action="store_true", default=True,
324             help="Check to ensure the DumpRenderTree build is up-to-date "
325                  "(default)."),
326         optparse.make_option("--no-build", dest="build",
327             action="store_false", help="Don't check to see if the "
328                                        "DumpRenderTree build is up-to-date."),
329         optparse.make_option("-n", "--dry-run", action="store_true",
330             default=False,
331             help="Do everything but actually run the tests or upload results."),
332         # old-run-webkit-tests has --valgrind instead of wrapper.
333         optparse.make_option("--wrapper",
334             help="wrapper command to insert before invocations of "
335                  "DumpRenderTree; option is split on whitespace before "
336                  "running. (Example: --wrapper='valgrind --smc-check=all')"),
337         # old-run-webkit-tests:
338         # -i|--ignore-tests               Comma-separated list of directories
339         #                                 or tests to ignore
340         optparse.make_option("--test-list", action="append",
341             help="read list of tests to run from file", metavar="FILE"),
342         # old-run-webkit-tests uses --skipped==[default|ignore|only]
343         # instead of --force:
344         optparse.make_option("--force", action="store_true", default=False,
345             help="Run all tests, even those marked SKIP in the test list"),
346         optparse.make_option("--use-apache", action="store_true",
347             default=False, help="Whether to use apache instead of lighttpd."),
348         optparse.make_option("--time-out-ms",
349             help="Set the timeout for each test"),
350         # old-run-webkit-tests calls --randomize-order --random:
351         optparse.make_option("--randomize-order", action="store_true",
352             default=False, help=("Run tests in random order (useful "
353                                 "for tracking down corruption)")),
354         optparse.make_option("--run-chunk",
355             help=("Run a specified chunk (n:l), the nth of len l, "
356                  "of the layout tests")),
357         optparse.make_option("--run-part", help=("Run a specified part (n:m), "
358                   "the nth of m parts, of the layout tests")),
359         # old-run-webkit-tests calls --batch-size: --nthly n
360         #   Restart DumpRenderTree every n tests (default: 1000)
361         optparse.make_option("--batch-size",
362             help=("Run a the tests in batches (n), after every n tests, "
363                   "DumpRenderTree is relaunched."), type="int", default=0),
364         # old-run-webkit-tests calls --run-singly: -1|--singly
365         # Isolate each test case run (implies --nthly 1 --verbose)
366         optparse.make_option("--run-singly", action="store_true",
367             default=False, help="run a separate DumpRenderTree for each test"),
368         optparse.make_option("--child-processes",
369             help="Number of DumpRenderTrees to run in parallel."),
370         # FIXME: Display default number of child processes that will run.
371         optparse.make_option("--worker-model", action="store",
372             default="old-threads", help=("controls worker model. Valid values "
373             "are 'old-inline', 'old-threads'.")),
374         optparse.make_option("--experimental-fully-parallel",
375             action="store_true", default=False,
376             help="run all tests in parallel"),
377         optparse.make_option("--exit-after-n-failures", type="int", nargs=1,
378             help="Exit after the first N failures instead of running all "
379             "tests"),
380         optparse.make_option("--exit-after-n-crashes-or-timeouts", type="int",
381             nargs=1, help="Exit after the first N crashes instead of running "
382             "all tests"),
383         # FIXME: consider: --iterations n
384         #      Number of times to run the set of tests (e.g. ABCABCABC)
385         optparse.make_option("--print-last-failures", action="store_true",
386             default=False, help="Print the tests in the last run that "
387             "had unexpected failures (or passes) and then exit."),
388         optparse.make_option("--retest-last-failures", action="store_true",
389             default=False, help="re-test the tests in the last run that "
390             "had unexpected failures (or passes)."),
391         optparse.make_option("--retry-failures", action="store_true",
392             default=True,
393             help="Re-try any tests that produce unexpected results (default)"),
394         optparse.make_option("--no-retry-failures", action="store_false",
395             dest="retry_failures",
396             help="Don't re-try any tests that produce unexpected results."),
397     ]
398
399     misc_options = [
400         optparse.make_option("--lint-test-files", action="store_true",
401         default=False, help=("Makes sure the test files parse for all "
402                             "configurations. Does not run any tests.")),
403     ]
404
405     # FIXME: Move these into json_results_generator.py
406     results_json_options = [
407         optparse.make_option("--master-name", help="The name of the buildbot master."),
408         optparse.make_option("--builder-name", default="DUMMY_BUILDER_NAME",
409             help=("The name of the builder shown on the waterfall running "
410                   "this script e.g. WebKit.")),
411         optparse.make_option("--build-name", default="DUMMY_BUILD_NAME",
412             help=("The name of the builder used in its path, e.g. "
413                   "webkit-rel.")),
414         optparse.make_option("--build-number", default="DUMMY_BUILD_NUMBER",
415             help=("The build number of the builder running this script.")),
416         optparse.make_option("--test-results-server", default="",
417             help=("If specified, upload results json files to this appengine "
418                   "server.")),
419         optparse.make_option("--upload-full-results",
420             action="store_true",
421             default=False,
422             help="If true, upload full json results to server."),
423     ]
424
425     option_list = (configuration_options + print_options +
426                    chromium_options + results_options + test_options +
427                    misc_options + results_json_options +
428                    old_run_webkit_tests_compat)
429     option_parser = optparse.OptionParser(option_list=option_list)
430
431     return option_parser.parse_args(args)
432
433
434 def main():
435     options, args = parse_args()
436     port_obj = port.get(options.platform, options)
437     return run(port_obj, options, args)
438
439
440 if '__main__' == __name__:
441     try:
442         sys.exit(main())
443     except KeyboardInterrupt:
444         # this mirrors what the shell normally does
445         sys.exit(signal.SIGINT + 128)