074d9c46db352930858cc8ea4859db3cbbcb5054
[WebKit.git] / Tools / Scripts / webkitpy / layout_tests / run_webkit_tests.py
1 #!/usr/bin/env python
2 # Copyright (C) 2010 Google Inc. All rights reserved.
3 # Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Szeged
4 #
5 # Redistribution and use in source and binary forms, with or without
6 # modification, are permitted provided that the following conditions are
7 # met:
8 #
9 #     * Redistributions of source code must retain the above copyright
10 # notice, this list of conditions and the following disclaimer.
11 #     * Redistributions in binary form must reproduce the above
12 # copyright notice, this list of conditions and the following disclaimer
13 # in the documentation and/or other materials provided with the
14 # distribution.
15 #     * Neither the name of Google Inc. nor the names of its
16 # contributors may be used to endorse or promote products derived from
17 # this software without specific prior written permission.
18 #
19 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30
31 """Run layout tests."""
32
33 import errno
34 import logging
35 import optparse
36 import os
37 import signal
38 import sys
39
40 from layout_package import printing
41 from layout_package import test_runner
42
43 from webkitpy.common.system import user
44 from webkitpy.thirdparty import simplejson
45
46 import port
47
48 _log = logging.getLogger(__name__)
49
50
51 def run(port, options, args, regular_output=sys.stderr,
52         buildbot_output=sys.stdout):
53     """Run the tests.
54
55     Args:
56       port: Port object for port-specific behavior
57       options: a dictionary of command line options
58       args: a list of sub directories or files to test
59       regular_output: a stream-like object that we can send logging/debug
60           output to
61       buildbot_output: a stream-like object that we can write all output that
62           is intended to be parsed by the buildbot to
63     Returns:
64       the number of unexpected results that occurred, or -1 if there is an
65           error.
66
67     """
68     warnings = _set_up_derived_options(port, options)
69
70     printer = printing.Printer(port, options, regular_output, buildbot_output,
71         int(options.child_processes), options.experimental_fully_parallel)
72     for w in warnings:
73         _log.warning(w)
74
75     if options.help_printing:
76         printer.help_printing()
77         printer.cleanup()
78         return 0
79
80     last_unexpected_results = _gather_unexpected_results(port._filesystem, options)
81     if options.print_last_failures:
82         printer.write("\n".join(last_unexpected_results) + "\n")
83         printer.cleanup()
84         return 0
85
86     # We wrap any parts of the run that are slow or likely to raise exceptions
87     # in a try/finally to ensure that we clean up the logging configuration.
88     num_unexpected_results = -1
89     try:
90         runner = test_runner.TestRunner(port, options, printer)
91         runner._print_config()
92
93         printer.print_update("Collecting tests ...")
94         try:
95             runner.collect_tests(args, last_unexpected_results)
96         except IOError, e:
97             if e.errno == errno.ENOENT:
98                 return -1
99             raise
100
101         if options.lint_test_files:
102             return runner.lint()
103
104         printer.print_update("Parsing expectations ...")
105         runner.parse_expectations()
106
107         printer.print_update("Checking build ...")
108         if not port.check_build(runner.needs_http()):
109             _log.error("Build check failed")
110             return -1
111
112         result_summary = runner.set_up_run()
113         if result_summary:
114             num_unexpected_results = runner.run(result_summary)
115             runner.clean_up_run()
116             _log.debug("Testing completed, Exit status: %d" %
117                        num_unexpected_results)
118     finally:
119         printer.cleanup()
120
121     return num_unexpected_results
122
123
124 def _set_up_derived_options(port_obj, options):
125     """Sets the options values that depend on other options values."""
126     # We return a list of warnings to print after the printer is initialized.
127     warnings = []
128
129     if options.worker_model is None:
130         options.worker_model = port_obj.default_worker_model()
131
132     if options.worker_model == 'old-inline':
133         if options.child_processes and int(options.child_processes) > 1:
134             warnings.append("--worker-model=old-inline overrides --child-processes")
135         options.child_processes = "1"
136     if not options.child_processes:
137         options.child_processes = os.environ.get("WEBKIT_TEST_CHILD_PROCESSES",
138                                                  str(port_obj.default_child_processes()))
139
140     if not options.configuration:
141         options.configuration = port_obj.default_configuration()
142
143     if options.pixel_tests is None:
144         options.pixel_tests = True
145
146     if not options.use_apache:
147         options.use_apache = sys.platform in ('darwin', 'linux2')
148
149     if not port_obj._filesystem.isabs(options.results_directory):
150         # This normalizes the path to the build dir.
151         # FIXME: how this happens is not at all obvious; this is a dumb
152         # interface and should be cleaned up.
153         options.results_directory = port_obj.results_directory()
154
155     if not options.time_out_ms:
156         if options.configuration == "Debug":
157             options.time_out_ms = str(2 * test_runner.TestRunner.DEFAULT_TEST_TIMEOUT_MS)
158         else:
159             options.time_out_ms = str(test_runner.TestRunner.DEFAULT_TEST_TIMEOUT_MS)
160
161     options.slow_time_out_ms = str(5 * int(options.time_out_ms))
162     return warnings
163
164
165 def _gather_unexpected_results(filesystem, options):
166     """Returns the unexpected results from the previous run, if any."""
167     last_unexpected_results = []
168     if options.print_last_failures or options.retest_last_failures:
169         unexpected_results_filename = filesystem.join(
170             options.results_directory, "unexpected_results.json")
171         if filesystem.exists(unexpected_results_filename):
172             content = filesystem.read_text_file(unexpected_results_filename)
173             results = simplejson.loads(content)
174             last_unexpected_results = results['tests'].keys()
175     return last_unexpected_results
176
177
178 def _compat_shim_callback(option, opt_str, value, parser):
179     print "Ignoring unsupported option: %s" % opt_str
180
181
182 def _compat_shim_option(option_name, **kwargs):
183     return optparse.make_option(option_name, action="callback",
184         callback=_compat_shim_callback,
185         help="Ignored, for old-run-webkit-tests compat only.", **kwargs)
186
187
188 def parse_args(args=None):
189     """Provides a default set of command line args.
190
191     Returns a tuple of options, args from optparse"""
192
193     # FIXME: All of these options should be stored closer to the code which
194     # FIXME: actually uses them. configuration_options should move
195     # FIXME: to WebKitPort and be shared across all scripts.
196     configuration_options = [
197         optparse.make_option("-t", "--target", dest="configuration",
198                              help="(DEPRECATED)"),
199         # FIXME: --help should display which configuration is default.
200         optparse.make_option('--debug', action='store_const', const='Debug',
201                              dest="configuration",
202                              help='Set the configuration to Debug'),
203         optparse.make_option('--release', action='store_const',
204                              const='Release', dest="configuration",
205                              help='Set the configuration to Release'),
206         # old-run-webkit-tests also accepts -c, --configuration CONFIGURATION.
207     ]
208
209     print_options = printing.print_options()
210
211     # FIXME: These options should move onto the ChromiumPort.
212     chromium_options = [
213         optparse.make_option("--chromium", action="store_true", default=False,
214             help="use the Chromium port"),
215         optparse.make_option("--startup-dialog", action="store_true",
216             default=False, help="create a dialog on DumpRenderTree startup"),
217         optparse.make_option("--gp-fault-error-box", action="store_true",
218             default=False, help="enable Windows GP fault error box"),
219         optparse.make_option("--js-flags",
220             type="string", help="JavaScript flags to pass to tests"),
221         optparse.make_option("--stress-opt", action="store_true",
222             default=False,
223             help="Enable additional stress test to JavaScript optimization"),
224         optparse.make_option("--stress-deopt", action="store_true",
225             default=False,
226             help="Enable additional stress test to JavaScript optimization"),
227         optparse.make_option("--nocheck-sys-deps", action="store_true",
228             default=False,
229             help="Don't check the system dependencies (themes)"),
230         optparse.make_option("--use-test-shell", action="store_true",
231             default=False,
232             help="Use test_shell instead of DRT"),
233         optparse.make_option("--accelerated-compositing",
234             action="store_true",
235             help="Use hardware-accelated compositing for rendering"),
236         optparse.make_option("--no-accelerated-compositing",
237             action="store_false",
238             dest="accelerated_compositing",
239             help="Don't use hardware-accelerated compositing for rendering"),
240         optparse.make_option("--accelerated-2d-canvas",
241             action="store_true",
242             help="Use hardware-accelerated 2D Canvas calls"),
243         optparse.make_option("--no-accelerated-2d-canvas",
244             action="store_false",
245             dest="accelerated_2d_canvas",
246             help="Don't use hardware-accelerated 2D Canvas calls"),
247         optparse.make_option("--enable-hardware-gpu",
248             action="store_true",
249             default=False,
250             help="Run graphics tests on real GPU hardware vs software"),
251     ]
252
253     # Missing Mac-specific old-run-webkit-tests options:
254     # FIXME: Need: -g, --guard for guard malloc support on Mac.
255     # FIXME: Need: -l --leaks    Enable leaks checking.
256     # FIXME: Need: --sample-on-timeout Run sample on timeout
257
258     old_run_webkit_tests_compat = [
259         # NRWT doesn't generate results by default anyway.
260         _compat_shim_option("--no-new-test-results"),
261         # NRWT doesn't sample on timeout yet anyway.
262         _compat_shim_option("--no-sample-on-timeout"),
263         # FIXME: NRWT needs to support remote links eventually.
264         _compat_shim_option("--use-remote-links-to-tests"),
265     ]
266
267     results_options = [
268         # NEED for bots: --use-remote-links-to-tests Link to test files
269         # within the SVN repository in the results.
270         optparse.make_option("-p", "--pixel-tests", action="store_true",
271             dest="pixel_tests", help="Enable pixel-to-pixel PNG comparisons"),
272         optparse.make_option("--no-pixel-tests", action="store_false",
273             dest="pixel_tests", help="Disable pixel-to-pixel PNG comparisons"),
274         optparse.make_option("--tolerance",
275             help="Ignore image differences less than this percentage (some "
276                 "ports may ignore this option)", type="float"),
277         optparse.make_option("--results-directory",
278             default="layout-test-results",
279             help="Output results directory source dir, relative to Debug or "
280                  "Release"),
281         optparse.make_option("--build-directory",
282             help="Path to the directory under which build files are kept (should not include configuration)"),
283         optparse.make_option("--new-baseline", action="store_true",
284             default=False, help="Save all generated results as new baselines "
285                  "into the platform directory, overwriting whatever's "
286                  "already there."),
287         optparse.make_option("--reset-results", action="store_true",
288             default=False, help="Reset any existing baselines to the "
289                  "generated results"),
290         optparse.make_option("--no-show-results", action="store_false",
291             default=True, dest="show_results",
292             help="Don't launch a browser with results after the tests "
293                  "are done"),
294         # FIXME: We should have a helper function to do this sort of
295         # deprectated mapping and automatically log, etc.
296         optparse.make_option("--noshow-results", action="store_false",
297             dest="show_results",
298             help="Deprecated, same as --no-show-results."),
299         optparse.make_option("--no-launch-safari", action="store_false",
300             dest="show_results",
301             help="old-run-webkit-tests compat, same as --noshow-results."),
302         # old-run-webkit-tests:
303         # --[no-]launch-safari    Launch (or do not launch) Safari to display
304         #                         test results (default: launch)
305         optparse.make_option("--full-results-html", action="store_true",
306             default=False,
307             help="Show all failures in results.html, rather than only "
308                  "regressions"),
309         optparse.make_option("--clobber-old-results", action="store_true",
310             default=False, help="Clobbers test results from previous runs."),
311         optparse.make_option("--platform",
312             help="Override the platform for expected results"),
313         optparse.make_option("--no-record-results", action="store_false",
314             default=True, dest="record_results",
315             help="Don't record the results."),
316         # old-run-webkit-tests also has HTTP toggle options:
317         # --[no-]http                     Run (or do not run) http tests
318         #                                 (default: run)
319     ]
320
321     test_options = [
322         optparse.make_option("--build", dest="build",
323             action="store_true", default=True,
324             help="Check to ensure the DumpRenderTree build is up-to-date "
325                  "(default)."),
326         optparse.make_option("--no-build", dest="build",
327             action="store_false", help="Don't check to see if the "
328                                        "DumpRenderTree build is up-to-date."),
329         optparse.make_option("-n", "--dry-run", action="store_true",
330             default=False,
331             help="Do everything but actually run the tests or upload results."),
332         # old-run-webkit-tests has --valgrind instead of wrapper.
333         optparse.make_option("--wrapper",
334             help="wrapper command to insert before invocations of "
335                  "DumpRenderTree; option is split on whitespace before "
336                  "running. (Example: --wrapper='valgrind --smc-check=all')"),
337         # old-run-webkit-tests:
338         # -i|--ignore-tests               Comma-separated list of directories
339         #                                 or tests to ignore
340         optparse.make_option("--test-list", action="append",
341             help="read list of tests to run from file", metavar="FILE"),
342         # old-run-webkit-tests uses --skipped==[default|ignore|only]
343         # instead of --force:
344         optparse.make_option("--force", action="store_true", default=False,
345             help="Run all tests, even those marked SKIP in the test list"),
346         optparse.make_option("--use-apache", action="store_true",
347             default=False, help="Whether to use apache instead of lighttpd."),
348         optparse.make_option("--time-out-ms",
349             help="Set the timeout for each test"),
350         # old-run-webkit-tests calls --randomize-order --random:
351         optparse.make_option("--randomize-order", action="store_true",
352             default=False, help=("Run tests in random order (useful "
353                                 "for tracking down corruption)")),
354         optparse.make_option("--run-chunk",
355             help=("Run a specified chunk (n:l), the nth of len l, "
356                  "of the layout tests")),
357         optparse.make_option("--run-part", help=("Run a specified part (n:m), "
358                   "the nth of m parts, of the layout tests")),
359         # old-run-webkit-tests calls --batch-size: --nthly n
360         #   Restart DumpRenderTree every n tests (default: 1000)
361         optparse.make_option("--batch-size",
362             help=("Run a the tests in batches (n), after every n tests, "
363                   "DumpRenderTree is relaunched."), type="int", default=0),
364         # old-run-webkit-tests calls --run-singly: -1|--singly
365         # Isolate each test case run (implies --nthly 1 --verbose)
366         optparse.make_option("--run-singly", action="store_true",
367             default=False, help="run a separate DumpRenderTree for each test"),
368         optparse.make_option("--child-processes",
369             help="Number of DumpRenderTrees to run in parallel."),
370         # FIXME: Display default number of child processes that will run.
371         optparse.make_option("--worker-model", action="store",
372             default=None, help=("controls worker model. Valid values "
373                                 "are 'old-inline', 'old-threads'.")),
374         optparse.make_option("--experimental-fully-parallel",
375             action="store_true", default=False,
376             help="run all tests in parallel"),
377         optparse.make_option("--exit-after-n-failures", type="int", nargs=1,
378             help="Exit after the first N failures instead of running all "
379             "tests"),
380         optparse.make_option("--exit-after-n-crashes-or-timeouts", type="int",
381             nargs=1, help="Exit after the first N crashes instead of running "
382             "all tests"),
383         # FIXME: consider: --iterations n
384         #      Number of times to run the set of tests (e.g. ABCABCABC)
385         optparse.make_option("--print-last-failures", action="store_true",
386             default=False, help="Print the tests in the last run that "
387             "had unexpected failures (or passes) and then exit."),
388         optparse.make_option("--retest-last-failures", action="store_true",
389             default=False, help="re-test the tests in the last run that "
390             "had unexpected failures (or passes)."),
391         optparse.make_option("--retry-failures", action="store_true",
392             default=True,
393             help="Re-try any tests that produce unexpected results (default)"),
394         optparse.make_option("--no-retry-failures", action="store_false",
395             dest="retry_failures",
396             help="Don't re-try any tests that produce unexpected results."),
397     ]
398
399     misc_options = [
400         optparse.make_option("--lint-test-files", action="store_true",
401         default=False, help=("Makes sure the test files parse for all "
402                             "configurations. Does not run any tests.")),
403     ]
404
405     # FIXME: Move these into json_results_generator.py
406     results_json_options = [
407         optparse.make_option("--master-name", help="The name of the buildbot master."),
408         optparse.make_option("--builder-name", default="DUMMY_BUILDER_NAME",
409             help=("The name of the builder shown on the waterfall running "
410                   "this script e.g. WebKit.")),
411         optparse.make_option("--build-name", default="DUMMY_BUILD_NAME",
412             help=("The name of the builder used in its path, e.g. "
413                   "webkit-rel.")),
414         optparse.make_option("--build-number", default="DUMMY_BUILD_NUMBER",
415             help=("The build number of the builder running this script.")),
416         optparse.make_option("--test-results-server", default="",
417             help=("If specified, upload results json files to this appengine "
418                   "server.")),
419         optparse.make_option("--upload-full-results",
420             action="store_true",
421             default=False,
422             help="If true, upload full json results to server."),
423     ]
424
425     option_list = (configuration_options + print_options +
426                    chromium_options + results_options + test_options +
427                    misc_options + results_json_options +
428                    old_run_webkit_tests_compat)
429     option_parser = optparse.OptionParser(option_list=option_list)
430
431     return option_parser.parse_args(args)
432
433
434 def main():
435     options, args = parse_args()
436     port_obj = port.get(options.platform, options)
437     return run(port_obj, options, args)
438
439
440 if '__main__' == __name__:
441     try:
442         sys.exit(main())
443     except KeyboardInterrupt:
444         # this mirrors what the shell normally does
445         sys.exit(signal.SIGINT + 128)