dcdcfaa33e7f706c39a3662b5167cdfcbf4440d1
[WebKit.git] / Tools / Scripts / webkitpy / layout_tests / run_webkit_tests.py
1 #!/usr/bin/env python
2 # Copyright (C) 2010 Google Inc. All rights reserved.
3 # Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Szeged
4 #
5 # Redistribution and use in source and binary forms, with or without
6 # modification, are permitted provided that the following conditions are
7 # met:
8 #
9 #     * Redistributions of source code must retain the above copyright
10 # notice, this list of conditions and the following disclaimer.
11 #     * Redistributions in binary form must reproduce the above
12 # copyright notice, this list of conditions and the following disclaimer
13 # in the documentation and/or other materials provided with the
14 # distribution.
15 #     * Neither the name of Google Inc. nor the names of its
16 # contributors may be used to endorse or promote products derived from
17 # this software without specific prior written permission.
18 #
19 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30
31 """Run layout tests."""
32
33 from __future__ import with_statement
34
35 import codecs
36 import errno
37 import logging
38 import optparse
39 import os
40 import signal
41 import sys
42
43 from layout_package import printing
44 from layout_package import test_runner
45
46 from webkitpy.common.system import user
47 from webkitpy.thirdparty import simplejson
48
49 import port
50
51 _log = logging.getLogger(__name__)
52
53
54 def run(port, options, args, regular_output=sys.stderr,
55         buildbot_output=sys.stdout):
56     """Run the tests.
57
58     Args:
59       port: Port object for port-specific behavior
60       options: a dictionary of command line options
61       args: a list of sub directories or files to test
62       regular_output: a stream-like object that we can send logging/debug
63           output to
64       buildbot_output: a stream-like object that we can write all output that
65           is intended to be parsed by the buildbot to
66     Returns:
67       the number of unexpected results that occurred, or -1 if there is an
68           error.
69
70     """
71     warnings = _set_up_derived_options(port, options)
72
73     printer = printing.Printer(port, options, regular_output, buildbot_output,
74         int(options.child_processes), options.experimental_fully_parallel)
75     for w in warnings:
76         _log.warning(w)
77
78     if options.help_printing:
79         printer.help_printing()
80         printer.cleanup()
81         return 0
82
83     last_unexpected_results = _gather_unexpected_results(options)
84     if options.print_last_failures:
85         printer.write("\n".join(last_unexpected_results) + "\n")
86         printer.cleanup()
87         return 0
88
89     # We wrap any parts of the run that are slow or likely to raise exceptions
90     # in a try/finally to ensure that we clean up the logging configuration.
91     num_unexpected_results = -1
92     try:
93         runner = test_runner.TestRunner(port, options, printer)
94         runner._print_config()
95
96         printer.print_update("Collecting tests ...")
97         try:
98             runner.collect_tests(args, last_unexpected_results)
99         except IOError, e:
100             if e.errno == errno.ENOENT:
101                 return -1
102             raise
103
104         printer.print_update("Parsing expectations ...")
105         if options.lint_test_files:
106             return runner.lint()
107         runner.parse_expectations(port.test_platform_name(),
108                                   options.configuration == 'Debug')
109
110         printer.print_update("Checking build ...")
111         if not port.check_build(runner.needs_http()):
112             _log.error("Build check failed")
113             return -1
114
115         result_summary = runner.set_up_run()
116         if result_summary:
117             num_unexpected_results = runner.run(result_summary)
118             runner.clean_up_run()
119             _log.debug("Testing completed, Exit status: %d" %
120                        num_unexpected_results)
121     finally:
122         printer.cleanup()
123
124     return num_unexpected_results
125
126
127 def _set_up_derived_options(port_obj, options):
128     """Sets the options values that depend on other options values."""
129     # We return a list of warnings to print after the printer is initialized.
130     warnings = []
131
132     if options.worker_model == 'old-inline':
133         if options.child_processes and int(options.child_processes) > 1:
134             warnings.append("--worker-model=old-inline overrides --child-processes")
135         options.child_processes = "1"
136     if not options.child_processes:
137         options.child_processes = os.environ.get("WEBKIT_TEST_CHILD_PROCESSES",
138                                                  str(port_obj.default_child_processes()))
139
140     if not options.configuration:
141         options.configuration = port_obj.default_configuration()
142
143     if options.pixel_tests is None:
144         options.pixel_tests = True
145
146     if not options.use_apache:
147         options.use_apache = sys.platform in ('darwin', 'linux2')
148
149     if not os.path.isabs(options.results_directory):
150         # This normalizes the path to the build dir.
151         # FIXME: how this happens is not at all obvious; this is a dumb
152         # interface and should be cleaned up.
153         options.results_directory = port_obj.results_directory()
154
155     if not options.time_out_ms:
156         if options.configuration == "Debug":
157             options.time_out_ms = str(2 * test_runner.TestRunner.DEFAULT_TEST_TIMEOUT_MS)
158         else:
159             options.time_out_ms = str(test_runner.TestRunner.DEFAULT_TEST_TIMEOUT_MS)
160
161     options.slow_time_out_ms = str(5 * int(options.time_out_ms))
162     return warnings
163
164
165 def _gather_unexpected_results(options):
166     """Returns the unexpected results from the previous run, if any."""
167     last_unexpected_results = []
168     if options.print_last_failures or options.retest_last_failures:
169         unexpected_results_filename = os.path.join(
170         options.results_directory, "unexpected_results.json")
171         with codecs.open(unexpected_results_filename, "r", "utf-8") as file:
172             results = simplejson.load(file)
173         last_unexpected_results = results['tests'].keys()
174     return last_unexpected_results
175
176
177 def _compat_shim_callback(option, opt_str, value, parser):
178     print "Ignoring unsupported option: %s" % opt_str
179
180
181 def _compat_shim_option(option_name, **kwargs):
182     return optparse.make_option(option_name, action="callback",
183         callback=_compat_shim_callback,
184         help="Ignored, for old-run-webkit-tests compat only.", **kwargs)
185
186
187 def parse_args(args=None):
188     """Provides a default set of command line args.
189
190     Returns a tuple of options, args from optparse"""
191
192     # FIXME: All of these options should be stored closer to the code which
193     # FIXME: actually uses them. configuration_options should move
194     # FIXME: to WebKitPort and be shared across all scripts.
195     configuration_options = [
196         optparse.make_option("-t", "--target", dest="configuration",
197                              help="(DEPRECATED)"),
198         # FIXME: --help should display which configuration is default.
199         optparse.make_option('--debug', action='store_const', const='Debug',
200                              dest="configuration",
201                              help='Set the configuration to Debug'),
202         optparse.make_option('--release', action='store_const',
203                              const='Release', dest="configuration",
204                              help='Set the configuration to Release'),
205         # old-run-webkit-tests also accepts -c, --configuration CONFIGURATION.
206     ]
207
208     print_options = printing.print_options()
209
210     # FIXME: These options should move onto the ChromiumPort.
211     chromium_options = [
212         optparse.make_option("--chromium", action="store_true", default=False,
213             help="use the Chromium port"),
214         optparse.make_option("--startup-dialog", action="store_true",
215             default=False, help="create a dialog on DumpRenderTree startup"),
216         optparse.make_option("--gp-fault-error-box", action="store_true",
217             default=False, help="enable Windows GP fault error box"),
218         optparse.make_option("--js-flags",
219             type="string", help="JavaScript flags to pass to tests"),
220         optparse.make_option("--stress-opt", action="store_true",
221             default=False,
222             help="Enable additional stress test to JavaScript optimization"),
223         optparse.make_option("--stress-deopt", action="store_true",
224             default=False,
225             help="Enable additional stress test to JavaScript optimization"),
226         optparse.make_option("--nocheck-sys-deps", action="store_true",
227             default=False,
228             help="Don't check the system dependencies (themes)"),
229         optparse.make_option("--use-test-shell", action="store_true",
230             default=False,
231             help="Use test_shell instead of DRT"),
232         optparse.make_option("--accelerated-compositing",
233             action="store_true",
234             help="Use hardware-accelated compositing for rendering"),
235         optparse.make_option("--no-accelerated-compositing",
236             action="store_false",
237             dest="accelerated_compositing",
238             help="Don't use hardware-accelerated compositing for rendering"),
239         optparse.make_option("--accelerated-2d-canvas",
240             action="store_true",
241             help="Use hardware-accelerated 2D Canvas calls"),
242         optparse.make_option("--no-accelerated-2d-canvas",
243             action="store_false",
244             dest="accelerated_2d_canvas",
245             help="Don't use hardware-accelerated 2D Canvas calls"),
246         optparse.make_option("--enable-hardware-gpu",
247             action="store_true",
248             default=False,
249             help="Run graphics tests on real GPU hardware vs software"),
250     ]
251
252     # Missing Mac-specific old-run-webkit-tests options:
253     # FIXME: Need: -g, --guard for guard malloc support on Mac.
254     # FIXME: Need: -l --leaks    Enable leaks checking.
255     # FIXME: Need: --sample-on-timeout Run sample on timeout
256
257     old_run_webkit_tests_compat = [
258         # NRWT doesn't generate results by default anyway.
259         _compat_shim_option("--no-new-test-results"),
260         # NRWT doesn't sample on timeout yet anyway.
261         _compat_shim_option("--no-sample-on-timeout"),
262         # FIXME: NRWT needs to support remote links eventually.
263         _compat_shim_option("--use-remote-links-to-tests"),
264     ]
265
266     results_options = [
267         # NEED for bots: --use-remote-links-to-tests Link to test files
268         # within the SVN repository in the results.
269         optparse.make_option("-p", "--pixel-tests", action="store_true",
270             dest="pixel_tests", help="Enable pixel-to-pixel PNG comparisons"),
271         optparse.make_option("--no-pixel-tests", action="store_false",
272             dest="pixel_tests", help="Disable pixel-to-pixel PNG comparisons"),
273         optparse.make_option("--tolerance",
274             help="Ignore image differences less than this percentage (some "
275                 "ports may ignore this option)", type="float"),
276         optparse.make_option("--results-directory",
277             default="layout-test-results",
278             help="Output results directory source dir, relative to Debug or "
279                  "Release"),
280         optparse.make_option("--build-directory",
281             help="Path to the directory under which build files are kept (should not include configuration)"),
282         optparse.make_option("--new-baseline", action="store_true",
283             default=False, help="Save all generated results as new baselines "
284                  "into the platform directory, overwriting whatever's "
285                  "already there."),
286         optparse.make_option("--reset-results", action="store_true",
287             default=False, help="Reset any existing baselines to the "
288                  "generated results"),
289         optparse.make_option("--no-show-results", action="store_false",
290             default=True, dest="show_results",
291             help="Don't launch a browser with results after the tests "
292                  "are done"),
293         # FIXME: We should have a helper function to do this sort of
294         # deprectated mapping and automatically log, etc.
295         optparse.make_option("--noshow-results", action="store_false",
296             dest="show_results",
297             help="Deprecated, same as --no-show-results."),
298         optparse.make_option("--no-launch-safari", action="store_false",
299             dest="show_results",
300             help="old-run-webkit-tests compat, same as --noshow-results."),
301         # old-run-webkit-tests:
302         # --[no-]launch-safari    Launch (or do not launch) Safari to display
303         #                         test results (default: launch)
304         optparse.make_option("--full-results-html", action="store_true",
305             default=False,
306             help="Show all failures in results.html, rather than only "
307                  "regressions"),
308         optparse.make_option("--clobber-old-results", action="store_true",
309             default=False, help="Clobbers test results from previous runs."),
310         optparse.make_option("--platform",
311             help="Override the platform for expected results"),
312         optparse.make_option("--no-record-results", action="store_false",
313             default=True, dest="record_results",
314             help="Don't record the results."),
315         # old-run-webkit-tests also has HTTP toggle options:
316         # --[no-]http                     Run (or do not run) http tests
317         #                                 (default: run)
318     ]
319
320     test_options = [
321         optparse.make_option("--build", dest="build",
322             action="store_true", default=True,
323             help="Check to ensure the DumpRenderTree build is up-to-date "
324                  "(default)."),
325         optparse.make_option("--no-build", dest="build",
326             action="store_false", help="Don't check to see if the "
327                                        "DumpRenderTree build is up-to-date."),
328         optparse.make_option("-n", "--dry-run", action="store_true",
329             default=False,
330             help="Do everything but actually run the tests or upload results."),
331         # old-run-webkit-tests has --valgrind instead of wrapper.
332         optparse.make_option("--wrapper",
333             help="wrapper command to insert before invocations of "
334                  "DumpRenderTree; option is split on whitespace before "
335                  "running. (Example: --wrapper='valgrind --smc-check=all')"),
336         # old-run-webkit-tests:
337         # -i|--ignore-tests               Comma-separated list of directories
338         #                                 or tests to ignore
339         optparse.make_option("--test-list", action="append",
340             help="read list of tests to run from file", metavar="FILE"),
341         # old-run-webkit-tests uses --skipped==[default|ignore|only]
342         # instead of --force:
343         optparse.make_option("--force", action="store_true", default=False,
344             help="Run all tests, even those marked SKIP in the test list"),
345         optparse.make_option("--use-apache", action="store_true",
346             default=False, help="Whether to use apache instead of lighttpd."),
347         optparse.make_option("--time-out-ms",
348             help="Set the timeout for each test"),
349         # old-run-webkit-tests calls --randomize-order --random:
350         optparse.make_option("--randomize-order", action="store_true",
351             default=False, help=("Run tests in random order (useful "
352                                 "for tracking down corruption)")),
353         optparse.make_option("--run-chunk",
354             help=("Run a specified chunk (n:l), the nth of len l, "
355                  "of the layout tests")),
356         optparse.make_option("--run-part", help=("Run a specified part (n:m), "
357                   "the nth of m parts, of the layout tests")),
358         # old-run-webkit-tests calls --batch-size: --nthly n
359         #   Restart DumpRenderTree every n tests (default: 1000)
360         optparse.make_option("--batch-size",
361             help=("Run a the tests in batches (n), after every n tests, "
362                   "DumpRenderTree is relaunched."), type="int", default=0),
363         # old-run-webkit-tests calls --run-singly: -1|--singly
364         # Isolate each test case run (implies --nthly 1 --verbose)
365         optparse.make_option("--run-singly", action="store_true",
366             default=False, help="run a separate DumpRenderTree for each test"),
367         optparse.make_option("--child-processes",
368             help="Number of DumpRenderTrees to run in parallel."),
369         # FIXME: Display default number of child processes that will run.
370         optparse.make_option("--worker-model", action="store",
371             default="old-threads", help=("controls worker model. Valid values "
372             "are 'old-inline', 'old-threads'.")),
373         optparse.make_option("--experimental-fully-parallel",
374             action="store_true", default=False,
375             help="run all tests in parallel"),
376         optparse.make_option("--exit-after-n-failures", type="int", nargs=1,
377             help="Exit after the first N failures instead of running all "
378             "tests"),
379         optparse.make_option("--exit-after-n-crashes-or-timeouts", type="int",
380             nargs=1, help="Exit after the first N crashes instead of running "
381             "all tests"),
382         # FIXME: consider: --iterations n
383         #      Number of times to run the set of tests (e.g. ABCABCABC)
384         optparse.make_option("--print-last-failures", action="store_true",
385             default=False, help="Print the tests in the last run that "
386             "had unexpected failures (or passes) and then exit."),
387         optparse.make_option("--retest-last-failures", action="store_true",
388             default=False, help="re-test the tests in the last run that "
389             "had unexpected failures (or passes)."),
390         optparse.make_option("--retry-failures", action="store_true",
391             default=True,
392             help="Re-try any tests that produce unexpected results (default)"),
393         optparse.make_option("--no-retry-failures", action="store_false",
394             dest="retry_failures",
395             help="Don't re-try any tests that produce unexpected results."),
396     ]
397
398     misc_options = [
399         optparse.make_option("--lint-test-files", action="store_true",
400         default=False, help=("Makes sure the test files parse for all "
401                             "configurations. Does not run any tests.")),
402     ]
403
404     # FIXME: Move these into json_results_generator.py
405     results_json_options = [
406         optparse.make_option("--master-name", help="The name of the buildbot master."),
407         optparse.make_option("--builder-name", default="DUMMY_BUILDER_NAME",
408             help=("The name of the builder shown on the waterfall running "
409                   "this script e.g. WebKit.")),
410         optparse.make_option("--build-name", default="DUMMY_BUILD_NAME",
411             help=("The name of the builder used in its path, e.g. "
412                   "webkit-rel.")),
413         optparse.make_option("--build-number", default="DUMMY_BUILD_NUMBER",
414             help=("The build number of the builder running this script.")),
415         optparse.make_option("--test-results-server", default="",
416             help=("If specified, upload results json files to this appengine "
417                   "server.")),
418         optparse.make_option("--upload-full-results",
419             action="store_true",
420             default=False,
421             help="If true, upload full json results to server."),
422     ]
423
424     option_list = (configuration_options + print_options +
425                    chromium_options + results_options + test_options +
426                    misc_options + results_json_options +
427                    old_run_webkit_tests_compat)
428     option_parser = optparse.OptionParser(option_list=option_list)
429
430     return option_parser.parse_args(args)
431
432
433 def main():
434     options, args = parse_args()
435     port_obj = port.get(options.platform, options)
436     return run(port_obj, options, args)
437
438
439 if '__main__' == __name__:
440     try:
441         sys.exit(main())
442     except KeyboardInterrupt:
443         # this mirrors what the shell normally does
444         sys.exit(signal.SIGINT + 128)