2011-02-01 Dirk Pranke <dpranke@chromium.org>
[WebKit.git] / Tools / Scripts / webkitpy / layout_tests / run_webkit_tests.py
1 #!/usr/bin/env python
2 # Copyright (C) 2010 Google Inc. All rights reserved.
3 # Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Szeged
4 #
5 # Redistribution and use in source and binary forms, with or without
6 # modification, are permitted provided that the following conditions are
7 # met:
8 #
9 #     * Redistributions of source code must retain the above copyright
10 # notice, this list of conditions and the following disclaimer.
11 #     * Redistributions in binary form must reproduce the above
12 # copyright notice, this list of conditions and the following disclaimer
13 # in the documentation and/or other materials provided with the
14 # distribution.
15 #     * Neither the name of Google Inc. nor the names of its
16 # contributors may be used to endorse or promote products derived from
17 # this software without specific prior written permission.
18 #
19 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30
31 """Run layout tests."""
32
33 import errno
34 import logging
35 import optparse
36 import os
37 import signal
38 import sys
39
40 from layout_package import printing
41 from layout_package import test_runner
42 from layout_package import test_runner2
43
44 from webkitpy.common.system import user
45 from webkitpy.thirdparty import simplejson
46
47 import port
48
49 _log = logging.getLogger(__name__)
50
51
52 def run(port, options, args, regular_output=sys.stderr,
53         buildbot_output=sys.stdout):
54     """Run the tests.
55
56     Args:
57       port: Port object for port-specific behavior
58       options: a dictionary of command line options
59       args: a list of sub directories or files to test
60       regular_output: a stream-like object that we can send logging/debug
61           output to
62       buildbot_output: a stream-like object that we can write all output that
63           is intended to be parsed by the buildbot to
64     Returns:
65       the number of unexpected results that occurred, or -1 if there is an
66           error.
67
68     """
69     warnings = _set_up_derived_options(port, options)
70
71     printer = printing.Printer(port, options, regular_output, buildbot_output,
72         int(options.child_processes), options.experimental_fully_parallel)
73     for w in warnings:
74         _log.warning(w)
75
76     if options.help_printing:
77         printer.help_printing()
78         printer.cleanup()
79         return 0
80
81     last_unexpected_results = _gather_unexpected_results(port._filesystem, options)
82     if options.print_last_failures:
83         printer.write("\n".join(last_unexpected_results) + "\n")
84         printer.cleanup()
85         return 0
86
87     # We wrap any parts of the run that are slow or likely to raise exceptions
88     # in a try/finally to ensure that we clean up the logging configuration.
89     num_unexpected_results = -1
90     try:
91         if options.worker_model in ('inline', 'threads', 'processes'):
92             runner = test_runner2.TestRunner2(port, options, printer)
93         else:
94             runner = test_runner.TestRunner(port, options, printer)
95
96         runner._print_config()
97
98         printer.print_update("Collecting tests ...")
99         try:
100             runner.collect_tests(args, last_unexpected_results)
101         except IOError, e:
102             if e.errno == errno.ENOENT:
103                 return -1
104             raise
105
106         if options.lint_test_files:
107             return runner.lint()
108
109         printer.print_update("Parsing expectations ...")
110         runner.parse_expectations()
111
112         printer.print_update("Checking build ...")
113         if not port.check_build(runner.needs_http()):
114             _log.error("Build check failed")
115             return -1
116
117         result_summary = runner.set_up_run()
118         if result_summary:
119             num_unexpected_results = runner.run(result_summary)
120             runner.clean_up_run()
121             _log.debug("Testing completed, Exit status: %d" %
122                        num_unexpected_results)
123     finally:
124         printer.cleanup()
125
126     return num_unexpected_results
127
128
129 def _set_up_derived_options(port_obj, options):
130     """Sets the options values that depend on other options values."""
131     # We return a list of warnings to print after the printer is initialized.
132     warnings = []
133
134     if options.worker_model is None:
135         options.worker_model = port_obj.default_worker_model()
136
137     if options.worker_model in ('inline', 'old-inline'):
138         if options.child_processes and int(options.child_processes) > 1:
139             warnings.append("--worker-model=%s overrides --child-processes" % options.worker_model)
140         options.child_processes = "1"
141     if not options.child_processes:
142         options.child_processes = os.environ.get("WEBKIT_TEST_CHILD_PROCESSES",
143                                                  str(port_obj.default_child_processes()))
144
145     if not options.configuration:
146         options.configuration = port_obj.default_configuration()
147
148     if options.pixel_tests is None:
149         options.pixel_tests = True
150
151     if not options.use_apache:
152         options.use_apache = sys.platform in ('darwin', 'linux2')
153
154     if not port_obj._filesystem.isabs(options.results_directory):
155         # This normalizes the path to the build dir.
156         # FIXME: how this happens is not at all obvious; this is a dumb
157         # interface and should be cleaned up.
158         options.results_directory = port_obj.results_directory()
159
160     if not options.time_out_ms:
161         if options.configuration == "Debug":
162             options.time_out_ms = str(2 * test_runner.TestRunner.DEFAULT_TEST_TIMEOUT_MS)
163         else:
164             options.time_out_ms = str(test_runner.TestRunner.DEFAULT_TEST_TIMEOUT_MS)
165
166     options.slow_time_out_ms = str(5 * int(options.time_out_ms))
167     return warnings
168
169
170 def _gather_unexpected_results(filesystem, options):
171     """Returns the unexpected results from the previous run, if any."""
172     last_unexpected_results = []
173     if options.print_last_failures or options.retest_last_failures:
174         unexpected_results_filename = filesystem.join(
175             options.results_directory, "unexpected_results.json")
176         if filesystem.exists(unexpected_results_filename):
177             content = filesystem.read_text_file(unexpected_results_filename)
178             results = simplejson.loads(content)
179             last_unexpected_results = results['tests'].keys()
180     return last_unexpected_results
181
182
183 def _compat_shim_callback(option, opt_str, value, parser):
184     print "Ignoring unsupported option: %s" % opt_str
185
186
187 def _compat_shim_option(option_name, **kwargs):
188     return optparse.make_option(option_name, action="callback",
189         callback=_compat_shim_callback,
190         help="Ignored, for old-run-webkit-tests compat only.", **kwargs)
191
192
193 def parse_args(args=None):
194     """Provides a default set of command line args.
195
196     Returns a tuple of options, args from optparse"""
197
198     # FIXME: All of these options should be stored closer to the code which
199     # FIXME: actually uses them. configuration_options should move
200     # FIXME: to WebKitPort and be shared across all scripts.
201     configuration_options = [
202         optparse.make_option("-t", "--target", dest="configuration",
203                              help="(DEPRECATED)"),
204         # FIXME: --help should display which configuration is default.
205         optparse.make_option('--debug', action='store_const', const='Debug',
206                              dest="configuration",
207                              help='Set the configuration to Debug'),
208         optparse.make_option('--release', action='store_const',
209                              const='Release', dest="configuration",
210                              help='Set the configuration to Release'),
211         # old-run-webkit-tests also accepts -c, --configuration CONFIGURATION.
212     ]
213
214     print_options = printing.print_options()
215
216     # FIXME: These options should move onto the ChromiumPort.
217     chromium_options = [
218         optparse.make_option("--chromium", action="store_true", default=False,
219             help="use the Chromium port"),
220         optparse.make_option("--startup-dialog", action="store_true",
221             default=False, help="create a dialog on DumpRenderTree startup"),
222         optparse.make_option("--gp-fault-error-box", action="store_true",
223             default=False, help="enable Windows GP fault error box"),
224         optparse.make_option("--js-flags",
225             type="string", help="JavaScript flags to pass to tests"),
226         optparse.make_option("--stress-opt", action="store_true",
227             default=False,
228             help="Enable additional stress test to JavaScript optimization"),
229         optparse.make_option("--stress-deopt", action="store_true",
230             default=False,
231             help="Enable additional stress test to JavaScript optimization"),
232         optparse.make_option("--nocheck-sys-deps", action="store_true",
233             default=False,
234             help="Don't check the system dependencies (themes)"),
235         optparse.make_option("--use-test-shell", action="store_true",
236             default=False,
237             help="Use test_shell instead of DRT"),
238         optparse.make_option("--accelerated-compositing",
239             action="store_true",
240             help="Use hardware-accelated compositing for rendering"),
241         optparse.make_option("--no-accelerated-compositing",
242             action="store_false",
243             dest="accelerated_compositing",
244             help="Don't use hardware-accelerated compositing for rendering"),
245         optparse.make_option("--accelerated-2d-canvas",
246             action="store_true",
247             help="Use hardware-accelerated 2D Canvas calls"),
248         optparse.make_option("--no-accelerated-2d-canvas",
249             action="store_false",
250             dest="accelerated_2d_canvas",
251             help="Don't use hardware-accelerated 2D Canvas calls"),
252         optparse.make_option("--enable-hardware-gpu",
253             action="store_true",
254             default=False,
255             help="Run graphics tests on real GPU hardware vs software"),
256     ]
257
258     # Missing Mac-specific old-run-webkit-tests options:
259     # FIXME: Need: -g, --guard for guard malloc support on Mac.
260     # FIXME: Need: -l --leaks    Enable leaks checking.
261     # FIXME: Need: --sample-on-timeout Run sample on timeout
262
263     old_run_webkit_tests_compat = [
264         # NRWT doesn't generate results by default anyway.
265         _compat_shim_option("--no-new-test-results"),
266         # NRWT doesn't sample on timeout yet anyway.
267         _compat_shim_option("--no-sample-on-timeout"),
268         # FIXME: NRWT needs to support remote links eventually.
269         _compat_shim_option("--use-remote-links-to-tests"),
270     ]
271
272     results_options = [
273         # NEED for bots: --use-remote-links-to-tests Link to test files
274         # within the SVN repository in the results.
275         optparse.make_option("-p", "--pixel-tests", action="store_true",
276             dest="pixel_tests", help="Enable pixel-to-pixel PNG comparisons"),
277         optparse.make_option("--no-pixel-tests", action="store_false",
278             dest="pixel_tests", help="Disable pixel-to-pixel PNG comparisons"),
279         optparse.make_option("--tolerance",
280             help="Ignore image differences less than this percentage (some "
281                 "ports may ignore this option)", type="float"),
282         optparse.make_option("--results-directory",
283             default="layout-test-results",
284             help="Output results directory source dir, relative to Debug or "
285                  "Release"),
286         optparse.make_option("--build-directory",
287             help="Path to the directory under which build files are kept (should not include configuration)"),
288         optparse.make_option("--new-baseline", action="store_true",
289             default=False, help="Save all generated results as new baselines "
290                  "into the platform directory, overwriting whatever's "
291                  "already there."),
292         optparse.make_option("--reset-results", action="store_true",
293             default=False, help="Reset any existing baselines to the "
294                  "generated results"),
295         optparse.make_option("--no-show-results", action="store_false",
296             default=True, dest="show_results",
297             help="Don't launch a browser with results after the tests "
298                  "are done"),
299         # FIXME: We should have a helper function to do this sort of
300         # deprectated mapping and automatically log, etc.
301         optparse.make_option("--noshow-results", action="store_false",
302             dest="show_results",
303             help="Deprecated, same as --no-show-results."),
304         optparse.make_option("--no-launch-safari", action="store_false",
305             dest="show_results",
306             help="old-run-webkit-tests compat, same as --noshow-results."),
307         # old-run-webkit-tests:
308         # --[no-]launch-safari    Launch (or do not launch) Safari to display
309         #                         test results (default: launch)
310         optparse.make_option("--full-results-html", action="store_true",
311             default=False,
312             help="Show all failures in results.html, rather than only "
313                  "regressions"),
314         optparse.make_option("--clobber-old-results", action="store_true",
315             default=False, help="Clobbers test results from previous runs."),
316         optparse.make_option("--platform",
317             help="Override the platform for expected results"),
318         optparse.make_option("--no-record-results", action="store_false",
319             default=True, dest="record_results",
320             help="Don't record the results."),
321         # old-run-webkit-tests also has HTTP toggle options:
322         # --[no-]http                     Run (or do not run) http tests
323         #                                 (default: run)
324     ]
325
326     test_options = [
327         optparse.make_option("--build", dest="build",
328             action="store_true", default=True,
329             help="Check to ensure the DumpRenderTree build is up-to-date "
330                  "(default)."),
331         optparse.make_option("--no-build", dest="build",
332             action="store_false", help="Don't check to see if the "
333                                        "DumpRenderTree build is up-to-date."),
334         optparse.make_option("-n", "--dry-run", action="store_true",
335             default=False,
336             help="Do everything but actually run the tests or upload results."),
337         # old-run-webkit-tests has --valgrind instead of wrapper.
338         optparse.make_option("--wrapper",
339             help="wrapper command to insert before invocations of "
340                  "DumpRenderTree; option is split on whitespace before "
341                  "running. (Example: --wrapper='valgrind --smc-check=all')"),
342         # old-run-webkit-tests:
343         # -i|--ignore-tests               Comma-separated list of directories
344         #                                 or tests to ignore
345         optparse.make_option("--test-list", action="append",
346             help="read list of tests to run from file", metavar="FILE"),
347         # old-run-webkit-tests uses --skipped==[default|ignore|only]
348         # instead of --force:
349         optparse.make_option("--force", action="store_true", default=False,
350             help="Run all tests, even those marked SKIP in the test list"),
351         optparse.make_option("--use-apache", action="store_true",
352             default=False, help="Whether to use apache instead of lighttpd."),
353         optparse.make_option("--time-out-ms",
354             help="Set the timeout for each test"),
355         # old-run-webkit-tests calls --randomize-order --random:
356         optparse.make_option("--randomize-order", action="store_true",
357             default=False, help=("Run tests in random order (useful "
358                                 "for tracking down corruption)")),
359         optparse.make_option("--run-chunk",
360             help=("Run a specified chunk (n:l), the nth of len l, "
361                  "of the layout tests")),
362         optparse.make_option("--run-part", help=("Run a specified part (n:m), "
363                   "the nth of m parts, of the layout tests")),
364         # old-run-webkit-tests calls --batch-size: --nthly n
365         #   Restart DumpRenderTree every n tests (default: 1000)
366         optparse.make_option("--batch-size",
367             help=("Run a the tests in batches (n), after every n tests, "
368                   "DumpRenderTree is relaunched."), type="int", default=0),
369         # old-run-webkit-tests calls --run-singly: -1|--singly
370         # Isolate each test case run (implies --nthly 1 --verbose)
371         optparse.make_option("--run-singly", action="store_true",
372             default=False, help="run a separate DumpRenderTree for each test"),
373         optparse.make_option("--child-processes",
374             help="Number of DumpRenderTrees to run in parallel."),
375         # FIXME: Display default number of child processes that will run.
376         optparse.make_option("--worker-model", action="store",
377             default=None, help=("controls worker model. Valid values are 'old-inline', "
378                                 "'old-threads', 'inline', 'threads', and 'processes'.")),
379         optparse.make_option("--experimental-fully-parallel",
380             action="store_true", default=False,
381             help="run all tests in parallel"),
382         optparse.make_option("--exit-after-n-failures", type="int", nargs=1,
383             help="Exit after the first N failures instead of running all "
384             "tests"),
385         optparse.make_option("--exit-after-n-crashes-or-timeouts", type="int",
386             nargs=1, help="Exit after the first N crashes instead of running "
387             "all tests"),
388         # FIXME: consider: --iterations n
389         #      Number of times to run the set of tests (e.g. ABCABCABC)
390         optparse.make_option("--print-last-failures", action="store_true",
391             default=False, help="Print the tests in the last run that "
392             "had unexpected failures (or passes) and then exit."),
393         optparse.make_option("--retest-last-failures", action="store_true",
394             default=False, help="re-test the tests in the last run that "
395             "had unexpected failures (or passes)."),
396         optparse.make_option("--retry-failures", action="store_true",
397             default=True,
398             help="Re-try any tests that produce unexpected results (default)"),
399         optparse.make_option("--no-retry-failures", action="store_false",
400             dest="retry_failures",
401             help="Don't re-try any tests that produce unexpected results."),
402     ]
403
404     misc_options = [
405         optparse.make_option("--lint-test-files", action="store_true",
406         default=False, help=("Makes sure the test files parse for all "
407                             "configurations. Does not run any tests.")),
408     ]
409
410     # FIXME: Move these into json_results_generator.py
411     results_json_options = [
412         optparse.make_option("--master-name", help="The name of the buildbot master."),
413         optparse.make_option("--builder-name", default="DUMMY_BUILDER_NAME",
414             help=("The name of the builder shown on the waterfall running "
415                   "this script e.g. WebKit.")),
416         optparse.make_option("--build-name", default="DUMMY_BUILD_NAME",
417             help=("The name of the builder used in its path, e.g. "
418                   "webkit-rel.")),
419         optparse.make_option("--build-number", default="DUMMY_BUILD_NUMBER",
420             help=("The build number of the builder running this script.")),
421         optparse.make_option("--test-results-server", default="",
422             help=("If specified, upload results json files to this appengine "
423                   "server.")),
424         optparse.make_option("--upload-full-results",
425             action="store_true",
426             default=False,
427             help="If true, upload full json results to server."),
428     ]
429
430     option_list = (configuration_options + print_options +
431                    chromium_options + results_options + test_options +
432                    misc_options + results_json_options +
433                    old_run_webkit_tests_compat)
434     option_parser = optparse.OptionParser(option_list=option_list)
435
436     return option_parser.parse_args(args)
437
438
439 def main():
440     options, args = parse_args()
441     port_obj = port.get(options.platform, options)
442     return run(port_obj, options, args)
443
444
445 if '__main__' == __name__:
446     try:
447         sys.exit(main())
448     except KeyboardInterrupt:
449         # this mirrors what the shell normally does
450         sys.exit(signal.SIGINT + 128)