1 # Copyright (C) 2010 Google Inc. All rights reserved.
2 # Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Szeged
3 # Copyright (C) 2011 Apple Inc. All rights reserved.
5 # Redistribution and use in source and binary forms, with or without
6 # modification, are permitted provided that the following conditions are
9 # * Redistributions of source code must retain the above copyright
10 # notice, this list of conditions and the following disclaimer.
11 # * Redistributions in binary form must reproduce the above
12 # copyright notice, this list of conditions and the following disclaimer
13 # in the documentation and/or other materials provided with the
15 # * Neither the name of Google Inc. nor the names of its
16 # contributors may be used to endorse or promote products derived from
17 # this software without specific prior written permission.
19 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
38 from webkitpy.common.host import Host
39 from webkitpy.layout_tests.controllers.manager import Manager
40 from webkitpy.port import configuration_options, platform_options
41 from webkitpy.layout_tests.views import buildbot_results
42 from webkitpy.layout_tests.views import printing
45 _log = logging.getLogger(__name__)
48 # This mirrors what the shell normally does.
49 INTERRUPTED_EXIT_STATUS = signal.SIGINT + 128
51 # This is a randomly chosen exit code that can be tested against to
52 # indicate that an unexpected exception occurred.
53 EXCEPTIONAL_EXIT_STATUS = 254
56 def main(argv, stdout, stderr):
57 options, args = parse_args(argv)
59 if options.platform and 'test' in options.platform:
60 # It's a bit lame to import mocks into real code, but this allows the user
61 # to run tests against the test platform interactively, which is useful for
62 # debugging test failures.
63 from webkitpy.common.host_mock import MockHost
68 if options.lint_test_files:
69 from webkitpy.layout_tests.lint_test_expectations import lint
70 return lint(host, options, stderr)
73 port = host.port_factory.get(options.platform, options)
74 except NotImplementedError, e:
75 # FIXME: is this the best way to handle unsupported port names?
76 print >> stderr, str(e)
77 return EXCEPTIONAL_EXIT_STATUS
80 run_details = run(port, options, args, stderr)
81 if run_details.exit_code != -1:
82 bot_printer = buildbot_results.BuildBotPrinter(stdout, options.debug_rwt_logging)
83 bot_printer.print_results(run_details)
85 return run_details.exit_code
86 except KeyboardInterrupt:
87 return INTERRUPTED_EXIT_STATUS
88 except BaseException as e:
89 if isinstance(e, Exception):
90 print >> stderr, '\n%s raised: %s' % (e.__class__.__name__, str(e))
91 traceback.print_exc(file=stderr)
92 return EXCEPTIONAL_EXIT_STATUS
96 option_group_definitions = []
98 option_group_definitions.append(("Platform options", platform_options()))
99 option_group_definitions.append(("Configuration options", configuration_options()))
100 option_group_definitions.append(("Printing Options", printing.print_options()))
102 option_group_definitions.append(("EFL-specific Options", [
103 optparse.make_option("--webprocess-cmd-prefix", type="string",
104 default=False, help="Prefix used when spawning the Web process (Debug mode only)"),
107 option_group_definitions.append(("WebKit Options", [
108 optparse.make_option("--gc-between-tests", action="store_true", default=False,
109 help="Force garbage collection between each test"),
110 optparse.make_option("--complex-text", action="store_true", default=False,
111 help="Use the complex text code path for all text (Mac OS X and Windows only)"),
112 optparse.make_option("-l", "--leaks", action="store_true", default=False,
113 help="Enable leaks checking (Mac OS X and Gtk+ only)"),
114 optparse.make_option("-g", "--guard-malloc", action="store_true", default=False,
115 help="Enable Guard Malloc (Mac OS X only)"),
116 optparse.make_option("--threaded", action="store_true", default=False,
117 help="Run a concurrent JavaScript thread with each test"),
118 optparse.make_option("--webkit-test-runner", "-2", action="store_true",
119 help="Use WebKitTestRunner rather than DumpRenderTree."),
120 # FIXME: We should merge this w/ --build-directory and only have one flag.
121 optparse.make_option("--root", action="store",
122 help="Path to a directory containing the executables needed to run tests."),
125 option_group_definitions.append(("Results Options", [
126 optparse.make_option("-p", "--pixel", "--pixel-tests", action="store_true",
127 dest="pixel_tests", help="Enable pixel-to-pixel PNG comparisons"),
128 optparse.make_option("--no-pixel", "--no-pixel-tests", action="store_false",
129 dest="pixel_tests", help="Disable pixel-to-pixel PNG comparisons"),
130 optparse.make_option("--no-sample-on-timeout", action="store_false",
131 dest="sample_on_timeout", help="Don't run sample on timeout (Mac OS X only)"),
132 optparse.make_option("--no-ref-tests", action="store_true",
133 dest="no_ref_tests", help="Skip all ref tests"),
134 optparse.make_option("--tolerance",
135 help="Ignore image differences less than this percentage (some "
136 "ports may ignore this option)", type="float"),
137 optparse.make_option("--results-directory", help="Location of test results"),
138 optparse.make_option("--build-directory",
139 help="Path to the directory under which build files are kept (should not include configuration)"),
140 optparse.make_option("--add-platform-exceptions", action="store_true", default=False,
141 help="Save generated results into the *most-specific-platform* directory rather than the *generic-platform* directory"),
142 optparse.make_option("--new-baseline", action="store_true",
143 default=False, help="Save generated results as new baselines "
144 "into the *most-specific-platform* directory, overwriting whatever's "
145 "already there. Equivalent to --reset-results --add-platform-exceptions"),
146 optparse.make_option("--reset-results", action="store_true",
147 default=False, help="Reset expectations to the "
148 "generated results in their existing location."),
149 optparse.make_option("--no-new-test-results", action="store_false",
150 dest="new_test_results", default=True,
151 help="Don't create new baselines when no expected results exist"),
153 #FIXME: we should support a comma separated list with --pixel-test-directory as well.
154 optparse.make_option("--pixel-test-directory", action="append", default=[], dest="pixel_test_directories",
155 help="A directory where it is allowed to execute tests as pixel tests. "
156 "Specify multiple times to add multiple directories. "
157 "This option implies --pixel-tests. If specified, only those tests "
158 "will be executed as pixel tests that are located in one of the "
159 "directories enumerated with the option. Some ports may ignore this "
160 "option while others can have a default value that can be overridden here."),
162 optparse.make_option("--skip-failing-tests", action="store_true",
163 default=False, help="Skip tests that are expected to fail. "
164 "Note: When using this option, you might miss new crashes "
166 optparse.make_option("--additional-drt-flag", action="append",
167 default=[], help="Additional command line flag to pass to DumpRenderTree "
168 "Specify multiple times to add multiple flags."),
169 optparse.make_option("--driver-name", type="string",
170 help="Alternative DumpRenderTree binary to use"),
171 optparse.make_option("--additional-platform-directory", action="append",
172 default=[], help="Additional directory where to look for test "
173 "baselines (will take precendence over platform baselines). "
174 "Specify multiple times to add multiple search path entries."),
175 optparse.make_option("--additional-expectations", action="append", default=[],
176 help="Path to a test_expectations file that will override previous expectations. "
177 "Specify multiple times for multiple sets of overrides."),
178 optparse.make_option("--compare-port", action="store", default=None,
179 help="Use the specified port's baselines first"),
180 optparse.make_option("--no-show-results", action="store_false",
181 default=True, dest="show_results",
182 help="Don't launch a browser with results after the tests "
184 optparse.make_option("--full-results-html", action="store_true",
186 help="Show all failures in results.html, rather than only regressions"),
187 optparse.make_option("--clobber-old-results", action="store_true",
188 default=False, help="Clobbers test results from previous runs."),
189 optparse.make_option("--http", action="store_true", dest="http",
190 default=True, help="Run HTTP and WebSocket tests (default)"),
191 optparse.make_option("--no-http", action="store_false", dest="http",
192 help="Don't run HTTP and WebSocket tests"),
193 optparse.make_option("--ignore-metrics", action="store_true", dest="ignore_metrics",
194 default=False, help="Ignore rendering metrics related information from test "
195 "output, only compare the structure of the rendertree."),
196 optparse.make_option("--nocheck-sys-deps", action="store_true",
198 help="Don't check the system dependencies (themes)"),
199 optparse.make_option("--nojava", action="store_true",
201 help="Don't build java support files"),
204 option_group_definitions.append(("Testing Options", [
205 optparse.make_option("--build", dest="build",
206 action="store_true", default=True,
207 help="Check to ensure the DumpRenderTree build is up-to-date "
209 optparse.make_option("--no-build", dest="build",
210 action="store_false", help="Don't check to see if the "
211 "DumpRenderTree build is up-to-date."),
212 optparse.make_option("-n", "--dry-run", action="store_true",
214 help="Do everything but actually run the tests or upload results."),
215 optparse.make_option("--wrapper",
216 help="wrapper command to insert before invocations of "
217 "DumpRenderTree; option is split on whitespace before "
218 "running. (Example: --wrapper='valgrind --smc-check=all')"),
219 optparse.make_option("-i", "--ignore-tests", action="append", default=[],
220 help="directories or test to ignore (may specify multiple times)"),
221 optparse.make_option("--test-list", action="append",
222 help="read list of tests to run from file", metavar="FILE"),
223 optparse.make_option("--skipped", action="store", default="default",
224 help=("control how tests marked SKIP are run. "
225 "'default' == Skip tests unless explicitly listed on the command line, "
226 "'ignore' == Run them anyway, "
227 "'only' == only run the SKIP tests, "
228 "'always' == always skip, even if listed on the command line.")),
229 optparse.make_option("--force", dest="skipped", action="store_const", const='ignore',
230 help="Run all tests, even those marked SKIP in the test list (same as --skipped=ignore)"),
231 optparse.make_option("--time-out-ms",
232 help="Set the timeout for each test"),
233 optparse.make_option("--order", action="store", default="natural",
234 help=("determine the order in which the test cases will be run. "
235 "'none' == use the order in which the tests were listed either in arguments or test list, "
236 "'natural' == use the natural order (default), "
237 "'random' == randomize the test order.")),
238 optparse.make_option("--run-chunk",
239 help=("Run a specified chunk (n:l), the nth of len l, "
240 "of the layout tests")),
241 optparse.make_option("--run-part", help=("Run a specified part (n:m), "
242 "the nth of m parts, of the layout tests")),
243 optparse.make_option("--batch-size",
244 help=("Run a the tests in batches (n), after every n tests, "
245 "DumpRenderTree is relaunched."), type="int", default=None),
246 optparse.make_option("--run-singly", action="store_true",
247 default=False, help="run a separate DumpRenderTree for each test (implies --verbose)"),
248 optparse.make_option("--child-processes",
249 help="Number of DumpRenderTrees to run in parallel."),
250 # FIXME: Display default number of child processes that will run.
251 optparse.make_option("-f", "--fully-parallel", action="store_true",
252 help="run all tests in parallel"),
253 optparse.make_option("--exit-after-n-failures", type="int", default=None,
254 help="Exit after the first N failures instead of running all "
256 optparse.make_option("--exit-after-n-crashes-or-timeouts", type="int",
257 default=None, help="Exit after the first N crashes instead of "
258 "running all tests"),
259 optparse.make_option("--iterations", type="int", default=1, help="Number of times to run the set of tests (e.g. ABCABCABC)"),
260 optparse.make_option("--repeat-each", type="int", default=1, help="Number of times to run each test (e.g. AAABBBCCC)"),
261 optparse.make_option("--retry-failures", action="store_true",
263 help="Re-try any tests that produce unexpected results (default)"),
264 optparse.make_option("--no-retry-failures", action="store_false",
265 dest="retry_failures",
266 help="Don't re-try any tests that produce unexpected results."),
267 optparse.make_option("--max-locked-shards", type="int", default=0,
268 help="Set the maximum number of locked shards"),
269 optparse.make_option("--additional-env-var", type="string", action="append", default=[],
270 help="Passes that environment variable to the tests (--additional-env-var=NAME=VALUE)"),
271 optparse.make_option("--profile", action="store_true",
272 help="Output per-test profile information."),
273 optparse.make_option("--profiler", action="store",
274 help="Output per-test profile information, using the specified profiler."),
277 option_group_definitions.append(("Miscellaneous Options", [
278 optparse.make_option("--lint-test-files", action="store_true",
279 default=False, help=("Makes sure the test files parse for all "
280 "configurations. Does not run any tests.")),
283 # FIXME: Move these into json_results_generator.py
284 option_group_definitions.append(("Result JSON Options", [
285 optparse.make_option("--master-name", help="The name of the buildbot master."),
286 optparse.make_option("--builder-name", default="",
287 help=("The name of the builder shown on the waterfall running this script. e.g. Apple MountainLion Release WK2 (Tests).")),
288 optparse.make_option("--build-name", default="DUMMY_BUILD_NAME",
289 help=("The name of the builder used in its path, e.g. webkit-rel.")),
290 optparse.make_option("--build-slave", default="DUMMY_BUILD_SLAVE",
291 help=("The name of the buildslave used. e.g. apple-macpro-6.")),
292 optparse.make_option("--build-number", default="DUMMY_BUILD_NUMBER",
293 help=("The build number of the builder running this script.")),
294 optparse.make_option("--got-revision", default="",
295 help=("The revision number. e.g. 12345")),
296 optparse.make_option("--test-results-server", default="",
297 help=("If specified, upload results json files to this appengine server.")),
298 optparse.make_option("--results-server-host", default="",
299 help=("If specified, upload results JSON file to this results server.")),
302 option_parser = optparse.OptionParser()
304 for group_name, group_options in option_group_definitions:
305 option_group = optparse.OptionGroup(option_parser, group_name)
306 option_group.add_options(group_options)
307 option_parser.add_option_group(option_group)
309 return option_parser.parse_args(args)
312 def _set_up_derived_options(port, options):
313 """Sets the options values that depend on other options values."""
314 if not options.child_processes:
315 options.child_processes = os.environ.get("WEBKIT_TEST_CHILD_PROCESSES",
316 str(port.default_child_processes()))
317 if not options.max_locked_shards:
318 options.max_locked_shards = int(os.environ.get("WEBKIT_TEST_MAX_LOCKED_SHARDS",
319 str(port.default_max_locked_shards())))
321 if not options.configuration:
322 options.configuration = port.default_configuration()
324 if options.pixel_tests is None:
325 options.pixel_tests = port.default_pixel_tests()
327 if not options.time_out_ms:
328 options.time_out_ms = str(port.default_timeout_ms())
330 options.slow_time_out_ms = str(5 * int(options.time_out_ms))
332 if options.additional_platform_directory:
333 additional_platform_directories = []
334 for path in options.additional_platform_directory:
335 additional_platform_directories.append(port.host.filesystem.abspath(path))
336 options.additional_platform_directory = additional_platform_directories
338 if not options.http and options.skipped in ('ignore', 'only'):
339 _log.warning("--force/--skipped=%s overrides --no-http." % (options.skipped))
342 if options.ignore_metrics and (options.new_baseline or options.reset_results):
343 _log.warning("--ignore-metrics has no effect with --new-baselines or with --reset-results")
345 if options.new_baseline:
346 options.reset_results = True
347 options.add_platform_exceptions = True
349 if options.pixel_test_directories:
350 options.pixel_tests = True
351 varified_dirs = set()
352 pixel_test_directories = options.pixel_test_directories
353 for directory in pixel_test_directories:
354 # FIXME: we should support specifying the directories all the ways we support it for additional
355 # arguments specifying which tests and directories to run. We should also move the logic for that
357 filesystem = port.host.filesystem
358 if not filesystem.isdir(filesystem.join(port.layout_tests_dir(), directory)):
359 _log.warning("'%s' was passed to --pixel-test-directories, which doesn't seem to be a directory" % str(directory))
361 varified_dirs.add(directory)
363 options.pixel_test_directories = list(varified_dirs)
365 if options.run_singly:
366 options.verbose = True
369 def run(port, options, args, logging_stream):
370 logger = logging.getLogger()
371 logger.setLevel(logging.DEBUG if options.debug_rwt_logging else logging.INFO)
374 printer = printing.Printer(port, options, logging_stream, logger=logger)
376 _set_up_derived_options(port, options)
377 manager = Manager(port, options, printer)
378 printer.print_config(port.results_directory())
380 run_details = manager.run(args)
381 _log.debug("Testing completed, Exit status: %d" % run_details.exit_code)
386 if __name__ == '__main__':
387 sys.exit(main(sys.argv[1:], sys.stdout, sys.stderr))