Unreviewed. Update W3C WebDriver imported tests.
[WebKit-https.git] / WebDriverTests / imported / w3c / tools / wptrunner / wptrunner / wptcommandline.py
1 import argparse
2 import ast
3 import os
4 import sys
5 from collections import OrderedDict
6 from distutils.spawn import find_executable
7
8 import config
9 import wpttest
10 import formatters
11
12
13 def abs_path(path):
14     return os.path.abspath(os.path.expanduser(path))
15
16
17 def url_or_path(path):
18     import urlparse
19
20     parsed = urlparse.urlparse(path)
21     if len(parsed.scheme) > 2:
22         return path
23     else:
24         return abs_path(path)
25
26
27 def require_arg(kwargs, name, value_func=None):
28     if value_func is None:
29         value_func = lambda x: x is not None
30
31     if not name in kwargs or not value_func(kwargs[name]):
32         print >> sys.stderr, "Missing required argument %s" % name
33         sys.exit(1)
34
35
36 def create_parser(product_choices=None):
37     from mozlog import commandline
38
39     import products
40
41     if product_choices is None:
42         config_data = config.load()
43         product_choices = products.products_enabled(config_data)
44
45     parser = argparse.ArgumentParser(description="""Runner for web-platform-tests tests.""",
46                                      usage="""%(prog)s [OPTION]... [TEST]...
47
48 TEST is either the full path to a test file to run, or the URL of a test excluding
49 scheme host and port.""")
50     parser.add_argument("--manifest-update", action="store_true", default=None,
51                         help="Regenerate the test manifest.")
52     parser.add_argument("--no-manifest-update", action="store_false", dest="manifest_update",
53                         help="Prevent regeneration of the test manifest.")
54     parser.add_argument("--manifest-download", action="store_true", default=None,
55                         help="Attempt to download a preexisting manifest when updating.")
56
57     parser.add_argument("--timeout-multiplier", action="store", type=float, default=None,
58                         help="Multiplier relative to standard test timeout to use")
59     parser.add_argument("--run-by-dir", type=int, nargs="?", default=False,
60                         help="Split run into groups by directories. With a parameter,"
61                         "limit the depth of splits e.g. --run-by-dir=1 to split by top-level"
62                         "directory")
63     parser.add_argument("--processes", action="store", type=int, default=None,
64                         help="Number of simultaneous processes to use")
65
66     parser.add_argument("--no-capture-stdio", action="store_true", default=False,
67                         help="Don't capture stdio and write to logging")
68
69     mode_group = parser.add_argument_group("Mode")
70     mode_group.add_argument("--list-test-groups", action="store_true",
71                             default=False,
72                             help="List the top level directories containing tests that will run.")
73     mode_group.add_argument("--list-disabled", action="store_true",
74                             default=False,
75                             help="List the tests that are disabled on the current platform")
76     mode_group.add_argument("--list-tests", action="store_true",
77                             default=False,
78                             help="List all tests that will run")
79     mode_group.add_argument("--verify", action="store_true",
80                             default=False,
81                             help="Run a stability check on the selected tests")
82     mode_group.add_argument("--verify-log-full", action="store_true",
83                             default=False,
84                             help="Output per-iteration test results when running verify")
85
86     test_selection_group = parser.add_argument_group("Test Selection")
87     test_selection_group.add_argument("--test-types", action="store",
88                                       nargs="*", default=wpttest.enabled_tests,
89                                       choices=wpttest.enabled_tests,
90                                       help="Test types to run")
91     test_selection_group.add_argument("--include", action="append",
92                                       help="URL prefix to include")
93     test_selection_group.add_argument("--exclude", action="append",
94                                       help="URL prefix to exclude")
95     test_selection_group.add_argument("--include-manifest", type=abs_path,
96                                       help="Path to manifest listing tests to include")
97     test_selection_group.add_argument("--skip-timeout", action="store_true",
98                                       help="Skip tests that are expected to time out")
99     test_selection_group.add_argument("--tag", action="append", dest="tags",
100                                       help="Labels applied to tests to include in the run. Labels starting dir: are equivalent to top-level directories.")
101
102     debugging_group = parser.add_argument_group("Debugging")
103     debugging_group.add_argument('--debugger', const="__default__", nargs="?",
104                                  help="run under a debugger, e.g. gdb or valgrind")
105     debugging_group.add_argument('--debugger-args', help="arguments to the debugger")
106     debugging_group.add_argument("--rerun", action="store", type=int, default=1,
107                                  help="Number of times to re run each test without restarts")
108     debugging_group.add_argument("--repeat", action="store", type=int, default=1,
109                                  help="Number of times to run the tests, restarting between each run")
110     debugging_group.add_argument("--repeat-until-unexpected", action="store_true", default=None,
111                                  help="Run tests in a loop until one returns an unexpected result")
112     debugging_group.add_argument('--pause-after-test', action="store_true", default=None,
113                                  help="Halt the test runner after each test (this happens by default if only a single test is run)")
114     debugging_group.add_argument('--no-pause-after-test', dest="pause_after_test", action="store_false",
115                                  help="Don't halt the test runner irrespective of the number of tests run")
116
117     debugging_group.add_argument('--pause-on-unexpected', action="store_true",
118                                  help="Halt the test runner when an unexpected result is encountered")
119     debugging_group.add_argument('--no-restart-on-unexpected', dest="restart_on_unexpected",
120                                  default=True, action="store_false",
121                                  help="Don't restart on an unexpected result")
122
123     debugging_group.add_argument("--symbols-path", action="store", type=url_or_path,
124                                  help="Path or url to symbols file used to analyse crash minidumps.")
125     debugging_group.add_argument("--stackwalk-binary", action="store", type=abs_path,
126                                  help="Path to stackwalker program used to analyse minidumps.")
127
128     debugging_group.add_argument("--pdb", action="store_true",
129                                  help="Drop into pdb on python exception")
130
131     config_group = parser.add_argument_group("Configuration")
132     config_group.add_argument("--binary", action="store",
133                               type=abs_path, help="Binary to run tests against")
134     config_group.add_argument('--binary-arg',
135                               default=[], action="append", dest="binary_args",
136                               help="Extra argument for the binary")
137     config_group.add_argument("--webdriver-binary", action="store", metavar="BINARY",
138                               type=abs_path, help="WebDriver server binary to use")
139     config_group.add_argument('--webdriver-arg',
140                               default=[], action="append", dest="webdriver_args",
141                               help="Extra argument for the WebDriver binary")
142
143     config_group.add_argument("--metadata", action="store", type=abs_path, dest="metadata_root",
144                               help="Path to root directory containing test metadata"),
145     config_group.add_argument("--tests", action="store", type=abs_path, dest="tests_root",
146                               help="Path to root directory containing test files"),
147     config_group.add_argument("--run-info", action="store", type=abs_path,
148                               help="Path to directory containing extra json files to add to run info")
149     config_group.add_argument("--product", action="store", choices=product_choices,
150                               default=None, help="Browser against which to run tests")
151     config_group.add_argument("--config", action="store", type=abs_path, dest="config",
152                               help="Path to config file")
153     config_group.add_argument("--install-fonts", action="store_true",
154                               default=None,
155                               help="Allow the wptrunner to install fonts on your system")
156     config_group.add_argument("--font-dir", action="store", type=abs_path, dest="font_dir",
157                               help="Path to local font installation directory", default=None)
158
159     build_type = parser.add_mutually_exclusive_group()
160     build_type.add_argument("--debug-build", dest="debug", action="store_true",
161                             default=None,
162                             help="Build is a debug build (overrides any mozinfo file)")
163     build_type.add_argument("--release-build", dest="debug", action="store_false",
164                             default=None,
165                             help="Build is a release (overrides any mozinfo file)")
166
167
168     chunking_group = parser.add_argument_group("Test Chunking")
169     chunking_group.add_argument("--total-chunks", action="store", type=int, default=1,
170                                 help="Total number of chunks to use")
171     chunking_group.add_argument("--this-chunk", action="store", type=int, default=1,
172                                 help="Chunk number to run")
173     chunking_group.add_argument("--chunk-type", action="store", choices=["none", "equal_time", "hash", "dir_hash"],
174                                 default=None, help="Chunking type to use")
175
176     ssl_group = parser.add_argument_group("SSL/TLS")
177     ssl_group.add_argument("--ssl-type", action="store", default=None,
178                         choices=["openssl", "pregenerated", "none"],
179                         help="Type of ssl support to enable (running without ssl may lead to spurious errors)")
180
181     ssl_group.add_argument("--openssl-binary", action="store",
182                         help="Path to openssl binary", default="openssl")
183     ssl_group.add_argument("--certutil-binary", action="store",
184                         help="Path to certutil binary for use with Firefox + ssl")
185
186     ssl_group.add_argument("--ca-cert-path", action="store", type=abs_path,
187                         help="Path to ca certificate when using pregenerated ssl certificates")
188     ssl_group.add_argument("--host-key-path", action="store", type=abs_path,
189                         help="Path to host private key when using pregenerated ssl certificates")
190     ssl_group.add_argument("--host-cert-path", action="store", type=abs_path,
191                         help="Path to host certificate when using pregenerated ssl certificates")
192
193     gecko_group = parser.add_argument_group("Gecko-specific")
194     gecko_group.add_argument("--prefs-root", dest="prefs_root", action="store", type=abs_path,
195                              help="Path to the folder containing browser prefs")
196     gecko_group.add_argument("--disable-e10s", dest="gecko_e10s", action="store_false", default=True,
197                              help="Run tests without electrolysis preferences")
198     gecko_group.add_argument("--stackfix-dir", dest="stackfix_dir", action="store",
199                              help="Path to directory containing assertion stack fixing scripts")
200     gecko_group.add_argument("--setpref", dest="extra_prefs", action='append',
201                              default=[], metavar="PREF=VALUE",
202                              help="Defines an extra user preference (overrides those in prefs_root)")
203     gecko_group.add_argument("--leak-check", dest="leak_check", action="store_true",
204                              help="Enable leak checking")
205     gecko_group.add_argument("--stylo-threads", action="store", type=int, default=1,
206                              help="Number of parallel threads to use for stylo")
207     gecko_group.add_argument("--reftest-internal", dest="reftest_internal", action="store_true",
208                              default=None, help="Enable reftest runner implemented inside Marionette")
209     gecko_group.add_argument("--reftest-external", dest="reftest_internal", action="store_false",
210                              help="Disable reftest runner implemented inside Marionette")
211     gecko_group.add_argument("--reftest-screenshot", dest="reftest_screenshot", action="store",
212                              choices=["always", "fail", "unexpected"], default="unexpected",
213                              help="With --reftest-internal, when to take a screenshot")
214     gecko_group.add_argument("--chaos", dest="chaos_mode_flags", action="store",
215                              nargs="?", const=0xFFFFFFFF, type=int,
216                              help="Enable chaos mode with the specified feature flag "
217                              "(see http://searchfox.org/mozilla-central/source/mfbt/ChaosMode.h for "
218                              "details). If no value is supplied, all features are activated")
219
220     servo_group = parser.add_argument_group("Servo-specific")
221     servo_group.add_argument("--user-stylesheet",
222                              default=[], action="append", dest="user_stylesheets",
223                              help="Inject a user CSS stylesheet into every test.")
224
225     sauce_group = parser.add_argument_group("Sauce Labs-specific")
226     sauce_group.add_argument("--sauce-browser", dest="sauce_browser",
227                              help="Sauce Labs browser name")
228     sauce_group.add_argument("--sauce-platform", dest="sauce_platform",
229                              help="Sauce Labs OS platform")
230     sauce_group.add_argument("--sauce-version", dest="sauce_version",
231                              help="Sauce Labs browser version")
232     sauce_group.add_argument("--sauce-build", dest="sauce_build",
233                              help="Sauce Labs build identifier")
234     sauce_group.add_argument("--sauce-tags", dest="sauce_tags", nargs="*",
235                              help="Sauce Labs identifying tag", default=[])
236     sauce_group.add_argument("--sauce-tunnel-id", dest="sauce_tunnel_id",
237                              help="Sauce Connect tunnel identifier")
238     sauce_group.add_argument("--sauce-user", dest="sauce_user",
239                              help="Sauce Labs user name")
240     sauce_group.add_argument("--sauce-key", dest="sauce_key",
241                              default=os.environ.get("SAUCE_ACCESS_KEY"),
242                              help="Sauce Labs access key")
243     sauce_group.add_argument("--sauce-connect-binary",
244                              dest="sauce_connect_binary",
245                              help="Path to Sauce Connect binary")
246
247     parser.add_argument("test_list", nargs="*",
248                         help="List of URLs for tests to run, or paths including tests to run. "
249                              "(equivalent to --include)")
250
251     commandline.log_formatters["wptreport"] = (formatters.WptreportFormatter, "wptreport format")
252
253     commandline.add_logging_group(parser)
254     return parser
255
256
257 def set_from_config(kwargs):
258     if kwargs["config"] is None:
259         config_path = config.path()
260     else:
261         config_path = kwargs["config"]
262
263     kwargs["config_path"] = config_path
264
265     kwargs["config"] = config.read(kwargs["config_path"])
266
267     keys = {"paths": [("prefs", "prefs_root", True),
268                       ("run_info", "run_info", True)],
269             "web-platform-tests": [("remote_url", "remote_url", False),
270                                    ("branch", "branch", False),
271                                    ("sync_path", "sync_path", True)],
272             "SSL": [("openssl_binary", "openssl_binary", True),
273                     ("certutil_binary", "certutil_binary", True),
274                     ("ca_cert_path", "ca_cert_path", True),
275                     ("host_cert_path", "host_cert_path", True),
276                     ("host_key_path", "host_key_path", True)]}
277
278     for section, values in keys.iteritems():
279         for config_value, kw_value, is_path in values:
280             if kw_value in kwargs and kwargs[kw_value] is None:
281                 if not is_path:
282                     new_value = kwargs["config"].get(section, config.ConfigDict({})).get(config_value)
283                 else:
284                     new_value = kwargs["config"].get(section, config.ConfigDict({})).get_path(config_value)
285                 kwargs[kw_value] = new_value
286
287     kwargs["test_paths"] = get_test_paths(kwargs["config"])
288
289     if kwargs["tests_root"]:
290         if "/" not in kwargs["test_paths"]:
291             kwargs["test_paths"]["/"] = {}
292         kwargs["test_paths"]["/"]["tests_path"] = kwargs["tests_root"]
293
294     if kwargs["metadata_root"]:
295         if "/" not in kwargs["test_paths"]:
296             kwargs["test_paths"]["/"] = {}
297         kwargs["test_paths"]["/"]["metadata_path"] = kwargs["metadata_root"]
298
299     kwargs["suite_name"] = kwargs["config"].get("web-platform-tests", {}).get("name", "web-platform-tests")
300
301
302 def get_test_paths(config):
303     # Set up test_paths
304     test_paths = OrderedDict()
305
306     for section in config.iterkeys():
307         if section.startswith("manifest:"):
308             manifest_opts = config.get(section)
309             url_base = manifest_opts.get("url_base", "/")
310             test_paths[url_base] = {
311                 "tests_path": manifest_opts.get_path("tests"),
312                 "metadata_path": manifest_opts.get_path("metadata")}
313
314     return test_paths
315
316
317 def exe_path(name):
318     if name is None:
319         return
320
321     path = find_executable(name)
322     if path and os.access(path, os.X_OK):
323         return path
324     else:
325         return None
326
327
328 def check_args(kwargs):
329     set_from_config(kwargs)
330
331     for test_paths in kwargs["test_paths"].itervalues():
332         if not ("tests_path" in test_paths and
333                 "metadata_path" in test_paths):
334             print "Fatal: must specify both a test path and metadata path"
335             sys.exit(1)
336         for key, path in test_paths.iteritems():
337             name = key.split("_", 1)[0]
338
339             if not os.path.exists(path):
340                 print "Fatal: %s path %s does not exist" % (name, path)
341                 sys.exit(1)
342
343             if not os.path.isdir(path):
344                 print "Fatal: %s path %s is not a directory" % (name, path)
345                 sys.exit(1)
346
347     if kwargs["product"] is None:
348         kwargs["product"] = "firefox"
349
350     if "sauce" in kwargs["product"]:
351         kwargs["pause_after_test"] = False
352
353     if kwargs["test_list"]:
354         if kwargs["include"] is not None:
355             kwargs["include"].extend(kwargs["test_list"])
356         else:
357             kwargs["include"] = kwargs["test_list"]
358
359     if kwargs["run_info"] is None:
360         kwargs["run_info"] = kwargs["config_path"]
361
362     if kwargs["this_chunk"] > 1:
363         require_arg(kwargs, "total_chunks", lambda x: x >= kwargs["this_chunk"])
364
365     if kwargs["chunk_type"] is None:
366         if kwargs["total_chunks"] > 1:
367             kwargs["chunk_type"] = "dir_hash"
368         else:
369             kwargs["chunk_type"] = "none"
370
371     if kwargs["processes"] is None:
372         kwargs["processes"] = 1
373
374     if kwargs["debugger"] is not None:
375         import mozdebug
376         if kwargs["debugger"] == "__default__":
377             kwargs["debugger"] = mozdebug.get_default_debugger_name()
378         debug_info = mozdebug.get_debugger_info(kwargs["debugger"],
379                                                 kwargs["debugger_args"])
380         if debug_info and debug_info.interactive:
381             if kwargs["processes"] != 1:
382                 kwargs["processes"] = 1
383             kwargs["no_capture_stdio"] = True
384         kwargs["debug_info"] = debug_info
385     else:
386         kwargs["debug_info"] = None
387
388     if kwargs["binary"] is not None:
389         if not os.path.exists(kwargs["binary"]):
390             print >> sys.stderr, "Binary path %s does not exist" % kwargs["binary"]
391             sys.exit(1)
392
393     if kwargs["ssl_type"] is None:
394         if None not in (kwargs["ca_cert_path"], kwargs["host_cert_path"], kwargs["host_key_path"]):
395             kwargs["ssl_type"] = "pregenerated"
396         elif exe_path(kwargs["openssl_binary"]) is not None:
397             kwargs["ssl_type"] = "openssl"
398         else:
399             kwargs["ssl_type"] = "none"
400
401     if kwargs["ssl_type"] == "pregenerated":
402         require_arg(kwargs, "ca_cert_path", lambda x:os.path.exists(x))
403         require_arg(kwargs, "host_cert_path", lambda x:os.path.exists(x))
404         require_arg(kwargs, "host_key_path", lambda x:os.path.exists(x))
405
406     elif kwargs["ssl_type"] == "openssl":
407         path = exe_path(kwargs["openssl_binary"])
408         if path is None:
409             print >> sys.stderr, "openssl-binary argument missing or not a valid executable"
410             sys.exit(1)
411         kwargs["openssl_binary"] = path
412
413     if kwargs["ssl_type"] != "none" and kwargs["product"] == "firefox" and kwargs["certutil_binary"]:
414         path = exe_path(kwargs["certutil_binary"])
415         if path is None:
416             print >> sys.stderr, "certutil-binary argument missing or not a valid executable"
417             sys.exit(1)
418         kwargs["certutil_binary"] = path
419
420     if kwargs['extra_prefs']:
421         missing = any('=' not in prefarg for prefarg in kwargs['extra_prefs'])
422         if missing:
423             print >> sys.stderr, "Preferences via --setpref must be in key=value format"
424             sys.exit(1)
425         kwargs['extra_prefs'] = [tuple(prefarg.split('=', 1)) for prefarg in
426                                  kwargs['extra_prefs']]
427
428     if kwargs["reftest_internal"] is None:
429         # Default to the internal reftest implementation on Linux and OSX
430         kwargs["reftest_internal"] = sys.platform.startswith("linux") or sys.platform.startswith("darwin")
431
432     return kwargs
433
434
435 def check_args_update(kwargs):
436     set_from_config(kwargs)
437
438     if kwargs["product"] is None:
439         kwargs["product"] = "firefox"
440     if kwargs["patch"] is None:
441         kwargs["patch"] = kwargs["sync"]
442
443     for item in kwargs["run_log"]:
444         if os.path.isdir(item):
445             print >> sys.stderr, "Log file %s is a directory" % item
446             sys.exit(1)
447
448     return kwargs
449
450
451 def create_parser_update(product_choices=None):
452     from mozlog.structured import commandline
453
454     import products
455
456     if product_choices is None:
457         config_data = config.load()
458         product_choices = products.products_enabled(config_data)
459
460     parser = argparse.ArgumentParser("web-platform-tests-update",
461                                      description="Update script for web-platform-tests tests.")
462     parser.add_argument("--product", action="store", choices=product_choices,
463                         default=None, help="Browser for which metadata is being updated")
464     parser.add_argument("--config", action="store", type=abs_path, help="Path to config file")
465     parser.add_argument("--metadata", action="store", type=abs_path, dest="metadata_root",
466                         help="Path to the folder containing test metadata"),
467     parser.add_argument("--tests", action="store", type=abs_path, dest="tests_root",
468                         help="Path to web-platform-tests"),
469     parser.add_argument("--sync-path", action="store", type=abs_path,
470                         help="Path to store git checkout of web-platform-tests during update"),
471     parser.add_argument("--remote_url", action="store",
472                         help="URL of web-platfrom-tests repository to sync against"),
473     parser.add_argument("--branch", action="store", type=abs_path,
474                         help="Remote branch to sync against")
475     parser.add_argument("--rev", action="store", help="Revision to sync to")
476     parser.add_argument("--patch", action="store_true", dest="patch", default=None,
477                         help="Create a VCS commit containing the changes.")
478     parser.add_argument("--no-patch", action="store_false", dest="patch",
479                         help="Don't create a VCS commit containing the changes.")
480     parser.add_argument("--sync", dest="sync", action="store_true", default=False,
481                         help="Sync the tests with the latest from upstream (implies --patch)")
482     parser.add_argument("--ignore-existing", action="store_true", help="When updating test results only consider results from the logfiles provided, not existing expectations.")
483     parser.add_argument("--stability", nargs="?", action="store", const="unstable", default=None,
484         help=("Reason for disabling tests. When updating test results, disable tests that have "
485               "inconsistent results across many runs with the given reason."))
486     parser.add_argument("--continue", action="store_true", help="Continue a previously started run of the update script")
487     parser.add_argument("--abort", action="store_true", help="Clear state from a previous incomplete run of the update script")
488     parser.add_argument("--exclude", action="store", nargs="*",
489                         help="List of glob-style paths to exclude when syncing tests")
490     parser.add_argument("--include", action="store", nargs="*",
491                         help="List of glob-style paths to include which would otherwise be excluded when syncing tests")
492     # Should make this required iff run=logfile
493     parser.add_argument("run_log", nargs="*", type=abs_path,
494                         help="Log file from run of tests")
495     commandline.add_logging_group(parser)
496     return parser
497
498
499 def create_parser_reduce(product_choices=None):
500     parser = create_parser(product_choices)
501     parser.add_argument("target", action="store", help="Test id that is unstable")
502     return parser
503
504
505 def parse_args():
506     parser = create_parser()
507     rv = vars(parser.parse_args())
508     check_args(rv)
509     return rv
510
511
512 def parse_args_update():
513     parser = create_parser_update()
514     rv = vars(parser.parse_args())
515     check_args_update(rv)
516     return rv
517
518
519 def parse_args_reduce():
520     parser = create_parser_reduce()
521     rv = vars(parser.parse_args())
522     check_args(rv)
523     return rv