a9a626429b162ef822a24e2024c7f858377f78ab
[WebKit-https.git] / WebDriverTests / imported / w3c / tools / wptrunner / wptrunner / wptcommandline.py
1 import argparse
2 import ast
3 import os
4 import sys
5 from collections import OrderedDict
6 from distutils.spawn import find_executable
7
8 import config
9 import wpttest
10 import formatters
11
12
13 def abs_path(path):
14     return os.path.abspath(os.path.expanduser(path))
15
16
17 def url_or_path(path):
18     import urlparse
19
20     parsed = urlparse.urlparse(path)
21     if len(parsed.scheme) > 2:
22         return path
23     else:
24         return abs_path(path)
25
26
27 def require_arg(kwargs, name, value_func=None):
28     if value_func is None:
29         value_func = lambda x: x is not None
30
31     if not name in kwargs or not value_func(kwargs[name]):
32         print >> sys.stderr, "Missing required argument %s" % name
33         sys.exit(1)
34
35
36 def create_parser(product_choices=None):
37     from mozlog import commandline
38
39     import products
40
41     if product_choices is None:
42         config_data = config.load()
43         product_choices = products.products_enabled(config_data)
44
45     parser = argparse.ArgumentParser(description="""Runner for web-platform-tests tests.""",
46                                      usage="""%(prog)s [OPTION]... [TEST]...
47
48 TEST is either the full path to a test file to run, or the URL of a test excluding
49 scheme host and port.""")
50     parser.add_argument("--manifest-update", action="store_true", default=None,
51                         help="Regenerate the test manifest.")
52     parser.add_argument("--no-manifest-update", action="store_false", dest="manifest_update",
53                         help="Prevent regeneration of the test manifest.")
54
55     parser.add_argument("--timeout-multiplier", action="store", type=float, default=None,
56                         help="Multiplier relative to standard test timeout to use")
57     parser.add_argument("--run-by-dir", type=int, nargs="?", default=False,
58                         help="Split run into groups by directories. With a parameter,"
59                         "limit the depth of splits e.g. --run-by-dir=1 to split by top-level"
60                         "directory")
61     parser.add_argument("--processes", action="store", type=int, default=None,
62                         help="Number of simultaneous processes to use")
63
64     parser.add_argument("--no-capture-stdio", action="store_true", default=False,
65                         help="Don't capture stdio and write to logging")
66
67     mode_group = parser.add_argument_group("Mode")
68     mode_group.add_argument("--list-test-groups", action="store_true",
69                             default=False,
70                             help="List the top level directories containing tests that will run.")
71     mode_group.add_argument("--list-disabled", action="store_true",
72                             default=False,
73                             help="List the tests that are disabled on the current platform")
74     mode_group.add_argument("--list-tests", action="store_true",
75                             default=False,
76                             help="List all tests that will run")
77
78     test_selection_group = parser.add_argument_group("Test Selection")
79     test_selection_group.add_argument("--test-types", action="store",
80                                       nargs="*", default=wpttest.enabled_tests,
81                                       choices=wpttest.enabled_tests,
82                                       help="Test types to run")
83     test_selection_group.add_argument("--include", action="append",
84                                       help="URL prefix to include")
85     test_selection_group.add_argument("--exclude", action="append",
86                                       help="URL prefix to exclude")
87     test_selection_group.add_argument("--include-manifest", type=abs_path,
88                                       help="Path to manifest listing tests to include")
89     test_selection_group.add_argument("--tag", action="append", dest="tags",
90                                       help="Labels applied to tests to include in the run. Labels starting dir: are equivalent to top-level directories.")
91
92     debugging_group = parser.add_argument_group("Debugging")
93     debugging_group.add_argument('--debugger', const="__default__", nargs="?",
94                                  help="run under a debugger, e.g. gdb or valgrind")
95     debugging_group.add_argument('--debugger-args', help="arguments to the debugger")
96     debugging_group.add_argument("--repeat", action="store", type=int, default=1,
97                                  help="Number of times to run the tests")
98     debugging_group.add_argument("--repeat-until-unexpected", action="store_true", default=None,
99                                  help="Run tests in a loop until one returns an unexpected result")
100     debugging_group.add_argument('--pause-after-test', action="store_true", default=None,
101                                  help="Halt the test runner after each test (this happens by default if only a single test is run)")
102     debugging_group.add_argument('--no-pause-after-test', dest="pause_after_test", action="store_false",
103                                  help="Don't halt the test runner irrespective of the number of tests run")
104
105     debugging_group.add_argument('--pause-on-unexpected', action="store_true",
106                                  help="Halt the test runner when an unexpected result is encountered")
107     debugging_group.add_argument('--no-restart-on-unexpected', dest="restart_on_unexpected",
108                                  default=True, action="store_false",
109                                  help="Don't restart on an unexpected result")
110
111     debugging_group.add_argument("--symbols-path", action="store", type=url_or_path,
112                                  help="Path or url to symbols file used to analyse crash minidumps.")
113     debugging_group.add_argument("--stackwalk-binary", action="store", type=abs_path,
114                                  help="Path to stackwalker program used to analyse minidumps.")
115
116     debugging_group.add_argument("--pdb", action="store_true",
117                                  help="Drop into pdb on python exception")
118
119     config_group = parser.add_argument_group("Configuration")
120     config_group.add_argument("--binary", action="store",
121                               type=abs_path, help="Binary to run tests against")
122     config_group.add_argument('--binary-arg',
123                               default=[], action="append", dest="binary_args",
124                               help="Extra argument for the binary")
125     config_group.add_argument("--webdriver-binary", action="store", metavar="BINARY",
126                               type=abs_path, help="WebDriver server binary to use")
127     config_group.add_argument('--webdriver-arg',
128                               default=[], action="append", dest="webdriver_args",
129                               help="Extra argument for the WebDriver binary")
130
131     config_group.add_argument("--metadata", action="store", type=abs_path, dest="metadata_root",
132                               help="Path to root directory containing test metadata"),
133     config_group.add_argument("--tests", action="store", type=abs_path, dest="tests_root",
134                               help="Path to root directory containing test files"),
135     config_group.add_argument("--run-info", action="store", type=abs_path,
136                               help="Path to directory containing extra json files to add to run info")
137     config_group.add_argument("--product", action="store", choices=product_choices,
138                               default=None, help="Browser against which to run tests")
139     config_group.add_argument("--config", action="store", type=abs_path, dest="config",
140                               help="Path to config file")
141     config_group.add_argument("--install-fonts", action="store_true",
142                               default=None,
143                               help="Allow the wptrunner to install fonts on your system")
144     config_group.add_argument("--font-dir", action="store", type=abs_path, dest="font_dir",
145                               help="Path to local font installation directory", default=None)
146
147     build_type = parser.add_mutually_exclusive_group()
148     build_type.add_argument("--debug-build", dest="debug", action="store_true",
149                             default=None,
150                             help="Build is a debug build (overrides any mozinfo file)")
151     build_type.add_argument("--release-build", dest="debug", action="store_false",
152                             default=None,
153                             help="Build is a release (overrides any mozinfo file)")
154
155
156     chunking_group = parser.add_argument_group("Test Chunking")
157     chunking_group.add_argument("--total-chunks", action="store", type=int, default=1,
158                                 help="Total number of chunks to use")
159     chunking_group.add_argument("--this-chunk", action="store", type=int, default=1,
160                                 help="Chunk number to run")
161     chunking_group.add_argument("--chunk-type", action="store", choices=["none", "equal_time", "hash", "dir_hash"],
162                                 default=None, help="Chunking type to use")
163
164     ssl_group = parser.add_argument_group("SSL/TLS")
165     ssl_group.add_argument("--ssl-type", action="store", default=None,
166                         choices=["openssl", "pregenerated", "none"],
167                         help="Type of ssl support to enable (running without ssl may lead to spurious errors)")
168
169     ssl_group.add_argument("--openssl-binary", action="store",
170                         help="Path to openssl binary", default="openssl")
171     ssl_group.add_argument("--certutil-binary", action="store",
172                         help="Path to certutil binary for use with Firefox + ssl")
173
174     ssl_group.add_argument("--ca-cert-path", action="store", type=abs_path,
175                         help="Path to ca certificate when using pregenerated ssl certificates")
176     ssl_group.add_argument("--host-key-path", action="store", type=abs_path,
177                         help="Path to host private key when using pregenerated ssl certificates")
178     ssl_group.add_argument("--host-cert-path", action="store", type=abs_path,
179                         help="Path to host certificate when using pregenerated ssl certificates")
180
181     gecko_group = parser.add_argument_group("Gecko-specific")
182     gecko_group.add_argument("--prefs-root", dest="prefs_root", action="store", type=abs_path,
183                              help="Path to the folder containing browser prefs")
184     gecko_group.add_argument("--disable-e10s", dest="gecko_e10s", action="store_false", default=True,
185                              help="Run tests without electrolysis preferences")
186     gecko_group.add_argument("--stackfix-dir", dest="stackfix_dir", action="store",
187                              help="Path to directory containing assertion stack fixing scripts")
188     gecko_group.add_argument("--setpref", dest="extra_prefs", action='append',
189                              default=[], metavar="PREF=VALUE",
190                              help="Defines an extra user preference (overrides those in prefs_root)")
191     gecko_group.add_argument("--leak-check", dest="leak_check", action="store_true",
192                              help="Enable leak checking")
193     gecko_group.add_argument("--stylo-threads", action="store", type=int, default=1,
194                              help="Number of parallel threads to use for stylo")
195     gecko_group.add_argument("--reftest-internal", dest="reftest_internal", action="store_true",
196                              default=None, help="Enable reftest runner implemented inside Marionette")
197     gecko_group.add_argument("--reftest-external", dest="reftest_internal", action="store_false",
198                              help="Disable reftest runner implemented inside Marionette")
199     gecko_group.add_argument("--reftest-screenshot", dest="reftest_screenshot", action="store",
200                              choices=["always", "fail", "unexpected"], default="unexpected",
201                              help="With --reftest-internal, when to take a screenshot")
202
203     servo_group = parser.add_argument_group("Servo-specific")
204     servo_group.add_argument("--user-stylesheet",
205                              default=[], action="append", dest="user_stylesheets",
206                              help="Inject a user CSS stylesheet into every test.")
207
208     sauce_group = parser.add_argument_group("Sauce Labs-specific")
209     sauce_group.add_argument("--sauce-browser", dest="sauce_browser",
210                              help="Sauce Labs browser name")
211     sauce_group.add_argument("--sauce-platform", dest="sauce_platform",
212                              help="Sauce Labs OS platform")
213     sauce_group.add_argument("--sauce-version", dest="sauce_version",
214                              help="Sauce Labs browser version")
215     sauce_group.add_argument("--sauce-build", dest="sauce_build",
216                              help="Sauce Labs build identifier")
217     sauce_group.add_argument("--sauce-tags", dest="sauce_tags", nargs="*",
218                              help="Sauce Labs identifying tag", default=[])
219     sauce_group.add_argument("--sauce-tunnel-id", dest="sauce_tunnel_id",
220                              help="Sauce Connect tunnel identifier")
221     sauce_group.add_argument("--sauce-user", dest="sauce_user",
222                              help="Sauce Labs user name")
223     sauce_group.add_argument("--sauce-key", dest="sauce_key",
224                              default=os.environ.get("SAUCE_ACCESS_KEY"),
225                              help="Sauce Labs access key")
226     sauce_group.add_argument("--sauce-connect-binary",
227                              dest="sauce_connect_binary",
228                              help="Path to Sauce Connect binary")
229
230     parser.add_argument("test_list", nargs="*",
231                         help="List of URLs for tests to run, or paths including tests to run. "
232                              "(equivalent to --include)")
233
234     commandline.log_formatters["wptreport"] = (formatters.WptreportFormatter, "wptreport format")
235
236     commandline.add_logging_group(parser)
237     return parser
238
239
240 def set_from_config(kwargs):
241     if kwargs["config"] is None:
242         config_path = config.path()
243     else:
244         config_path = kwargs["config"]
245
246     kwargs["config_path"] = config_path
247
248     kwargs["config"] = config.read(kwargs["config_path"])
249
250     keys = {"paths": [("prefs", "prefs_root", True),
251                       ("run_info", "run_info", True)],
252             "web-platform-tests": [("remote_url", "remote_url", False),
253                                    ("branch", "branch", False),
254                                    ("sync_path", "sync_path", True)],
255             "SSL": [("openssl_binary", "openssl_binary", True),
256                     ("certutil_binary", "certutil_binary", True),
257                     ("ca_cert_path", "ca_cert_path", True),
258                     ("host_cert_path", "host_cert_path", True),
259                     ("host_key_path", "host_key_path", True)]}
260
261     for section, values in keys.iteritems():
262         for config_value, kw_value, is_path in values:
263             if kw_value in kwargs and kwargs[kw_value] is None:
264                 if not is_path:
265                     new_value = kwargs["config"].get(section, config.ConfigDict({})).get(config_value)
266                 else:
267                     new_value = kwargs["config"].get(section, config.ConfigDict({})).get_path(config_value)
268                 kwargs[kw_value] = new_value
269
270     kwargs["test_paths"] = get_test_paths(kwargs["config"])
271
272     if kwargs["tests_root"]:
273         if "/" not in kwargs["test_paths"]:
274             kwargs["test_paths"]["/"] = {}
275         kwargs["test_paths"]["/"]["tests_path"] = kwargs["tests_root"]
276
277     if kwargs["metadata_root"]:
278         if "/" not in kwargs["test_paths"]:
279             kwargs["test_paths"]["/"] = {}
280         kwargs["test_paths"]["/"]["metadata_path"] = kwargs["metadata_root"]
281
282     kwargs["suite_name"] = kwargs["config"].get("web-platform-tests", {}).get("name", "web-platform-tests")
283
284
285 def get_test_paths(config):
286     # Set up test_paths
287     test_paths = OrderedDict()
288
289     for section in config.iterkeys():
290         if section.startswith("manifest:"):
291             manifest_opts = config.get(section)
292             url_base = manifest_opts.get("url_base", "/")
293             test_paths[url_base] = {
294                 "tests_path": manifest_opts.get_path("tests"),
295                 "metadata_path": manifest_opts.get_path("metadata")}
296
297     return test_paths
298
299
300 def exe_path(name):
301     if name is None:
302         return
303
304     path = find_executable(name)
305     if path and os.access(path, os.X_OK):
306         return path
307     else:
308         return None
309
310
311 def check_args(kwargs):
312     set_from_config(kwargs)
313
314     for test_paths in kwargs["test_paths"].itervalues():
315         if not ("tests_path" in test_paths and
316                 "metadata_path" in test_paths):
317             print "Fatal: must specify both a test path and metadata path"
318             sys.exit(1)
319         for key, path in test_paths.iteritems():
320             name = key.split("_", 1)[0]
321
322             if not os.path.exists(path):
323                 print "Fatal: %s path %s does not exist" % (name, path)
324                 sys.exit(1)
325
326             if not os.path.isdir(path):
327                 print "Fatal: %s path %s is not a directory" % (name, path)
328                 sys.exit(1)
329
330     if kwargs["product"] is None:
331         kwargs["product"] = "firefox"
332
333     if "sauce" in kwargs["product"]:
334         kwargs["pause_after_test"] = False
335
336     if kwargs["test_list"]:
337         if kwargs["include"] is not None:
338             kwargs["include"].extend(kwargs["test_list"])
339         else:
340             kwargs["include"] = kwargs["test_list"]
341
342     if kwargs["run_info"] is None:
343         kwargs["run_info"] = kwargs["config_path"]
344
345     if kwargs["this_chunk"] > 1:
346         require_arg(kwargs, "total_chunks", lambda x: x >= kwargs["this_chunk"])
347
348     if kwargs["chunk_type"] is None:
349         if kwargs["total_chunks"] > 1:
350             kwargs["chunk_type"] = "dir_hash"
351         else:
352             kwargs["chunk_type"] = "none"
353
354     if kwargs["processes"] is None:
355         kwargs["processes"] = 1
356
357     if kwargs["debugger"] is not None:
358         import mozdebug
359         if kwargs["debugger"] == "__default__":
360             kwargs["debugger"] = mozdebug.get_default_debugger_name()
361         debug_info = mozdebug.get_debugger_info(kwargs["debugger"],
362                                                 kwargs["debugger_args"])
363         if debug_info and debug_info.interactive:
364             if kwargs["processes"] != 1:
365                 kwargs["processes"] = 1
366             kwargs["no_capture_stdio"] = True
367         kwargs["debug_info"] = debug_info
368     else:
369         kwargs["debug_info"] = None
370
371     if kwargs["binary"] is not None:
372         if not os.path.exists(kwargs["binary"]):
373             print >> sys.stderr, "Binary path %s does not exist" % kwargs["binary"]
374             sys.exit(1)
375
376     if kwargs["ssl_type"] is None:
377         if None not in (kwargs["ca_cert_path"], kwargs["host_cert_path"], kwargs["host_key_path"]):
378             kwargs["ssl_type"] = "pregenerated"
379         elif exe_path(kwargs["openssl_binary"]) is not None:
380             kwargs["ssl_type"] = "openssl"
381         else:
382             kwargs["ssl_type"] = "none"
383
384     if kwargs["ssl_type"] == "pregenerated":
385         require_arg(kwargs, "ca_cert_path", lambda x:os.path.exists(x))
386         require_arg(kwargs, "host_cert_path", lambda x:os.path.exists(x))
387         require_arg(kwargs, "host_key_path", lambda x:os.path.exists(x))
388
389     elif kwargs["ssl_type"] == "openssl":
390         path = exe_path(kwargs["openssl_binary"])
391         if path is None:
392             print >> sys.stderr, "openssl-binary argument missing or not a valid executable"
393             sys.exit(1)
394         kwargs["openssl_binary"] = path
395
396     if kwargs["ssl_type"] != "none" and kwargs["product"] == "firefox":
397         path = exe_path(kwargs["certutil_binary"])
398         if path is None:
399             print >> sys.stderr, "certutil-binary argument missing or not a valid executable"
400             sys.exit(1)
401         kwargs["certutil_binary"] = path
402
403     if kwargs['extra_prefs']:
404         missing = any('=' not in prefarg for prefarg in kwargs['extra_prefs'])
405         if missing:
406             print >> sys.stderr, "Preferences via --setpref must be in key=value format"
407             sys.exit(1)
408         kwargs['extra_prefs'] = [tuple(prefarg.split('=', 1)) for prefarg in
409                                  kwargs['extra_prefs']]
410
411     if kwargs["reftest_internal"] is None:
412         # Default to the internal reftest implementation on Linux and OSX
413         kwargs["reftest_internal"] = sys.platform.startswith("linux") or sys.platform.startswith("darwin")
414
415     return kwargs
416
417
418 def check_args_update(kwargs):
419     set_from_config(kwargs)
420
421     if kwargs["product"] is None:
422         kwargs["product"] = "firefox"
423     if kwargs["patch"] is None:
424         kwargs["patch"] = kwargs["sync"]
425
426     for item in kwargs["run_log"]:
427         if os.path.isdir(item):
428             print >> sys.stderr, "Log file %s is a directory" % item
429             sys.exit(1)
430
431     return kwargs
432
433
434 def create_parser_update(product_choices=None):
435     from mozlog.structured import commandline
436
437     import products
438
439     if product_choices is None:
440         config_data = config.load()
441         product_choices = products.products_enabled(config_data)
442
443     parser = argparse.ArgumentParser("web-platform-tests-update",
444                                      description="Update script for web-platform-tests tests.")
445     parser.add_argument("--product", action="store", choices=product_choices,
446                         default=None, help="Browser for which metadata is being updated")
447     parser.add_argument("--config", action="store", type=abs_path, help="Path to config file")
448     parser.add_argument("--metadata", action="store", type=abs_path, dest="metadata_root",
449                         help="Path to the folder containing test metadata"),
450     parser.add_argument("--tests", action="store", type=abs_path, dest="tests_root",
451                         help="Path to web-platform-tests"),
452     parser.add_argument("--sync-path", action="store", type=abs_path,
453                         help="Path to store git checkout of web-platform-tests during update"),
454     parser.add_argument("--remote_url", action="store",
455                         help="URL of web-platfrom-tests repository to sync against"),
456     parser.add_argument("--branch", action="store", type=abs_path,
457                         help="Remote branch to sync against")
458     parser.add_argument("--rev", action="store", help="Revision to sync to")
459     parser.add_argument("--patch", action="store_true", dest="patch", default=None,
460                         help="Create a VCS commit containing the changes.")
461     parser.add_argument("--no-patch", action="store_false", dest="patch",
462                         help="Don't create a VCS commit containing the changes.")
463     parser.add_argument("--sync", dest="sync", action="store_true", default=False,
464                         help="Sync the tests with the latest from upstream (implies --patch)")
465     parser.add_argument("--ignore-existing", action="store_true", help="When updating test results only consider results from the logfiles provided, not existing expectations.")
466     parser.add_argument("--stability", nargs="?", action="store", const="unstable", default=None,
467         help=("Reason for disabling tests. When updating test results, disable tests that have "
468               "inconsistent results across many runs with the given reason."))
469     parser.add_argument("--continue", action="store_true", help="Continue a previously started run of the update script")
470     parser.add_argument("--abort", action="store_true", help="Clear state from a previous incomplete run of the update script")
471     parser.add_argument("--exclude", action="store", nargs="*",
472                         help="List of glob-style paths to exclude when syncing tests")
473     parser.add_argument("--include", action="store", nargs="*",
474                         help="List of glob-style paths to include which would otherwise be excluded when syncing tests")
475     # Should make this required iff run=logfile
476     parser.add_argument("run_log", nargs="*", type=abs_path,
477                         help="Log file from run of tests")
478     commandline.add_logging_group(parser)
479     return parser
480
481
482 def create_parser_reduce(product_choices=None):
483     parser = create_parser(product_choices)
484     parser.add_argument("target", action="store", help="Test id that is unstable")
485     return parser
486
487
488 def parse_args():
489     parser = create_parser()
490     rv = vars(parser.parse_args())
491     check_args(rv)
492     return rv
493
494
495 def parse_args_update():
496     parser = create_parser_update()
497     rv = vars(parser.parse_args())
498     check_args_update(rv)
499     return rv
500
501
502 def parse_args_reduce():
503     parser = create_parser_reduce()
504     rv = vars(parser.parse_args())
505     check_args(rv)
506     return rv