Unreviewed. Update W3C WebDriver imported tests.
[WebKit-https.git] / WebDriverTests / imported / w3c / tools / wptrunner / wptrunner / executors / pytestrunner / runner.py
1 """Provides interface to deal with pytest.
2
3 Usage::
4
5     session = webdriver.client.Session("127.0.0.1", "4444", "/")
6     harness_result = ("OK", None)
7     subtest_results = pytestrunner.run("/path/to/test", session.url)
8     return (harness_result, subtest_results)
9 """
10
11 import errno
12 import json
13 import os
14 import shutil
15 import tempfile
16
17
18 pytest = None
19
20
21 def do_delayed_imports():
22     global pytest
23     import pytest
24
25
26 def run(path, server_config, session_config, timeout=0):
27     """Run Python test at ``path`` in pytest.  The provided ``session``
28     is exposed as a fixture available in the scope of the test functions.
29
30     :param path: Path to the test file.
31     :param session_config: dictionary of host, port,capabilities parameters
32     to pass through to the webdriver session
33     :param timeout: Duration before interrupting potentially hanging
34         tests.  If 0, there is no timeout.
35
36     :returns: List of subtest results, which are tuples of (test id,
37         status, message, stacktrace).
38     """
39
40     if pytest is None:
41         do_delayed_imports()
42
43     recorder = SubtestResultRecorder()
44
45     os.environ["WD_HOST"] = session_config["host"]
46     os.environ["WD_PORT"] = str(session_config["port"])
47     os.environ["WD_CAPABILITIES"] = json.dumps(session_config["capabilities"])
48     os.environ["WD_SERVER_CONFIG"] = json.dumps(server_config)
49
50     plugins = [recorder]
51
52     # TODO(ato): Deal with timeouts
53
54     with TemporaryDirectory() as cache:
55         pytest.main(["--strict",  # turn warnings into errors
56                      "--verbose",  # show each individual subtest
57                      "--capture", "no",  # enable stdout/stderr from tests
58                      "--basetemp", cache,  # temporary directory
59                      "--showlocals",  # display contents of variables in local scope
60                      "-p", "no:mozlog",  # use the WPT result recorder
61                      "-p", "no:cacheprovider",  # disable state preservation across invocations
62                      path],
63                     plugins=plugins)
64
65     return recorder.results
66
67
68 class SubtestResultRecorder(object):
69     def __init__(self):
70         self.results = []
71
72     def pytest_runtest_logreport(self, report):
73         if report.passed and report.when == "call":
74             self.record_pass(report)
75         elif report.failed:
76             if report.when != "call":
77                 self.record_error(report)
78             else:
79                 self.record_fail(report)
80         elif report.skipped:
81             self.record_skip(report)
82
83     def record_pass(self, report):
84         self.record(report.nodeid, "PASS")
85
86     def record_fail(self, report):
87         self.record(report.nodeid, "FAIL", stack=report.longrepr)
88
89     def record_error(self, report):
90         # error in setup/teardown
91         if report.when != "call":
92             message = "%s error" % report.when
93         self.record(report.nodeid, "ERROR", message, report.longrepr)
94
95     def record_skip(self, report):
96         self.record(report.nodeid, "ERROR",
97                     "In-test skip decorators are disallowed, "
98                     "please use WPT metadata to ignore tests.")
99
100     def record(self, test, status, message=None, stack=None):
101         if stack is not None:
102             stack = str(stack)
103         new_result = (test, status, message, stack)
104         self.results.append(new_result)
105
106
107 class TemporaryDirectory(object):
108     def __enter__(self):
109         self.path = tempfile.mkdtemp(prefix="pytest-")
110         return self.path
111
112     def __exit__(self, *args):
113         try:
114             shutil.rmtree(self.path)
115         except OSError as e:
116             # no such file or directory
117             if e.errno != errno.ENOENT:
118                 raise