Unreviewed. Update W3C WebDriver imported tests.
[WebKit-https.git] / WebDriverTests / imported / w3c / tools / wptrunner / wptrunner / executors / pytestrunner / runner.py
1 """
2 Provides interface to deal with pytest.
3
4 Usage::
5
6     session = webdriver.client.Session("127.0.0.1", "4444", "/")
7     harness_result = ("OK", None)
8     subtest_results = pytestrunner.run("/path/to/test", session.url)
9     return (harness_result, subtest_results)
10 """
11
12 import errno
13 import json
14 import os
15 import shutil
16 import tempfile
17
18
19 pytest = None
20
21
22 def do_delayed_imports():
23     global pytest
24     import pytest
25
26
27 def run(path, server_config, session_config, timeout=0):
28     """
29     Run Python test at ``path`` in pytest.  The provided ``session``
30     is exposed as a fixture available in the scope of the test functions.
31
32     :param path: Path to the test file.
33     :param session_config: dictionary of host, port,capabilities parameters
34     to pass through to the webdriver session
35     :param timeout: Duration before interrupting potentially hanging
36         tests.  If 0, there is no timeout.
37
38     :returns: (<harness result>, [<subtest result>, ...]),
39         where <subtest result> is (test id, status, message, stacktrace).
40     """
41     if pytest is None:
42         do_delayed_imports()
43
44     os.environ["WD_HOST"] = session_config["host"]
45     os.environ["WD_PORT"] = str(session_config["port"])
46     os.environ["WD_CAPABILITIES"] = json.dumps(session_config["capabilities"])
47     os.environ["WD_SERVER_CONFIG"] = json.dumps(server_config)
48
49     harness = HarnessResultRecorder()
50     subtests = SubtestResultRecorder()
51
52     with TemporaryDirectory() as cache:
53         try:
54             pytest.main(["--strict",  # turn warnings into errors
55                          "-vv",  # show each individual subtest and full failure logs
56                          "--capture", "no",  # enable stdout/stderr from tests
57                          "--basetemp", cache,  # temporary directory
58                          "--showlocals",  # display contents of variables in local scope
59                          "-p", "no:mozlog",  # use the WPT result recorder
60                          "-p", "no:cacheprovider",  # disable state preservation across invocations
61                          "-o=console_output_style=classic",  # disable test progress bar
62                          path],
63                         plugins=[harness, subtests])
64         except Exception as e:
65             harness.outcome = ("ERROR", str(e))
66
67     return (harness.outcome, subtests.results)
68
69
70 class HarnessResultRecorder(object):
71     outcomes = {
72         "failed": "ERROR",
73         "passed": "OK",
74         "skipped": "SKIP",
75     }
76
77     def __init__(self):
78         # we are ok unless told otherwise
79         self.outcome = ("OK", None)
80
81     def pytest_collectreport(self, report):
82         harness_result = self.outcomes[report.outcome]
83         self.outcome = (harness_result, None)
84
85
86 class SubtestResultRecorder(object):
87     def __init__(self):
88         self.results = []
89
90     def pytest_runtest_logreport(self, report):
91         if report.passed and report.when == "call":
92             self.record_pass(report)
93         elif report.failed:
94             if report.when != "call":
95                 self.record_error(report)
96             else:
97                 self.record_fail(report)
98         elif report.skipped:
99             self.record_skip(report)
100
101     def record_pass(self, report):
102         self.record(report.nodeid, "PASS")
103
104     def record_fail(self, report):
105         self.record(report.nodeid, "FAIL", stack=report.longrepr)
106
107     def record_error(self, report):
108         # error in setup/teardown
109         if report.when != "call":
110             message = "%s error" % report.when
111         self.record(report.nodeid, "ERROR", message, report.longrepr)
112
113     def record_skip(self, report):
114         self.record(report.nodeid, "ERROR",
115                     "In-test skip decorators are disallowed, "
116                     "please use WPT metadata to ignore tests.")
117
118     def record(self, test, status, message=None, stack=None):
119         if stack is not None:
120             stack = str(stack)
121         new_result = (test.split("::")[-1], status, message, stack)
122         self.results.append(new_result)
123
124
125 class TemporaryDirectory(object):
126     def __enter__(self):
127         self.path = tempfile.mkdtemp(prefix="pytest-")
128         return self.path
129
130     def __exit__(self, *args):
131         try:
132             shutil.rmtree(self.path)
133         except OSError as e:
134             # no such file or directory
135             if e.errno != errno.ENOENT:
136                 raise