0870b1232ad5fb831bb62adbb2787b663eda6163
[WebKit-https.git] / WebDriverTests / imported / w3c / tools / wptrunner / wptrunner / executors / pytestrunner / runner.py
1 """
2 Provides interface to deal with pytest.
3
4 Usage::
5
6     session = webdriver.client.Session("127.0.0.1", "4444", "/")
7     harness_result = ("OK", None)
8     subtest_results = pytestrunner.run("/path/to/test", session.url)
9     return (harness_result, subtest_results)
10 """
11
12 import errno
13 import json
14 import os
15 import shutil
16 import tempfile
17
18
19 pytest = None
20
21
22 def do_delayed_imports():
23     global pytest
24     import pytest
25
26
27 def run(path, server_config, session_config, timeout=0):
28     """
29     Run Python test at ``path`` in pytest.  The provided ``session``
30     is exposed as a fixture available in the scope of the test functions.
31
32     :param path: Path to the test file.
33     :param session_config: dictionary of host, port,capabilities parameters
34     to pass through to the webdriver session
35     :param timeout: Duration before interrupting potentially hanging
36         tests.  If 0, there is no timeout.
37
38     :returns: (<harness result>, [<subtest result>, ...]),
39         where <subtest result> is (test id, status, message, stacktrace).
40     """
41     if pytest is None:
42         do_delayed_imports()
43
44     os.environ["WD_HOST"] = session_config["host"]
45     os.environ["WD_PORT"] = str(session_config["port"])
46     os.environ["WD_CAPABILITIES"] = json.dumps(session_config["capabilities"])
47     os.environ["WD_SERVER_CONFIG"] = json.dumps(server_config)
48
49     harness = HarnessResultRecorder()
50     subtests = SubtestResultRecorder()
51
52     with TemporaryDirectory() as cache:
53         try:
54             pytest.main(["--strict",  # turn warnings into errors
55                          "--verbose",  # show each individual subtest
56                          "--capture", "no",  # enable stdout/stderr from tests
57                          "--basetemp", cache,  # temporary directory
58                          "--showlocals",  # display contents of variables in local scope
59                          "-p", "no:mozlog",  # use the WPT result recorder
60                          "-p", "no:cacheprovider",  # disable state preservation across invocations
61                          path],
62                         plugins=[harness, subtests])
63         except Exception as e:
64             harness.outcome = ("ERROR", str(e))
65
66     return (harness.outcome, subtests.results)
67
68
69 class HarnessResultRecorder(object):
70     outcomes = {
71         "failed": "ERROR",
72         "passed": "OK",
73         "skipped": "SKIP",
74     }
75
76     def __init__(self):
77         # we are ok unless told otherwise
78         self.outcome = ("OK", None)
79
80     def pytest_collectreport(self, report):
81         harness_result = self.outcomes[report.outcome]
82         self.outcome = (harness_result, None)
83
84
85 class SubtestResultRecorder(object):
86     def __init__(self):
87         self.results = []
88
89     def pytest_runtest_logreport(self, report):
90         if report.passed and report.when == "call":
91             self.record_pass(report)
92         elif report.failed:
93             if report.when != "call":
94                 self.record_error(report)
95             else:
96                 self.record_fail(report)
97         elif report.skipped:
98             self.record_skip(report)
99
100     def record_pass(self, report):
101         self.record(report.nodeid, "PASS")
102
103     def record_fail(self, report):
104         self.record(report.nodeid, "FAIL", stack=report.longrepr)
105
106     def record_error(self, report):
107         # error in setup/teardown
108         if report.when != "call":
109             message = "%s error" % report.when
110         self.record(report.nodeid, "ERROR", message, report.longrepr)
111
112     def record_skip(self, report):
113         self.record(report.nodeid, "ERROR",
114                     "In-test skip decorators are disallowed, "
115                     "please use WPT metadata to ignore tests.")
116
117     def record(self, test, status, message=None, stack=None):
118         if stack is not None:
119             stack = str(stack)
120         new_result = (test.split("::")[-1], status, message, stack)
121         self.results.append(new_result)
122
123
124 class TemporaryDirectory(object):
125     def __enter__(self):
126         self.path = tempfile.mkdtemp(prefix="pytest-")
127         return self.path
128
129     def __exit__(self, *args):
130         try:
131             shutil.rmtree(self.path)
132         except OSError as e:
133             # no such file or directory
134             if e.errno != errno.ENOENT:
135                 raise