Unreviewed. Update W3C WebDriver imported tests.
[WebKit-https.git] / WebDriverTests / imported / w3c / tools / wptrunner / wptrunner / executors / pytestrunner / runner.py
index 611a498..a2b603d 100644 (file)
@@ -1,4 +1,5 @@
-"""Provides interface to deal with pytest.
+"""
+Provides interface to deal with pytest.
 
 Usage::
 
@@ -24,7 +25,8 @@ def do_delayed_imports():
 
 
 def run(path, server_config, session_config, timeout=0):
-    """Run Python test at ``path`` in pytest.  The provided ``session``
+    """
+    Run Python test at ``path`` in pytest.  The provided ``session``
     is exposed as a fixture available in the scope of the test functions.
 
     :param path: Path to the test file.
@@ -33,36 +35,51 @@ def run(path, server_config, session_config, timeout=0):
     :param timeout: Duration before interrupting potentially hanging
         tests.  If 0, there is no timeout.
 
-    :returns: List of subtest results, which are tuples of (test id,
-        status, message, stacktrace).
+    :returns: (<harness result>, [<subtest result>, ...]),
+        where <subtest result> is (test id, status, message, stacktrace).
     """
-
     if pytest is None:
         do_delayed_imports()
 
-    recorder = SubtestResultRecorder()
-
     os.environ["WD_HOST"] = session_config["host"]
     os.environ["WD_PORT"] = str(session_config["port"])
     os.environ["WD_CAPABILITIES"] = json.dumps(session_config["capabilities"])
     os.environ["WD_SERVER_CONFIG"] = json.dumps(server_config)
 
-    plugins = [recorder]
-
-    # TODO(ato): Deal with timeouts
+    harness = HarnessResultRecorder()
+    subtests = SubtestResultRecorder()
 
     with TemporaryDirectory() as cache:
-        pytest.main(["--strict",  # turn warnings into errors
-                     "--verbose",  # show each individual subtest
-                     "--capture", "no",  # enable stdout/stderr from tests
-                     "--basetemp", cache,  # temporary directory
-                     "--showlocals",  # display contents of variables in local scope
-                     "-p", "no:mozlog",  # use the WPT result recorder
-                     "-p", "no:cacheprovider",  # disable state preservation across invocations
-                     path],
-                    plugins=plugins)
-
-    return recorder.results
+        try:
+            pytest.main(["--strict",  # turn warnings into errors
+                         "--verbose",  # show each individual subtest
+                         "--capture", "no",  # enable stdout/stderr from tests
+                         "--basetemp", cache,  # temporary directory
+                         "--showlocals",  # display contents of variables in local scope
+                         "-p", "no:mozlog",  # use the WPT result recorder
+                         "-p", "no:cacheprovider",  # disable state preservation across invocations
+                         path],
+                        plugins=[harness, subtests])
+        except Exception as e:
+            harness.outcome = ("ERROR", str(e))
+
+    return (harness.outcome, subtests.results)
+
+
+class HarnessResultRecorder(object):
+    outcomes = {
+        "failed": "ERROR",
+        "passed": "OK",
+        "skipped": "SKIP",
+    }
+
+    def __init__(self):
+        # we are ok unless told otherwise
+        self.outcome = ("OK", None)
+
+    def pytest_collectreport(self, report):
+        harness_result = self.outcomes[report.outcome]
+        self.outcome = (harness_result, None)
 
 
 class SubtestResultRecorder(object):