Unreviewed. Update W3C WebDriver imported tests.
[WebKit-https.git] / WebDriverTests / imported / w3c / tools / wptrunner / wptrunner / stability.py
index dc1a1d0..6eb0604 100644 (file)
@@ -1,14 +1,14 @@
 import copy
 import functools
 import imp
+import io
 import os
-import sys
 from collections import OrderedDict, defaultdict
 from datetime import datetime
 
 from mozlog import reader
-from mozlog.formatters import JSONFormatter, TbplFormatter
-from mozlog.handlers import BaseHandler, LogLevelFilter, StreamHandler
+from mozlog.formatters import JSONFormatter
+from mozlog.handlers import BaseHandler, StreamHandler, LogLevelFilter
 
 here = os.path.dirname(__file__)
 localpaths = imp.load_source("localpaths", os.path.abspath(os.path.join(here, os.pardir, os.pardir, "localpaths.py")))
@@ -86,6 +86,8 @@ class LogHandler(reader.LogHandler):
 
 def is_inconsistent(results_dict, iterations):
     """Return whether or not a single test is inconsistent."""
+    if 'SKIP' in results_dict:
+        return False
     return len(results_dict) > 1 or sum(results_dict.values()) != iterations
 
 
@@ -178,31 +180,29 @@ def run_step(logger, iterations, restart_after_iteration, kwargs_extras, **kwarg
     kwargs["pause_after_test"] = False
     kwargs.update(kwargs_extras)
 
-    handler = LogActionFilter(
-        LogLevelFilter(
-            StreamHandler(
-                sys.stdout,
-                TbplFormatter()
-            ),
-            "WARNING"),
-        ["log", "process_output"])
+    def wrap_handler(x):
+        x = LogLevelFilter(x, "WARNING")
+        if not kwargs["verify_log_full"]:
+            x = LogActionFilter(x, ["log", "process_output"])
+        return x
 
-    # There is a public API for this in the next mozlog
     initial_handlers = logger._state.handlers
-    logger._state.handlers = []
+    logger._state.handlers = [wrap_handler(handler)
+                              for handler in initial_handlers]
 
-    with open("raw.log", "wb") as log:
-        # Setup logging for wptrunner that keeps process output and
-        # warning+ level logs only
-        logger.add_handler(handler)
-        logger.add_handler(StreamHandler(log, JSONFormatter()))
+    log = io.BytesIO()
+    # Setup logging for wptrunner that keeps process output and
+    # warning+ level logs only
+    logger.add_handler(StreamHandler(log, JSONFormatter()))
 
-        wptrunner.run_tests(**kwargs)
+    wptrunner.run_tests(**kwargs)
 
     logger._state.handlers = initial_handlers
+    logger._state.running_tests = set()
+    logger._state.suite_started = False
 
-    with open("raw.log", "rb") as log:
-        results, inconsistent = process_results(log, iterations)
+    log.seek(0)
+    results, inconsistent = process_results(log, iterations)
     return results, inconsistent, iterations