WebDriver: add support for slow tests
authorcarlosgc@webkit.org <carlosgc@webkit.org@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Thu, 25 Jan 2018 13:53:59 +0000 (13:53 +0000)
committercarlosgc@webkit.org <carlosgc@webkit.org@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Thu, 25 Jan 2018 13:53:59 +0000 (13:53 +0000)
https://bugs.webkit.org/show_bug.cgi?id=182095

Reviewed by Carlos Alberto Lopez Perez.

Tools:

Add timeout mark to slow tests.

* Scripts/webkitpy/webdriver_tests/pytest_runner.py:
(TestExpectationsMarker.__init__): Save the timeout.
(TestExpectationsMarker.pytest_collection_modifyitems): If test is slow use timeout * 5.
(run): Pass timeout to TestExpectationsMarker constructor.

WebDriverTests:

Mark several tests as slow.

* TestExpectations.json:

git-svn-id: https://svn.webkit.org/repository/webkit/trunk@227604 268f45cc-cd09-0410-ab3c-d52691b4dbfc

Tools/ChangeLog
Tools/Scripts/webkitpy/webdriver_tests/pytest_runner.py
WebDriverTests/ChangeLog
WebDriverTests/TestExpectations.json

index bcbbb87..f9f36f4 100644 (file)
@@ -1,3 +1,17 @@
+2018-01-25  Carlos Garcia Campos  <cgarcia@igalia.com>
+
+        WebDriver: add support for slow tests
+        https://bugs.webkit.org/show_bug.cgi?id=182095
+
+        Reviewed by Carlos Alberto Lopez Perez.
+
+        Add timeout mark to slow tests.
+
+        * Scripts/webkitpy/webdriver_tests/pytest_runner.py:
+        (TestExpectationsMarker.__init__): Save the timeout.
+        (TestExpectationsMarker.pytest_collection_modifyitems): If test is slow use timeout * 5.
+        (run): Pass timeout to TestExpectationsMarker constructor.
+
 2018-01-24  Alex Christensen  <achristensen@webkit.org>
 
         Gracefully recover from NetworkProcess crashes in private browsing
index cdf1966..46ac8db 100644 (file)
@@ -130,8 +130,9 @@ class SubtestResultRecorder(object):
 
 class TestExpectationsMarker(object):
 
-    def __init__(self, expectations, ignore_param):
+    def __init__(self, expectations, timeout, ignore_param):
         self._expectations = expectations
+        self._timeout = timeout
         self._ignore_param = ignore_param
         self._base_dir = WebKitFinder(FileSystem()).path_from_webkit_base('WebDriverTests')
 
@@ -152,7 +153,10 @@ class TestExpectationsMarker(object):
     def pytest_collection_modifyitems(self, session, config, items):
         for item in items:
             test = os.path.relpath(str(item.fspath), self._base_dir)
-            expected = self._expectations.get_expectation(test, self._item_name(item))[0]
+            item_name = self._item_name(item)
+            if self._expectations.is_slow(test, item_name):
+                item.add_marker(pytest.mark.timeout(self._timeout * 5))
+            expected = self._expectations.get_expectation(test, item_name)[0]
             if expected == 'FAIL':
                 item.add_marker(pytest.mark.xfail)
             elif expected == 'TIMEOUT':
@@ -179,7 +183,7 @@ def collect(directory, args):
 def run(path, args, timeout, env, expectations, ignore_param=None):
     harness_recorder = HarnessResultRecorder()
     subtests_recorder = SubtestResultRecorder()
-    expectations_marker = TestExpectationsMarker(expectations, ignore_param)
+    expectations_marker = TestExpectationsMarker(expectations, timeout, ignore_param)
     _environ = dict(os.environ)
     os.environ.clear()
     os.environ.update(env)
index ded5f38..82aa0c4 100644 (file)
@@ -1,5 +1,16 @@
 2018-01-25  Carlos Garcia Campos  <cgarcia@igalia.com>
 
+        WebDriver: add support for slow tests
+        https://bugs.webkit.org/show_bug.cgi?id=182095
+
+        Reviewed by Carlos Alberto Lopez Perez.
+
+        Mark several tests as slow.
+
+        * TestExpectations.json:
+
+2018-01-25  Carlos Garcia Campos  <cgarcia@igalia.com>
+
         WebDriver: test imported/w3c/webdriver/tests/sessions/new_session/merge.py::test_merge_browserName fails
         https://bugs.webkit.org/show_bug.cgi?id=181985
 
index e0167b2..544589d 100644 (file)
             }
         }
     },
+    "imported/selenium/py/test/selenium/webdriver/common/frame_switching_tests.py": {
+        "subtests": {
+            "testShouldNotSwitchMagicallyToTheTopWindow": {
+                "expected": {"all": {"slow": true}}
+            }
+        }
+    },
     "imported/selenium/py/test/selenium/webdriver/common/interactions_tests.py": {
         "expected": {"all": {"status": ["SKIP"], "bug": "webkit.org/b/174616"}}
     },
             }
         }
     },
+    "imported/selenium/py/test/selenium/webdriver/common/select_class_tests.py": {
+        "subtests": {
+            "testSelectByIndexMultiple": {
+                "expected": {"all": {"slow": true}}
+            },
+            "testSelectByValueMultiple": {
+                "expected": {"all": {"slow": true}}
+            },
+            "testSelectByVisibleTextMultiple": {
+                "expected": {"all": {"slow": true}}
+            }
+        }
+    },
     "imported/selenium/py/test/selenium/webdriver/common/text_handling_tests.py": {
         "subtests": {
             "testReadALargeAmountOfData": {