WebDriver: ignore the driver in selenium test names when getting expectations
authorcarlosgc@webkit.org <carlosgc@webkit.org@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Thu, 18 Jan 2018 06:51:30 +0000 (06:51 +0000)
committercarlosgc@webkit.org <carlosgc@webkit.org@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Thu, 18 Jan 2018 06:51:30 +0000 (06:51 +0000)
https://bugs.webkit.org/show_bug.cgi?id=181738

Reviewed by Carlos Alberto Lopez Perez.

Tools:

In selenium tests, the driver is added as a parameter to every test which results in tests names like
foo[DriverName] or foo[DriverName-param2] in case of tests using more parameters. We don't want to include the
driver name in the test expectations file, so we need to remove it when querying the expectations.

* Scripts/webkitpy/webdriver_tests/pytest_runner.py:
(TestExpectationsMarker.__init__): Save the param to ignore.
(TestExpectationsMarker._item_name): Return the name of the test without the para to ignore.
(TestExpectationsMarker.pytest_collection_modifyitems): Use _item_name().
(run): Pass param to ignore to TestExpectationsMarker().
* Scripts/webkitpy/webdriver_tests/webdriver_selenium_executor.py:
(WebDriverSeleniumExecutor.__init__): Save the driver name.
(WebDriverSeleniumExecutor.run): Pass the driver name as param to ignore.

WebDriverTests:

Add some expectations for selenium tests.

* TestExpectations.json:

git-svn-id: https://svn.webkit.org/repository/webkit/trunk@227127 268f45cc-cd09-0410-ab3c-d52691b4dbfc

Tools/ChangeLog
Tools/Scripts/webkitpy/webdriver_tests/pytest_runner.py
Tools/Scripts/webkitpy/webdriver_tests/webdriver_selenium_executor.py
WebDriverTests/ChangeLog
WebDriverTests/TestExpectations.json

index cb92929..ee24498 100644 (file)
@@ -1,5 +1,25 @@
 2018-01-17  Carlos Garcia Campos  <cgarcia@igalia.com>
 
+        WebDriver: ignore the driver in selenium test names when getting expectations
+        https://bugs.webkit.org/show_bug.cgi?id=181738
+
+        Reviewed by Carlos Alberto Lopez Perez.
+
+        In selenium tests, the driver is added as a parameter to every test which results in tests names like
+        foo[DriverName] or foo[DriverName-param2] in case of tests using more parameters. We don't want to include the
+        driver name in the test expectations file, so we need to remove it when querying the expectations.
+
+        * Scripts/webkitpy/webdriver_tests/pytest_runner.py:
+        (TestExpectationsMarker.__init__): Save the param to ignore.
+        (TestExpectationsMarker._item_name): Return the name of the test without the para to ignore.
+        (TestExpectationsMarker.pytest_collection_modifyitems): Use _item_name().
+        (run): Pass param to ignore to TestExpectationsMarker().
+        * Scripts/webkitpy/webdriver_tests/webdriver_selenium_executor.py:
+        (WebDriverSeleniumExecutor.__init__): Save the driver name.
+        (WebDriverSeleniumExecutor.run): Pass the driver name as param to ignore.
+
+2018-01-17  Carlos Garcia Campos  <cgarcia@igalia.com>
+
         [GTK][WPE] Show unit tests unexpectedly passing in the bots
         https://bugs.webkit.org/show_bug.cgi?id=181726
 
index 9948763..cdf1966 100644 (file)
@@ -130,14 +130,29 @@ class SubtestResultRecorder(object):
 
 class TestExpectationsMarker(object):
 
-    def __init__(self, expectations):
+    def __init__(self, expectations, ignore_param):
         self._expectations = expectations
+        self._ignore_param = ignore_param
         self._base_dir = WebKitFinder(FileSystem()).path_from_webkit_base('WebDriverTests')
 
+    def _item_name(self, item):
+        if self._ignore_param is None:
+            return item.name
+
+        single_param = '[%s]' % self._ignore_param
+        if item.name.endswith(single_param):
+            return item.name[:-len(single_param)]
+
+        param = '[%s-' % self._ignore_param
+        if param in item.name:
+            return item.name.replace('%s-' % self._ignore_param, '')
+
+        return item.name
+
     def pytest_collection_modifyitems(self, session, config, items):
         for item in items:
             test = os.path.relpath(str(item.fspath), self._base_dir)
-            expected = self._expectations.get_expectation(test, item.name)[0]
+            expected = self._expectations.get_expectation(test, self._item_name(item))[0]
             if expected == 'FAIL':
                 item.add_marker(pytest.mark.xfail)
             elif expected == 'TIMEOUT':
@@ -161,10 +176,10 @@ def collect(directory, args):
     return collect_recorder.tests
 
 
-def run(path, args, timeout, env, expectations):
+def run(path, args, timeout, env, expectations, ignore_param=None):
     harness_recorder = HarnessResultRecorder()
     subtests_recorder = SubtestResultRecorder()
-    expectations_marker = TestExpectationsMarker(expectations)
+    expectations_marker = TestExpectationsMarker(expectations, ignore_param)
     _environ = dict(os.environ)
     os.environ.clear()
     os.environ.update(env)
index 0037fca..a1e9cb8 100644 (file)
@@ -41,10 +41,11 @@ _log = logging.getLogger(__name__)
 class WebDriverSeleniumExecutor(object):
 
     def __init__(self, driver, display_driver):
+        self._driver_name = driver.selenium_name()
         self._env = display_driver._setup_environ_for_test()
         self._env.update(driver.browser_env())
 
-        self._args = ['--driver=%s' % driver.selenium_name(), '--driver-binary=%s' % driver.binary_path().encode()]
+        self._args = ['--driver=%s' % self._driver_name, '--driver-binary=%s' % driver.binary_path().encode()]
         browser_path = driver.browser_path().encode()
         if browser_path:
             self._args.extend(['--browser-binary=%s' % browser_path])
@@ -59,4 +60,4 @@ class WebDriverSeleniumExecutor(object):
         return pytest_runner.collect(directory, self._args)
 
     def run(self, test, timeout, expectations):
-        return pytest_runner.run(test, self._args, timeout, self._env, expectations)
+        return pytest_runner.run(test, self._args, timeout, self._env, expectations, self._driver_name)
index 79d8090..41a582a 100644 (file)
@@ -1,5 +1,16 @@
 2018-01-17  Carlos Garcia Campos  <cgarcia@igalia.com>
 
+        WebDriver: ignore the driver in selenium test names when getting expectations
+        https://bugs.webkit.org/show_bug.cgi?id=181738
+
+        Reviewed by Carlos Alberto Lopez Perez.
+
+        Add some expectations for selenium tests.
+
+        * TestExpectations.json:
+
+2018-01-17  Carlos Garcia Campos  <cgarcia@igalia.com>
+
         WebDriver: add support for test expectations
         https://bugs.webkit.org/show_bug.cgi?id=180420
 
index 73d319f..24f399e 100644 (file)
@@ -1,4 +1,111 @@
 {
+    "imported/selenium/py/test/selenium/webdriver/common/alerts_tests.py": {
+        "subtests": {
+            "testShouldAllowUsersToAcceptAnAlertInAFrame": {
+                "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/181727"}}
+            },
+            "testShouldAllowUsersToAcceptAnAlertInANestedFrame": {
+                "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/181727"}}
+            }
+        }
+    },
+    "imported/selenium/py/test/selenium/webdriver/common/api_example_tests.py": {
+        "subtests": {
+            "testChangeWindowSize": {
+                "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/181728"}}
+            },
+            "testGetLogTypes": {
+                "expected": {"all": {"status": ["SKIP"]}}
+            },
+            "testGetLog": {
+                "expected": {"all": {"status": ["SKIP"]}}
+            }
+        }
+    },
+    "imported/selenium/py/test/selenium/webdriver/common/appcache_tests.py": {
+        "expected": {"all": {"status": ["SKIP"]}}
+    },
+    "imported/selenium/py/test/selenium/webdriver/common/click_scrolling_tests.py": {
+        "subtests": {
+            "testShouldBeAbleToClickOnAnElementHiddenByDoubleOverflow": {
+                "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/174902"}}
+            },
+            "testShouldNotScrollIfAlreadyScrolledAndElementIsInView": {
+                "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/181729"}}
+            },
+            "testShouldScrollOverflowElementsIfClickPointIsOutOfViewButElementIsInView": {
+                "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/181730"}}
+            },
+            "testShouldBeAbleToClickElementThatIsOutOfViewInAFrameThatIsOutOfView": {
+                "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/181731"}}
+            },
+            "testShouldBeAbleToClickElementThatIsOutOfViewInANestedFrame": {
+                "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/181731"}}
+            },
+            "testShouldBeAbleToClickElementThatIsOutOfViewInANestedFrameThatIsOutOfView": {
+                "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/181731"}}
+            },
+            "testShouldNotScrollWhenGettingElementSize": {
+                "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/181731"}}
+            },
+            "testShouldBeAbleToClickElementInATallFrame": {
+                "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/181731"}}
+            }
+        }
+    },
+    "imported/selenium/py/test/selenium/webdriver/common/correct_event_firing_tests.py": {
+        "subtests": {
+            "testShouldEmitOnChangeEventsWhenChangingTheStateOfACheckbox": {
+                "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/181732"}}
+            },
+            "testClearingAnElementShouldCauseTheOnChangeHandlerToFire": {
+                "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/181732"}}
+            }
+        }
+    },
+    "imported/selenium/py/test/selenium/webdriver/common/form_handling_tests.py": {
+        "subtests": {
+            "testShouldThrowAnExceptionWhenSelectingAnUnselectableElement": {
+                "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/181733"}}
+            }
+        }
+    },
+    "imported/selenium/py/test/selenium/webdriver/common/interactions_tests.py": {
+        "expected": {"all": {"status": ["SKIP"], "bug": "webkit.org/b/174616"}}
+    },
+    "imported/selenium/py/test/selenium/webdriver/common/position_and_size_tests.py": {
+        "subtests": {
+            "testShouldScrollPageAndGetCoordinatesOfAnElementThatIsOutOfViewPort": {
+                "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/181734"}}
+            },
+            "testShouldGetCoordinatesOfAnElementInAFrame": {
+                "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/181735"}}
+            },
+            "testShouldGetCoordinatesOfAnElementInANestedFrame": {
+                "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/181735"}}
+            }
+        }
+    },
+    "imported/selenium/py/test/selenium/webdriver/common/rendered_webelement_tests.py": {
+        "subtests": {
+            "testShouldPickUpStyleOfAnElement": {
+                "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/181736"}}
+            },
+            "testShouldAllowInheritedStylesToBeUsed": {
+                "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/181736"}}
+            }
+        }
+    },
+    "imported/selenium/py/test/selenium/webdriver/common/text_handling_tests.py": {
+        "subtests": {
+            "testReadALargeAmountOfData": {
+                "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/181737"}}
+            }
+        }
+    },
+    "imported/selenium/py/test/selenium/webdriver/common/w3c_interaction_tests.py": {
+        "expected": {"all": {"status": ["SKIP"], "bug": "webkit.org/b/174616"}}
+    },
     "imported/w3c/webdriver/tests/actions/key.py": {
         "expected": {"all": {"status": ["SKIP"], "bug": "webkit.org/b/174616"}}
     },