WebDriver: use in-view center point for clicks instead of bounding box center point
authorcarlosgc@webkit.org <carlosgc@webkit.org@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Sat, 5 Aug 2017 08:11:11 +0000 (08:11 +0000)
committercarlosgc@webkit.org <carlosgc@webkit.org@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Sat, 5 Aug 2017 08:11:11 +0000 (08:11 +0000)
https://bugs.webkit.org/show_bug.cgi?id=174863

Reviewed by Simon Fraser.

Source/WebCore:

Make DOMRect, and FloatPoint::narrowPrecision() available to WebKit layer. Also add
FrameView::clientToDocumentPoint().

* WebCore.xcodeproj/project.pbxproj:
* dom/Element.h:
* page/FrameView.h:
* platform/graphics/FloatPoint.h:

Source/WebDriver:

The center of the element bounding box is not always part of the element, like in multiline links, for example.

11.1 Element Interactability.
https://www.w3.org/TR/webdriver/#dfn-in-view-center-point

* CommandResult.cpp:
(WebDriver::CommandResult::httpStatusCode): Add ElementClickIntercepted and ElementNotInteractable errors.
(WebDriver::CommandResult::errorString): Ditto.
* CommandResult.h: Ditto.
* Session.cpp:
(WebDriver::Session::computeElementLayout): Get the in-view center point and isObscured from the result too.
(WebDriver::Session::getElementRect): Ignore in-view center point and isObscured.
(WebDriver::Session::elementClick): Fail in case the element is not interactable or is obscured.
* Session.h:

Source/WebKit:

Change computeElementLayout to also return the in-view center point and whether it's obscured by another
element.

* UIProcess/Automation/Automation.json: Add optional inViewCenterPoint to the result and isObscured.
* UIProcess/Automation/WebAutomationSession.cpp:
(WebKit::WebAutomationSession::didComputeElementLayout): Handle inViewCenterPoint and isObscured.
* UIProcess/Automation/WebAutomationSession.h:
* UIProcess/Automation/WebAutomationSession.messages.in:
* WebProcess/Automation/WebAutomationSessionProxy.cpp:
(WebKit::elementInViewClientCenterPoint): Get the client in-view center point and whether it's obscured
according to the spec.
(WebKit::WebAutomationSessionProxy::computeElementLayout): Pass inViewCenterPoint and isObscured to
DidComputeElementLayout message.

git-svn-id: https://svn.webkit.org/repository/webkit/trunk@220314 268f45cc-cd09-0410-ab3c-d52691b4dbfc

17 files changed:
Source/WebCore/ChangeLog
Source/WebCore/WebCore.xcodeproj/project.pbxproj
Source/WebCore/dom/Element.h
Source/WebCore/page/FrameView.cpp
Source/WebCore/page/FrameView.h
Source/WebCore/platform/graphics/FloatPoint.h
Source/WebDriver/ChangeLog
Source/WebDriver/CommandResult.cpp
Source/WebDriver/CommandResult.h
Source/WebDriver/Session.cpp
Source/WebDriver/Session.h
Source/WebKit/ChangeLog
Source/WebKit/UIProcess/Automation/Automation.json
Source/WebKit/UIProcess/Automation/WebAutomationSession.cpp
Source/WebKit/UIProcess/Automation/WebAutomationSession.h
Source/WebKit/UIProcess/Automation/WebAutomationSession.messages.in
Source/WebKit/WebProcess/Automation/WebAutomationSessionProxy.cpp

index 8cf33c7410fbae1f860573d54521f5f94bbbc5ec..dbb97bcf4f05ee64e53ac7102871a1480fee3c8f 100644 (file)
@@ -1,3 +1,18 @@
+2017-08-05  Carlos Garcia Campos  <cgarcia@igalia.com>
+
+        WebDriver: use in-view center point for clicks instead of bounding box center point
+        https://bugs.webkit.org/show_bug.cgi?id=174863
+
+        Reviewed by Simon Fraser.
+
+        Make DOMRect, and FloatPoint::narrowPrecision() available to WebKit layer. Also add
+        FrameView::clientToDocumentPoint().
+
+        * WebCore.xcodeproj/project.pbxproj:
+        * dom/Element.h:
+        * page/FrameView.h:
+        * platform/graphics/FloatPoint.h:
+
 2017-08-05  Carlos Garcia Campos  <cgarcia@igalia.com>
 
         getClientRects doesn't work with list box option elements
index 77b54566ff23bf73b3d762484504d1c3d4e26190..ecc6556b9cfb3f462cf8bd3966a819839ef3b4fc 100644 (file)
                0F3F0E5A157030C3006DA57F /* RenderGeometryMap.h in Headers */ = {isa = PBXBuildFile; fileRef = 0F3F0E58157030C3006DA57F /* RenderGeometryMap.h */; };
                0F43C85D189E10CF00019AE2 /* PerformanceTiming.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 0F43C85C189E10CF00019AE2 /* PerformanceTiming.cpp */; };
                0F43C85F189E15A600019AE2 /* JSPerformanceTiming.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 0F43C85E189E15A600019AE2 /* JSPerformanceTiming.cpp */; };
-               0F4710AF1DB56AFC002DCEC3 /* DOMRect.h in Headers */ = {isa = PBXBuildFile; fileRef = 0F4710A91DB56AFC002DCEC3 /* DOMRect.h */; };
-               0F4710B11DB56AFC002DCEC3 /* DOMRectInit.h in Headers */ = {isa = PBXBuildFile; fileRef = 0F4710AB1DB56AFC002DCEC3 /* DOMRectInit.h */; };
-               0F4710B31DB56AFC002DCEC3 /* DOMRectReadOnly.h in Headers */ = {isa = PBXBuildFile; fileRef = 0F4710AD1DB56AFC002DCEC3 /* DOMRectReadOnly.h */; };
+               0F4710AF1DB56AFC002DCEC3 /* DOMRect.h in Headers */ = {isa = PBXBuildFile; fileRef = 0F4710A91DB56AFC002DCEC3 /* DOMRect.h */; settings = {ATTRIBUTES = (Private, ); }; };
+               0F4710B11DB56AFC002DCEC3 /* DOMRectInit.h in Headers */ = {isa = PBXBuildFile; fileRef = 0F4710AB1DB56AFC002DCEC3 /* DOMRectInit.h */; settings = {ATTRIBUTES = (Private, ); }; };
+               0F4710B31DB56AFC002DCEC3 /* DOMRectReadOnly.h in Headers */ = {isa = PBXBuildFile; fileRef = 0F4710AD1DB56AFC002DCEC3 /* DOMRectReadOnly.h */; settings = {ATTRIBUTES = (Private, ); }; };
                0F4710BB1DB56BE8002DCEC3 /* JSDOMRect.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 0F4710B51DB56BE8002DCEC3 /* JSDOMRect.cpp */; };
                0F4710BC1DB56BE8002DCEC3 /* JSDOMRect.h in Headers */ = {isa = PBXBuildFile; fileRef = 0F4710B61DB56BE8002DCEC3 /* JSDOMRect.h */; };
                0F4710BD1DB56BE8002DCEC3 /* JSDOMRectInit.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 0F4710B71DB56BE8002DCEC3 /* JSDOMRectInit.cpp */; };
                4671E0661D67A59600C6B497 /* CanvasPath.h in Headers */ = {isa = PBXBuildFile; fileRef = 4671E0641D67A57B00C6B497 /* CanvasPath.h */; };
                467302021C4EFE7800BCB357 /* IgnoreOpensDuringUnloadCountIncrementer.h in Headers */ = {isa = PBXBuildFile; fileRef = 467302011C4EFE6600BCB357 /* IgnoreOpensDuringUnloadCountIncrementer.h */; };
                468344DF1EDDFAAA00B7795B /* DOMRectList.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 468344DD1EDDFA5F00B7795B /* DOMRectList.cpp */; };
-               468344E01EDDFAAA00B7795B /* DOMRectList.h in Headers */ = {isa = PBXBuildFile; fileRef = 468344DE1EDDFA5F00B7795B /* DOMRectList.h */; };
+               468344E01EDDFAAA00B7795B /* DOMRectList.h in Headers */ = {isa = PBXBuildFile; fileRef = 468344DE1EDDFA5F00B7795B /* DOMRectList.h */; settings = {ATTRIBUTES = (Private, ); }; };
                4689F1AF1267BAE100E8D380 /* FileMetadata.h in Headers */ = {isa = PBXBuildFile; fileRef = 4689F1AE1267BAE100E8D380 /* FileMetadata.h */; };
                46B63F6C1C6E8D19002E914B /* JSEventTargetCustom.h in Headers */ = {isa = PBXBuildFile; fileRef = 46B63F6B1C6E8CDF002E914B /* JSEventTargetCustom.h */; settings = {ATTRIBUTES = (Private, ); }; };
                46C696CB1E7205F700597937 /* CPUMonitor.h in Headers */ = {isa = PBXBuildFile; fileRef = 46C696C91E7205E400597937 /* CPUMonitor.h */; settings = {ATTRIBUTES = (Private, ); }; };
index 1d75109b230be70290c9695c7add0a50e3eaf663..42f46cfb0c3ed93f4ba5cef939ef04bb031361e8 100644 (file)
@@ -175,7 +175,7 @@ public:
 
     FloatRect boundingClientRect();
 
-    Ref<DOMRectList> getClientRects();
+    WEBCORE_EXPORT Ref<DOMRectList> getClientRects();
     Ref<DOMRect> getBoundingClientRect();
 
     // Returns the absolute bounding box translated into client coordinates.
index f67b83a7b1fbaf2e5a11be310852760edf159e7c..a6cb59d5080d093d57c4bd84ffc4eae1b6439ba1 100644 (file)
@@ -4951,6 +4951,12 @@ FloatPoint FrameView::documentToClientPoint(FloatPoint p) const
     return p;
 }
 
+FloatPoint FrameView::clientToDocumentPoint(FloatPoint point) const
+{
+    point.move(-documentToClientOffset());
+    return point;
+}
+
 FloatRect FrameView::layoutViewportToAbsoluteRect(FloatRect rect) const
 {
     ASSERT(frame().settings().visualViewportEnabled());
index dbc085ed4d2adbdb008fd4d4c4cd680f2420b028..4d4ef9afe70c97e47b983438bc3e96a1f8a4065f 100644 (file)
@@ -477,6 +477,7 @@ public:
     FloatSize documentToClientOffset() const;
     FloatRect documentToClientRect(FloatRect) const;
     FloatPoint documentToClientPoint(FloatPoint) const;
+    WEBCORE_EXPORT FloatPoint clientToDocumentPoint(FloatPoint) const;
 
     FloatRect layoutViewportToAbsoluteRect(FloatRect) const;
     FloatPoint layoutViewportToAbsolutePoint(FloatPoint) const;
index 6ed4d21a292cdb0a4d1a5a21c68246edcf7150b6..07a79b6c2cef5cf450d547268df31e7c2a6dd2dd 100644 (file)
@@ -68,7 +68,7 @@ public:
 
     static FloatPoint zero() { return FloatPoint(); }
 
-    static FloatPoint narrowPrecision(double x, double y);
+    WEBCORE_EXPORT static FloatPoint narrowPrecision(double x, double y);
 
     float x() const { return m_x; }
     float y() const { return m_y; }
index 8935ebabd428cfa74129fe3db9ee42cf18c0f693..a9b3c68cb790bb148c94dcfd78fbc1aebbcbbe23 100644 (file)
@@ -1,3 +1,25 @@
+2017-08-05  Carlos Garcia Campos  <cgarcia@igalia.com>
+
+        WebDriver: use in-view center point for clicks instead of bounding box center point
+        https://bugs.webkit.org/show_bug.cgi?id=174863
+
+        Reviewed by Simon Fraser.
+
+        The center of the element bounding box is not always part of the element, like in multiline links, for example.
+
+        11.1 Element Interactability.
+        https://www.w3.org/TR/webdriver/#dfn-in-view-center-point
+
+        * CommandResult.cpp:
+        (WebDriver::CommandResult::httpStatusCode): Add ElementClickIntercepted and ElementNotInteractable errors.
+        (WebDriver::CommandResult::errorString): Ditto.
+        * CommandResult.h: Ditto.
+        * Session.cpp:
+        (WebDriver::Session::computeElementLayout): Get the in-view center point and isObscured from the result too.
+        (WebDriver::Session::getElementRect): Ignore in-view center point and isObscured.
+        (WebDriver::Session::elementClick): Fail in case the element is not interactable or is obscured.
+        * Session.h:
+
 2017-08-01  Michael Catanzaro  <mcatanzaro@igalia.com>
 
         [CMake] WebKitFS.cmake depends on options set in Option cmake files that are included later
index 3798a1812b1580f00f9c166b938e33c64f6a6f04..633b8995bbaba202c7f5a372fa6f78c08a94300b 100644 (file)
@@ -122,6 +122,8 @@ unsigned CommandResult::httpStatusCode() const
     // §6.6 Handling Errors.
     // https://www.w3.org/TR/webdriver/#handling-errors
     switch (m_errorCode.value()) {
+    case ErrorCode::ElementClickIntercepted:
+    case ErrorCode::ElementNotInteractable:
     case ErrorCode::InvalidArgument:
     case ErrorCode::InvalidElementState:
     case ErrorCode::InvalidSelector:
@@ -152,6 +154,10 @@ String CommandResult::errorString() const
     ASSERT(isError());
 
     switch (m_errorCode.value()) {
+    case ErrorCode::ElementClickIntercepted:
+        return ASCIILiteral("element click intercepted");
+    case ErrorCode::ElementNotInteractable:
+        return ASCIILiteral("element not interactable");
     case ErrorCode::InvalidArgument:
         return ASCIILiteral("invalid argument");
     case ErrorCode::InvalidElementState:
index 296e4b71119c8d9057b0637de0e3e53efa73172e..ec533980147142939cc07da666d28fdad6fffd3d 100644 (file)
@@ -39,6 +39,8 @@ public:
     // §6.6 Handling Errors.
     // https://www.w3.org/TR/webdriver/#handling-errors
     enum class ErrorCode {
+        ElementClickIntercepted,
+        ElementNotInteractable,
         InvalidArgument,
         InvalidElementState,
         InvalidSelector,
index b335c1d0f4a16208fe88caa49e31cd2c6f7de0e9..8a8f105c0abe6d7cc10d08293975337280257bd5 100644 (file)
@@ -580,7 +580,7 @@ String Session::extractElementID(InspectorValue& value)
     return elementID;
 }
 
-void Session::computeElementLayout(const String& elementID, OptionSet<ElementLayoutOption> options, Function<void (std::optional<Rect>&&, RefPtr<InspectorObject>&&)>&& completionHandler)
+void Session::computeElementLayout(const String& elementID, OptionSet<ElementLayoutOption> options, Function<void (std::optional<Rect>&&, std::optional<Point>&&, bool, RefPtr<InspectorObject>&&)>&& completionHandler)
 {
     ASSERT(m_toplevelBrowsingContext.value());
 
@@ -592,12 +592,12 @@ void Session::computeElementLayout(const String& elementID, OptionSet<ElementLay
     parameters->setBoolean(ASCIILiteral("useViewportCoordinates"), options.contains(ElementLayoutOption::UseViewportCoordinates));
     m_host->sendCommandToBackend(ASCIILiteral("computeElementLayout"), WTFMove(parameters), [this, protectedThis = makeRef(*this), completionHandler = WTFMove(completionHandler)](SessionHost::CommandResponse&& response) mutable {
         if (response.isError || !response.responseObject) {
-            completionHandler(std::nullopt, WTFMove(response.responseObject));
+            completionHandler(std::nullopt, std::nullopt, false, WTFMove(response.responseObject));
             return;
         }
         RefPtr<InspectorObject> rectObject;
         if (!response.responseObject->getObject(ASCIILiteral("rect"), rectObject)) {
-            completionHandler(std::nullopt, nullptr);
+            completionHandler(std::nullopt, std::nullopt, false, nullptr);
             return;
         }
         std::optional<int> elementX;
@@ -611,7 +611,7 @@ void Session::computeElementLayout(const String& elementID, OptionSet<ElementLay
             }
         }
         if (!elementX || !elementY) {
-            completionHandler(std::nullopt, nullptr);
+            completionHandler(std::nullopt, std::nullopt, false, nullptr);
             return;
         }
         std::optional<int> elementWidth;
@@ -625,11 +625,29 @@ void Session::computeElementLayout(const String& elementID, OptionSet<ElementLay
             }
         }
         if (!elementWidth || !elementHeight) {
-            completionHandler(std::nullopt, nullptr);
+            completionHandler(std::nullopt, std::nullopt, false, nullptr);
             return;
         }
         Rect rect = { { elementX.value(), elementY.value() }, { elementWidth.value(), elementHeight.value() } };
-        completionHandler(rect, nullptr);
+
+        bool isObscured;
+        if (!response.responseObject->getBoolean(ASCIILiteral("isObscured"), isObscured)) {
+            completionHandler(std::nullopt, std::nullopt, false, nullptr);
+            return;
+        }
+        RefPtr<InspectorObject> inViewCenterPointObject;
+        if (!response.responseObject->getObject(ASCIILiteral("inViewCenterPoint"), inViewCenterPointObject)) {
+            completionHandler(rect, std::nullopt, isObscured, nullptr);
+            return;
+        }
+        int inViewCenterPointX, inViewCenterPointY;
+        if (!inViewCenterPointObject->getInteger(ASCIILiteral("x"), inViewCenterPointX)
+            || !inViewCenterPointObject->getInteger(ASCIILiteral("y"), inViewCenterPointY)) {
+            completionHandler(std::nullopt, std::nullopt, isObscured, nullptr);
+            return;
+        }
+        Point inViewCenterPoint = { inViewCenterPointX, inViewCenterPointY };
+        completionHandler(rect, inViewCenterPoint, isObscured, nullptr);
     });
 }
 
@@ -830,7 +848,7 @@ void Session::getElementRect(const String& elementID, Function<void (CommandResu
         return;
     }
 
-    computeElementLayout(elementID, { }, [this, protectedThis = makeRef(*this), completionHandler = WTFMove(completionHandler)](std::optional<Rect>&& rect, RefPtr<InspectorObject>&& error) {
+    computeElementLayout(elementID, { }, [this, protectedThis = makeRef(*this), completionHandler = WTFMove(completionHandler)](std::optional<Rect>&& rect, std::optional<Point>&&, bool, RefPtr<InspectorObject>&& error) {
         if (!rect || error) {
             completionHandler(CommandResult::fail(WTFMove(error)));
             return;
@@ -987,15 +1005,21 @@ void Session::elementClick(const String& elementID, Function<void (CommandResult
 
     OptionSet<ElementLayoutOption> options = ElementLayoutOption::ScrollIntoViewIfNeeded;
     options |= ElementLayoutOption::UseViewportCoordinates;
-    computeElementLayout(elementID, options, [this, protectedThis = makeRef(*this), completionHandler = WTFMove(completionHandler)](std::optional<Rect>&& rect, RefPtr<InspectorObject>&& error) mutable {
+    computeElementLayout(elementID, options, [this, protectedThis = makeRef(*this), completionHandler = WTFMove(completionHandler)](std::optional<Rect>&& rect, std::optional<Point>&& inViewCenter, bool isObscured, RefPtr<InspectorObject>&& error) mutable {
         if (!rect || error) {
             completionHandler(CommandResult::fail(WTFMove(error)));
             return;
         }
+        if (isObscured) {
+            completionHandler(CommandResult::fail(CommandResult::ErrorCode::ElementClickIntercepted));
+            return;
+        }
+        if (!inViewCenter) {
+            completionHandler(CommandResult::fail(CommandResult::ErrorCode::ElementNotInteractable));
+            return;
+        }
 
-        // FIXME: the center of the bounding box is not always part of the element.
-        performMouseInteraction(rect.value().origin.x + rect.value().size.width / 2, rect.value().origin.y + rect.value().size.height / 2,
-            MouseButton::Left, MouseInteraction::SingleClick, WTFMove(completionHandler));
+        performMouseInteraction(inViewCenter.value().x, inViewCenter.value().y, MouseButton::Left, MouseInteraction::SingleClick, WTFMove(completionHandler));
 
         waitForNavigationToComplete(WTFMove(completionHandler));
     });
index 143e9ea45094b2aebcfcb97bb11e790f7f9ec49b..5d4f24fdc9bc600d9b4463978c95ff2a00b0afcd 100644 (file)
@@ -133,7 +133,7 @@ private:
         ScrollIntoViewIfNeeded = 1 << 0,
         UseViewportCoordinates = 1 << 1,
     };
-    void computeElementLayout(const String& elementID, OptionSet<ElementLayoutOption>, Function<void (std::optional<Rect>&&, RefPtr<Inspector::InspectorObject>&&)>&&);
+    void computeElementLayout(const String& elementID, OptionSet<ElementLayoutOption>, Function<void (std::optional<Rect>&&, std::optional<Point>&&, bool, RefPtr<Inspector::InspectorObject>&&)>&&);
 
     enum class MouseButton { None, Left, Middle, Right };
     enum class MouseInteraction { Move, Down, Up, SingleClick, DoubleClick };
index 32a60a4cf0505178203d5b3b50c70dea8ddb539e..fa0098fba886dc0b6c39a11f43b469cb4135f9ea 100644 (file)
@@ -1,3 +1,24 @@
+2017-08-05  Carlos Garcia Campos  <cgarcia@igalia.com>
+
+        WebDriver: use in-view center point for clicks instead of bounding box center point
+        https://bugs.webkit.org/show_bug.cgi?id=174863
+
+        Reviewed by Simon Fraser.
+
+        Change computeElementLayout to also return the in-view center point and whether it's obscured by another
+        element.
+
+        * UIProcess/Automation/Automation.json: Add optional inViewCenterPoint to the result and isObscured.
+        * UIProcess/Automation/WebAutomationSession.cpp:
+        (WebKit::WebAutomationSession::didComputeElementLayout): Handle inViewCenterPoint and isObscured.
+        * UIProcess/Automation/WebAutomationSession.h:
+        * UIProcess/Automation/WebAutomationSession.messages.in:
+        * WebProcess/Automation/WebAutomationSessionProxy.cpp:
+        (WebKit::elementInViewClientCenterPoint): Get the client in-view center point and whether it's obscured
+        according to the spec.
+        (WebKit::WebAutomationSessionProxy::computeElementLayout): Pass inViewCenterPoint and isObscured to
+        DidComputeElementLayout message.
+
 2017-08-05  Brian Burg  <bburg@apple.com>
 
         Web Automation: files selected for upload should be matched against 'accept' attribute values case-insensitively
index f14fbedd103acfc40c19dbdd0c065317b01405b4..2e026ec953df837f3619cccdef417a3a3c4acd88 100644 (file)
                 { "name": "useViewportCoordinates", "optional": true, "type": "boolean", "description": "If the result coordinates should be represented as viewport coordinates or not. Defaults to false, which means coordinates should be represented as page coordinates." }
             ],
             "returns": [
-                { "name": "rect", "$ref": "Rect", "description": "The layout rect for the requested element. Specified in page or viewport coordinates based on the useViewportCoordinates parameter." }
+                { "name": "rect", "$ref": "Rect", "description": "The layout rect for the requested element. Specified in page or viewport coordinates based on the useViewportCoordinates parameter." },
+                { "name": "inViewCenterPoint", "optional": true, "$ref": "Point", "description": "The in-view center point for the requested element. Specified in page or viewport coordinates based on the useViewportCoordinates parameter." },
+                { "name": "isObscured", "type": "boolean", "description": "If the in-view center point of the requested element is obscured by another element." }
             ],
             "async": true
         },
index 84de5e7d992f73be11ab72d32cdb07de18260104..7f6d6cd1218a3711b89d4be593f2b3cccc47ca15 100644 (file)
@@ -743,7 +743,7 @@ void WebAutomationSession::computeElementLayout(Inspector::ErrorString& errorStr
     page->process().send(Messages::WebAutomationSessionProxy::ComputeElementLayout(page->pageID(), frameID.value(), nodeHandle, scrollIntoViewIfNeeded, useViewportCoordinates, callbackID), 0);
 }
 
-void WebAutomationSession::didComputeElementLayout(uint64_t callbackID, WebCore::IntRect rect, const String& errorType)
+void WebAutomationSession::didComputeElementLayout(uint64_t callbackID, WebCore::IntRect rect, std::optional<WebCore::IntPoint> inViewCenterPoint, bool isObscured, const String& errorType)
 {
     auto callback = m_computeElementLayoutCallbacks.take(callbackID);
     if (!callback)
@@ -769,7 +769,17 @@ void WebAutomationSession::didComputeElementLayout(uint64_t callbackID, WebCore:
         .setSize(WTFMove(sizeObject))
         .release();
 
-    callback->sendSuccess(WTFMove(rectObject));
+    if (!inViewCenterPoint) {
+        callback->sendSuccess(WTFMove(rectObject), nullptr, isObscured);
+        return;
+    }
+
+    auto inViewCenterPointObject = Inspector::Protocol::Automation::Point::create()
+        .setX(inViewCenterPoint.value().x())
+        .setY(inViewCenterPoint.value().y())
+        .release();
+
+    callback->sendSuccess(WTFMove(rectObject), WTFMove(inViewCenterPointObject), isObscured);
 }
 
 void WebAutomationSession::isShowingJavaScriptDialog(Inspector::ErrorString& errorString, const String& browsingContextHandle, bool* result)
index 7b7657a95cc9ba6d2261f91743752916c5b0dcf8..c9ce90165f0eae64749e39dc69bacdb6f1dc4ed1 100644 (file)
@@ -174,7 +174,7 @@ private:
     void didEvaluateJavaScriptFunction(uint64_t callbackID, const String& result, const String& errorType);
     void didResolveChildFrame(uint64_t callbackID, uint64_t frameID, const String& errorType);
     void didResolveParentFrame(uint64_t callbackID, uint64_t frameID, const String& errorType);
-    void didComputeElementLayout(uint64_t callbackID, WebCore::IntRect, const String& errorType);
+    void didComputeElementLayout(uint64_t callbackID, WebCore::IntRect, std::optional<WebCore::IntPoint>, bool isObscured, const String& errorType);
     void didTakeScreenshot(uint64_t callbackID, const ShareableBitmap::Handle&, const String& errorType);
     void didGetCookiesForFrame(uint64_t callbackID, Vector<WebCore::Cookie>, const String& errorType);
     void didDeleteCookie(uint64_t callbackID, const String& errorType);
index cb4d56288308cb6ff672fa63c09dd56fcd901105..f38ac953f613d3f3b84ee2f4ea045e824792cea0 100644 (file)
@@ -26,7 +26,7 @@ messages -> WebAutomationSession {
     DidResolveChildFrame(uint64_t callbackID, uint64_t frameID, String errorType)
     DidResolveParentFrame(uint64_t callbackID, uint64_t frameID, String errorType)
 
-    DidComputeElementLayout(uint64_t callbackID, WebCore::IntRect rect, String errorType)
+    DidComputeElementLayout(uint64_t callbackID, WebCore::IntRect rect, std::optional<WebCore::IntPoint> inViewCenterPoint, bool isObscured, String errorType)
 
     DidTakeScreenshot(uint64_t callbackID, WebKit::ShareableBitmap::Handle imageDataHandle, String errorType)
 
index 8ee427427868fce21296a8581878a911886537c1..52d347d37f0f02ed602963bccb81080e9fcf5783 100644 (file)
@@ -41,6 +41,8 @@
 #include <JavaScriptCore/JSStringRefPrivate.h>
 #include <JavaScriptCore/OpaqueJSString.h>
 #include <WebCore/CookieJar.h>
+#include <WebCore/DOMRect.h>
+#include <WebCore/DOMRectList.h>
 #include <WebCore/DOMWindow.h>
 #include <WebCore/Frame.h>
 #include <WebCore/FrameTree.h>
@@ -470,12 +472,43 @@ void WebAutomationSessionProxy::focusFrame(uint64_t pageID, uint64_t frameID)
     coreDOMWindow->focus(true);
 }
 
+static std::optional<WebCore::FloatPoint> elementInViewClientCenterPoint(WebCore::Element& element, bool& isObscured)
+{
+    // §11.1 Element Interactability.
+    // https://www.w3.org/TR/webdriver/#dfn-in-view-center-point
+    auto* clientRect = element.getClientRects()->item(0);
+    if (!clientRect)
+        return std::nullopt;
+
+    auto clientCenterPoint = WebCore::FloatPoint::narrowPrecision(0.5 * (clientRect->left() + clientRect->right()), 0.5 * (clientRect->top() + clientRect->bottom()));
+    auto elementList = element.treeScope().elementsFromPoint(clientCenterPoint.x(), clientCenterPoint.y());
+    if (elementList.isEmpty()) {
+        // An element is obscured if the pointer-interactable paint tree at its center point is empty,
+        // or the first element in this tree is not an inclusive descendant of itself.
+        // https://w3c.github.io/webdriver/webdriver-spec.html#dfn-obscured
+        isObscured = true;
+        return clientCenterPoint;
+    }
+
+    auto index = elementList.findMatching([&element] (auto& item) { return item.get() == &element; });
+    if (index == notFound)
+        return std::nullopt;
+
+    if (index) {
+        // Element is not the first one in the list.
+        auto firstElement = elementList[0];
+        isObscured = !firstElement->isDescendantOf(element);
+    }
+
+    return clientCenterPoint;
+}
+
 void WebAutomationSessionProxy::computeElementLayout(uint64_t pageID, uint64_t frameID, String nodeHandle, bool scrollIntoViewIfNeeded, bool useViewportCoordinates, uint64_t callbackID)
 {
     WebPage* page = WebProcess::singleton().webPage(pageID);
     if (!page) {
         String windowNotFoundErrorType = Inspector::Protocol::AutomationHelpers::getEnumConstantValue(Inspector::Protocol::Automation::ErrorMessage::WindowNotFound);
-        WebProcess::singleton().parentProcessConnection()->send(Messages::WebAutomationSession::DidComputeElementLayout(callbackID, WebCore::IntRect(), windowNotFoundErrorType), 0);
+        WebProcess::singleton().parentProcessConnection()->send(Messages::WebAutomationSession::DidComputeElementLayout(callbackID, { }, std::nullopt, false, windowNotFoundErrorType), 0);
         return;
     }
 
@@ -483,40 +516,39 @@ void WebAutomationSessionProxy::computeElementLayout(uint64_t pageID, uint64_t f
     String nodeNotFoundErrorType = Inspector::Protocol::AutomationHelpers::getEnumConstantValue(Inspector::Protocol::Automation::ErrorMessage::NodeNotFound);
 
     WebFrame* frame = frameID ? WebProcess::singleton().webFrame(frameID) : page->mainWebFrame();
-    if (!frame) {
-        WebProcess::singleton().parentProcessConnection()->send(Messages::WebAutomationSession::DidComputeElementLayout(callbackID, WebCore::IntRect(), frameNotFoundErrorType), 0);
+    if (!frame || !frame->coreFrame() || !frame->coreFrame()->view()) {
+        WebProcess::singleton().parentProcessConnection()->send(Messages::WebAutomationSession::DidComputeElementLayout(callbackID, { }, std::nullopt, false, frameNotFoundErrorType), 0);
         return;
     }
 
     WebCore::Element* coreElement = elementForNodeHandle(*frame, nodeHandle);
     if (!coreElement) {
-        WebProcess::singleton().parentProcessConnection()->send(Messages::WebAutomationSession::DidComputeElementLayout(callbackID, WebCore::IntRect(), nodeNotFoundErrorType), 0);
+        WebProcess::singleton().parentProcessConnection()->send(Messages::WebAutomationSession::DidComputeElementLayout(callbackID, { }, std::nullopt, false, nodeNotFoundErrorType), 0);
         return;
     }
 
-    if (scrollIntoViewIfNeeded)
+    if (scrollIntoViewIfNeeded) {
+        // FIXME: Wait in an implementation-specific way up to the session implicit wait timeout for the element to become in view.
         coreElement->scrollIntoViewIfNeeded(false);
-
-    WebCore::IntRect rect = coreElement->clientRect();
-
-    WebCore::Frame* coreFrame = frame->coreFrame();
-    if (!coreFrame) {
-        WebProcess::singleton().parentProcessConnection()->send(Messages::WebAutomationSession::DidComputeElementLayout(callbackID, WebCore::IntRect(), frameNotFoundErrorType), 0);
-        return;
     }
 
-    WebCore::FrameView *coreFrameView = coreFrame->view();
-    if (!coreFrameView) {
-        WebProcess::singleton().parentProcessConnection()->send(Messages::WebAutomationSession::DidComputeElementLayout(callbackID, WebCore::IntRect(), frameNotFoundErrorType), 0);
-        return;
-    }
+    WebCore::IntRect rect = coreElement->clientRect();
 
+    auto* coreFrameView = frame->coreFrame()->view();
     if (useViewportCoordinates)
         rect.moveBy(WebCore::IntPoint(0, -coreFrameView->topContentInset()));
     else
         rect = coreFrameView->rootViewToContents(rect);
 
-    WebProcess::singleton().parentProcessConnection()->send(Messages::WebAutomationSession::DidComputeElementLayout(callbackID, rect, String()), 0);
+    bool isObscured = false;
+    std::optional<WebCore::IntPoint> inViewCenter;
+    if (auto clientCenterPoint = elementInViewClientCenterPoint(*coreElement, isObscured)) {
+        inViewCenter = WebCore::IntPoint(coreFrameView->clientToDocumentPoint(clientCenterPoint.value()));
+        if (useViewportCoordinates)
+            inViewCenter = coreFrameView->contentsToRootView(inViewCenter.value());
+    }
+
+    WebProcess::singleton().parentProcessConnection()->send(Messages::WebAutomationSession::DidComputeElementLayout(callbackID, rect, inViewCenter, isObscured, String()), 0);
 }
 
 void WebAutomationSessionProxy::takeScreenshot(uint64_t pageID, uint64_t callbackID)