[Cocoa] Web Automation: add SPI to tell whether the automation session is currently...
authorbburg@apple.com <bburg@apple.com@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Tue, 24 Oct 2017 05:13:56 +0000 (05:13 +0000)
committerbburg@apple.com <bburg@apple.com@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Tue, 24 Oct 2017 05:13:56 +0000 (05:13 +0000)
https://bugs.webkit.org/show_bug.cgi?id=178616

Reviewed by Joseph Pecoraro.

This is needed to disambiguate whether an action (such as selectAll:) came from
a user clicking on "Edit > Select All" in a menu or whether it was produced by
simulating the keystrokes to produce the chord for "Command + a". Some clients,
such as Safari, would allow the latter but not the former during automation.

* UIProcess/API/Cocoa/_WKAutomationSession.h:
* UIProcess/API/Cocoa/_WKAutomationSession.mm:
(-[_WKAutomationSession isSimulatingUserInteraction]):
Add new SPI property that's backed by the same WebAutomationSession method.

* UIProcess/Automation/WebAutomationSession.h:
* UIProcess/Automation/WebAutomationSession.cpp:
(WebKit::WebAutomationSession::keyboardEventsFlushedForPage):
(WebKit::WebAutomationSession::performMouseInteraction):
(WebKit::WebAutomationSession::performKeyboardInteractions):
Set m_simulatingUserInteraction prior to sending the synthesized events. It will
be cleared when keyboardEventsFlushedForPage() is called by WebPageProxy.

* UIProcess/WebPageProxy.cpp:
(WebKit::WebPageProxy::didReceiveEvent):
Notify the automation session that the key event queue was flushed *after* giving
delegates a chance to do something with the key event. This is necessary so that
any actions that are created from the NSEvent by the delegates are handled prior
to the automation session finishing its keyboard interaction command.

git-svn-id: https://svn.webkit.org/repository/webkit/trunk@223880 268f45cc-cd09-0410-ab3c-d52691b4dbfc

Source/WebKit/ChangeLog
Source/WebKit/UIProcess/API/Cocoa/_WKAutomationSession.h
Source/WebKit/UIProcess/API/Cocoa/_WKAutomationSession.mm
Source/WebKit/UIProcess/Automation/WebAutomationSession.cpp
Source/WebKit/UIProcess/Automation/WebAutomationSession.h
Source/WebKit/UIProcess/WebPageProxy.cpp

index 654a994..71f444d 100644 (file)
@@ -1,5 +1,37 @@
 2017-10-23  Brian Burg  <bburg@apple.com>
 
 2017-10-23  Brian Burg  <bburg@apple.com>
 
+        [Cocoa] Web Automation: add SPI to tell whether the automation session is currently simulating user interactions
+        https://bugs.webkit.org/show_bug.cgi?id=178616
+
+        Reviewed by Joseph Pecoraro.
+
+        This is needed to disambiguate whether an action (such as selectAll:) came from
+        a user clicking on "Edit > Select All" in a menu or whether it was produced by
+        simulating the keystrokes to produce the chord for "Command + a". Some clients,
+        such as Safari, would allow the latter but not the former during automation.
+
+        * UIProcess/API/Cocoa/_WKAutomationSession.h:
+        * UIProcess/API/Cocoa/_WKAutomationSession.mm:
+        (-[_WKAutomationSession isSimulatingUserInteraction]):
+        Add new SPI property that's backed by the same WebAutomationSession method.
+
+        * UIProcess/Automation/WebAutomationSession.h:
+        * UIProcess/Automation/WebAutomationSession.cpp:
+        (WebKit::WebAutomationSession::keyboardEventsFlushedForPage):
+        (WebKit::WebAutomationSession::performMouseInteraction):
+        (WebKit::WebAutomationSession::performKeyboardInteractions):
+        Set m_simulatingUserInteraction prior to sending the synthesized events. It will
+        be cleared when keyboardEventsFlushedForPage() is called by WebPageProxy.
+
+        * UIProcess/WebPageProxy.cpp:
+        (WebKit::WebPageProxy::didReceiveEvent):
+        Notify the automation session that the key event queue was flushed *after* giving
+        delegates a chance to do something with the key event. This is necessary so that
+        any actions that are created from the NSEvent by the delegates are handled prior
+        to the automation session finishing its keyboard interaction command.
+
+2017-10-23  Brian Burg  <bburg@apple.com>
+
         [Mac] Web Automation: key modifiers for synthesized NSEvents are incorrect
         https://bugs.webkit.org/show_bug.cgi?id=178615
 
         [Mac] Web Automation: key modifiers for synthesized NSEvents are incorrect
         https://bugs.webkit.org/show_bug.cgi?id=178615
 
index 99f7dad..be290e6 100644 (file)
@@ -43,6 +43,8 @@ WK_CLASS_AVAILABLE(macosx(10.12), ios(10.0))
 @property (nonatomic, weak) id <_WKAutomationSessionDelegate> delegate;
 @property (nonatomic, readonly, getter=isPaired) BOOL paired;
 
 @property (nonatomic, weak) id <_WKAutomationSessionDelegate> delegate;
 @property (nonatomic, readonly, getter=isPaired) BOOL paired;
 
+@property (nonatomic, readonly, getter=isSimulatingUserInteraction) BOOL simulatingUserInteraction WK_API_AVAILABLE(macosx(WK_MAC_TBA), ios(WK_IOS_TBA));
+
 - (instancetype)initWithConfiguration:(_WKAutomationSessionConfiguration *)configuration NS_DESIGNATED_INITIALIZER;
 
 #if !TARGET_OS_IPHONE
 - (instancetype)initWithConfiguration:(_WKAutomationSessionConfiguration *)configuration NS_DESIGNATED_INITIALIZER;
 
 #if !TARGET_OS_IPHONE
index b400fae..45c0d02 100644 (file)
     return _session->isPaired();
 }
 
     return _session->isPaired();
 }
 
+- (BOOL)isSimulatingUserInteraction
+{
+    return _session->isSimulatingUserInteraction();
+}
+
 #if PLATFORM(MAC)
 - (BOOL)wasEventSynthesizedForAutomation:(NSEvent *)event
 {
 #if PLATFORM(MAC)
 - (BOOL)wasEventSynthesizedForAutomation:(NSEvent *)event
 {
index 8494d88..bf19b0a 100644 (file)
@@ -663,8 +663,12 @@ void WebAutomationSession::inspectorFrontendLoaded(const WebPageProxy& page)
 
 void WebAutomationSession::keyboardEventsFlushedForPage(const WebPageProxy& page)
 {
 
 void WebAutomationSession::keyboardEventsFlushedForPage(const WebPageProxy& page)
 {
-    if (auto callback = m_pendingKeyboardEventsFlushedCallbacksPerPage.take(page.pageID()))
+    if (auto callback = m_pendingKeyboardEventsFlushedCallbacksPerPage.take(page.pageID())) {
         callback->sendSuccess(InspectorObject::create());
         callback->sendSuccess(InspectorObject::create());
+
+        if (m_pendingKeyboardEventsFlushedCallbacksPerPage.isEmpty())
+            m_simulatingUserInteraction = false;
+    }
 }
 
 void WebAutomationSession::willClosePage(const WebPageProxy& page)
 }
 
 void WebAutomationSession::willClosePage(const WebPageProxy& page)
@@ -1284,7 +1288,7 @@ static WebEvent::Modifiers protocolModifierToWebEventModifier(Inspector::Protoco
 
     RELEASE_ASSERT_NOT_REACHED();
 }
 
     RELEASE_ASSERT_NOT_REACHED();
 }
-#endif // USE(APPKIT)
+#endif // USE(APPKIT) || PLATFORM(GTK)
 
 void WebAutomationSession::performMouseInteraction(Inspector::ErrorString& errorString, const String& handle, const Inspector::InspectorObject& requestedPositionObject, const String& mouseButtonString, const String& mouseInteractionString, const Inspector::InspectorArray& keyModifierStrings, Ref<PerformMouseInteractionCallback>&& callback)
 {
 
 void WebAutomationSession::performMouseInteraction(Inspector::ErrorString& errorString, const String& handle, const Inspector::InspectorObject& requestedPositionObject, const String& mouseButtonString, const String& mouseInteractionString, const Inspector::InspectorArray& keyModifierStrings, Ref<PerformMouseInteractionCallback>&& callback)
 {
@@ -1338,7 +1342,7 @@ void WebAutomationSession::performMouseInteraction(Inspector::ErrorString& error
             .setY(y - page->topContentInset())
             .release());
     });
             .setY(y - page->topContentInset())
             .release());
     });
-#endif // USE(APPKIT)
+#endif // USE(APPKIT) || PLATFORM(GTK)
 }
 
 void WebAutomationSession::performKeyboardInteractions(ErrorString& errorString, const String& handle, const Inspector::InspectorArray& interactions, Ref<PerformKeyboardInteractionsCallback>&& callback)
 }
 
 void WebAutomationSession::performKeyboardInteractions(ErrorString& errorString, const String& handle, const Inspector::InspectorArray& interactions, Ref<PerformKeyboardInteractionsCallback>&& callback)
@@ -1411,9 +1415,12 @@ void WebAutomationSession::performKeyboardInteractions(ErrorString& errorString,
         callbackInMap->sendFailure(STRING_FOR_PREDEFINED_ERROR_NAME(Timeout));
     callbackInMap = WTFMove(callback);
 
         callbackInMap->sendFailure(STRING_FOR_PREDEFINED_ERROR_NAME(Timeout));
     callbackInMap = WTFMove(callback);
 
+    // This is cleared when all keyboard events are flushed.
+    m_simulatingUserInteraction = true;
+
     for (auto& action : actionsToPerform)
         action();
     for (auto& action : actionsToPerform)
         action();
-#endif // PLATFORM(COCOA)
+#endif // PLATFORM(COCOA) || PLATFORM(GTK)
 }
 
 void WebAutomationSession::takeScreenshot(ErrorString& errorString, const String& handle, const String* optionalFrameHandle, const String* optionalNodeHandle, const bool* optionalScrollIntoViewIfNeeded, Ref<TakeScreenshotCallback>&& callback)
 }
 
 void WebAutomationSession::takeScreenshot(ErrorString& errorString, const String& handle, const String* optionalFrameHandle, const String* optionalNodeHandle, const bool* optionalScrollIntoViewIfNeeded, Ref<TakeScreenshotCallback>&& callback)
index 604ca0d..3b6012c 100644 (file)
@@ -156,6 +156,7 @@ public:
 #endif
 
     // Event Simulation Support.
 #endif
 
     // Event Simulation Support.
+    bool isSimulatingUserInteraction() const { return m_simulatingUserInteraction; }
 #if PLATFORM(MAC)
     bool wasEventSynthesizedForAutomation(NSEvent *);
     void markEventAsSynthesizedForAutomation(NSEvent *);
 #if PLATFORM(MAC)
     bool wasEventSynthesizedForAutomation(NSEvent *);
     void markEventAsSynthesizedForAutomation(NSEvent *);
@@ -259,6 +260,8 @@ private:
 
     bool m_permissionForGetUserMedia { true };
 
 
     bool m_permissionForGetUserMedia { true };
 
+    // True if a synthesized key event is still being processed.
+    bool m_simulatingUserInteraction { false };
 
     // Keep track of currently active modifiers across multiple keystrokes.
     // Most platforms do not track current modifiers from synthesized events.
 
     // Keep track of currently active modifiers across multiple keystrokes.
     // Most platforms do not track current modifiers from synthesized events.
index cc1fb17..f8ac186 100644 (file)
@@ -5085,9 +5085,6 @@ void WebPageProxy::didReceiveEvent(uint32_t opaqueType, bool handled)
         if (!m_keyEventQueue.isEmpty()) {
             LOG(KeyHandling, " UI process: sent keyEvent from didReceiveEvent");
             m_process->send(Messages::WebPage::KeyEvent(m_keyEventQueue.first()), m_pageID);
         if (!m_keyEventQueue.isEmpty()) {
             LOG(KeyHandling, " UI process: sent keyEvent from didReceiveEvent");
             m_process->send(Messages::WebPage::KeyEvent(m_keyEventQueue.first()), m_pageID);
-        } else {
-            if (auto* automationSession = process().processPool().automationSession())
-                automationSession->keyboardEventsFlushedForPage(*this);
         }
 
         // The call to doneWithKeyEvent may close this WebPage.
         }
 
         // The call to doneWithKeyEvent may close this WebPage.
@@ -5095,10 +5092,14 @@ void WebPageProxy::didReceiveEvent(uint32_t opaqueType, bool handled)
         Ref<WebPageProxy> protect(*this);
 
         m_pageClient.doneWithKeyEvent(event, handled);
         Ref<WebPageProxy> protect(*this);
 
         m_pageClient.doneWithKeyEvent(event, handled);
-        if (handled)
-            break;
+        if (!handled)
+            m_uiClient->didNotHandleKeyEvent(this, event);
 
 
-        m_uiClient->didNotHandleKeyEvent(this, event);
+        // Notify the session after -[NSApp sendEvent:] has a crack at turning the event into an action.
+        if (m_keyEventQueue.isEmpty()) {
+            if (auto* automationSession = process().processPool().automationSession())
+                automationSession->keyboardEventsFlushedForPage(*this);
+        }
         break;
     }
 #if ENABLE(MAC_GESTURE_EVENTS)
         break;
     }
 #if ENABLE(MAC_GESTURE_EVENTS)