[Cocoa] Web Automation: add SPI to tell whether the automation session is currently...
[WebKit-https.git] / Source / WebKit / UIProcess / Automation / WebAutomationSession.h
1 /*
2  * Copyright (C) 2016, 2017 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
14  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
15  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
17  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
18  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
19  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
20  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
21  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
22  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
23  * THE POSSIBILITY OF SUCH DAMAGE.
24  */
25
26 #pragma once
27
28 #include "APIObject.h"
29 #include "AutomationBackendDispatchers.h"
30 #include "AutomationFrontendDispatchers.h"
31 #include "Connection.h"
32 #include "ShareableBitmap.h"
33 #include "WebEvent.h"
34 #include <wtf/Forward.h>
35 #include <wtf/RunLoop.h>
36
37 #if ENABLE(REMOTE_INSPECTOR)
38 #include <JavaScriptCore/RemoteAutomationTarget.h>
39 #endif
40
41 namespace API {
42 class AutomationSessionClient;
43 }
44
45 namespace Inspector {
46 class BackendDispatcher;
47 class FrontendRouter;
48 }
49
50 namespace WebCore {
51 class IntPoint;
52 class IntRect;
53
54 struct Cookie;
55 }
56
57 #if PLATFORM(COCOA)
58 OBJC_CLASS NSArray;
59 typedef unsigned short unichar;
60 #endif
61
62 #if USE(APPKIT)
63 OBJC_CLASS NSEvent;
64 #endif
65
66 namespace API {
67 class OpenPanelParameters;
68 }
69
70 namespace WebKit {
71
72 class WebAutomationSessionClient;
73 class WebFrameProxy;
74 class WebOpenPanelResultListenerProxy;
75 class WebPageProxy;
76 class WebProcessPool;
77
78 class WebAutomationSession final : public API::ObjectImpl<API::Object::Type::AutomationSession>, public IPC::MessageReceiver
79 #if ENABLE(REMOTE_INSPECTOR)
80     , public Inspector::RemoteAutomationTarget
81 #endif
82     , public Inspector::AutomationBackendDispatcherHandler
83 {
84 public:
85     WebAutomationSession();
86     ~WebAutomationSession();
87
88     void setClient(std::unique_ptr<API::AutomationSessionClient>&&);
89
90     void setSessionIdentifier(const String& sessionIdentifier) { m_sessionIdentifier = sessionIdentifier; }
91     String sessionIdentifier() const { return m_sessionIdentifier; }
92
93     WebProcessPool* processPool() const { return m_processPool; }
94     void setProcessPool(WebProcessPool*);
95
96     void navigationOccurredForFrame(const WebFrameProxy&);
97     void documentLoadedForFrame(const WebFrameProxy&);
98     void inspectorFrontendLoaded(const WebPageProxy&);
99     void keyboardEventsFlushedForPage(const WebPageProxy&);
100     void willClosePage(const WebPageProxy&);
101     void handleRunOpenPanel(const WebPageProxy&, const WebFrameProxy&, const API::OpenPanelParameters&, WebOpenPanelResultListenerProxy&);
102     void willShowJavaScriptDialog(WebPageProxy&);
103
104     bool shouldAllowGetUserMediaForPage(const WebPageProxy&) const;
105
106 #if ENABLE(REMOTE_INSPECTOR)
107     // Inspector::RemoteAutomationTarget API
108     String name() const override { return m_sessionIdentifier; }
109     void dispatchMessageFromRemote(const String& message) override;
110     void connect(Inspector::FrontendChannel*, bool isAutomaticConnection = false, bool immediatelyPause = false) override;
111     void disconnect(Inspector::FrontendChannel*) override;
112 #endif
113     void terminate();
114
115     // Inspector::AutomationBackendDispatcherHandler API
116     // NOTE: the set of declarations included in this interface depend on the "platform" property in Automation.json
117     // and the --platform argument passed to the protocol bindings generator.
118
119     // Platform: Generic
120     void getBrowsingContexts(Inspector::ErrorString&, Ref<GetBrowsingContextsCallback>&&) final;
121     void getBrowsingContext(Inspector::ErrorString&, const String&, Ref<GetBrowsingContextCallback>&&) final;
122     void createBrowsingContext(Inspector::ErrorString&, String*) override;
123     void closeBrowsingContext(Inspector::ErrorString&, const String&) override;
124     void switchToBrowsingContext(Inspector::ErrorString&, const String& browsingContextHandle, const String* optionalFrameHandle) override;
125     void resizeWindowOfBrowsingContext(Inspector::ErrorString&, const String& handle, const Inspector::InspectorObject& size, Ref<ResizeWindowOfBrowsingContextCallback>&&) final;
126     void moveWindowOfBrowsingContext(Inspector::ErrorString&, const String& handle, const Inspector::InspectorObject& position, Ref<MoveWindowOfBrowsingContextCallback>&&) final;
127     void navigateBrowsingContext(Inspector::ErrorString&, const String& handle, const String& url, const String* optionalPageLoadStrategyString, const int* optionalPageLoadTimeout, Ref<NavigateBrowsingContextCallback>&&) override;
128     void goBackInBrowsingContext(Inspector::ErrorString&, const String&, const String* optionalPageLoadStrategyString, const int* optionalPageLoadTimeout, Ref<GoBackInBrowsingContextCallback>&&) override;
129     void goForwardInBrowsingContext(Inspector::ErrorString&, const String&, const String* optionalPageLoadStrategyString, const int* optionalPageLoadTimeout, Ref<GoForwardInBrowsingContextCallback>&&) override;
130     void reloadBrowsingContext(Inspector::ErrorString&, const String&, const String* optionalPageLoadStrategyString, const int* optionalPageLoadTimeout, Ref<ReloadBrowsingContextCallback>&&) override;
131     void waitForNavigationToComplete(Inspector::ErrorString&, const String& browsingContextHandle, const String* optionalFrameHandle, const String* optionalPageLoadStrategyString, const int* optionalPageLoadTimeout, Ref<WaitForNavigationToCompleteCallback>&&) override;
132     void evaluateJavaScriptFunction(Inspector::ErrorString&, const String& browsingContextHandle, const String* optionalFrameHandle, const String& function, const Inspector::InspectorArray& arguments, const bool* optionalExpectsImplicitCallbackArgument, const int* optionalCallbackTimeout, Ref<Inspector::AutomationBackendDispatcherHandler::EvaluateJavaScriptFunctionCallback>&&) override;
133     void performMouseInteraction(Inspector::ErrorString&, const String& handle, const Inspector::InspectorObject& requestedPosition, const String& mouseButton, const String& mouseInteraction, const Inspector::InspectorArray& keyModifiers, Ref<PerformMouseInteractionCallback>&&) final;
134     void performKeyboardInteractions(Inspector::ErrorString&, const String& handle, const Inspector::InspectorArray& interactions, Ref<PerformKeyboardInteractionsCallback>&&) override;
135     void takeScreenshot(Inspector::ErrorString&, const String& handle, const String* optionalFrameHandle, const String* optionalNodeHandle, const bool* optionalScrollIntoViewIfNeeded, Ref<TakeScreenshotCallback>&&) override;
136     void resolveChildFrameHandle(Inspector::ErrorString&, const String& browsingContextHandle, const String* optionalFrameHandle, const int* optionalOrdinal, const String* optionalName, const String* optionalNodeHandle, Ref<ResolveChildFrameHandleCallback>&&) override;
137     void resolveParentFrameHandle(Inspector::ErrorString&, const String& browsingContextHandle, const String& frameHandle, Ref<ResolveParentFrameHandleCallback>&&) override;
138     void computeElementLayout(Inspector::ErrorString&, const String& browsingContextHandle, const String& frameHandle, const String& nodeHandle, const bool* optionalScrollIntoViewIfNeeded, const bool* useViewportCoordinates, Ref<Inspector::AutomationBackendDispatcherHandler::ComputeElementLayoutCallback>&&) override;
139     void selectOptionElement(Inspector::ErrorString&, const String& browsingContextHandle, const String& frameHandle, const String& nodeHandle, Ref<Inspector::AutomationBackendDispatcherHandler::SelectOptionElementCallback>&&) override;
140     void isShowingJavaScriptDialog(Inspector::ErrorString&, const String& browsingContextHandle, bool* result) override;
141     void dismissCurrentJavaScriptDialog(Inspector::ErrorString&, const String& browsingContextHandle) override;
142     void acceptCurrentJavaScriptDialog(Inspector::ErrorString&, const String& browsingContextHandle) override;
143     void messageOfCurrentJavaScriptDialog(Inspector::ErrorString&, const String& browsingContextHandle, String* text) override;
144     void setUserInputForCurrentJavaScriptPrompt(Inspector::ErrorString&, const String& browsingContextHandle, const String& text) override;
145     void setFilesToSelectForFileUpload(Inspector::ErrorString&, const String& browsingContextHandle, const Inspector::InspectorArray& filenames) override;
146     void getAllCookies(Inspector::ErrorString&, const String& browsingContextHandle, Ref<GetAllCookiesCallback>&&) override;
147     void deleteSingleCookie(Inspector::ErrorString&, const String& browsingContextHandle, const String& cookieName, Ref<DeleteSingleCookieCallback>&&) override;
148     void addSingleCookie(Inspector::ErrorString&, const String& browsingContextHandle, const Inspector::InspectorObject& cookie, Ref<AddSingleCookieCallback>&&) override;
149     void deleteAllCookies(Inspector::ErrorString&, const String& browsingContextHandle) override;
150     void getSessionPermissions(Inspector::ErrorString&, RefPtr<Inspector::Protocol::Array<Inspector::Protocol::Automation::SessionPermissionData>>& out_permissions) override;
151     void setSessionPermissions(Inspector::ErrorString&, const Inspector::InspectorArray& in_permissions) override;
152
153     // Platform: macOS
154 #if PLATFORM(MAC)
155     void inspectBrowsingContext(Inspector::ErrorString&, const String&, const bool* optionalEnableAutoCapturing, Ref<InspectBrowsingContextCallback>&&) override;
156 #endif
157
158     // Event Simulation Support.
159     bool isSimulatingUserInteraction() const { return m_simulatingUserInteraction; }
160 #if PLATFORM(MAC)
161     bool wasEventSynthesizedForAutomation(NSEvent *);
162     void markEventAsSynthesizedForAutomation(NSEvent *);
163 #endif
164
165 private:
166     WebPageProxy* webPageProxyForHandle(const String&);
167     String handleForWebPageProxy(const WebPageProxy&);
168     Ref<Inspector::Protocol::Automation::BrowsingContext> buildBrowsingContextForPage(WebPageProxy&, WebCore::FloatRect windowFrame);
169     void getNextContext(Ref<WebAutomationSession>&&, Vector<Ref<WebPageProxy>>&&, Ref<Inspector::Protocol::Array<Inspector::Protocol::Automation::BrowsingContext>>, Ref<WebAutomationSession::GetBrowsingContextsCallback>&&);
170
171     std::optional<uint64_t> webFrameIDForHandle(const String&);
172     String handleForWebFrameID(uint64_t frameID);
173     String handleForWebFrameProxy(const WebFrameProxy&);
174
175     void waitForNavigationToCompleteOnPage(WebPageProxy&, Inspector::Protocol::Automation::PageLoadStrategy, Seconds, Ref<Inspector::BackendDispatcher::CallbackBase>&&);
176     void waitForNavigationToCompleteOnFrame(WebFrameProxy&, Inspector::Protocol::Automation::PageLoadStrategy, Seconds, Ref<Inspector::BackendDispatcher::CallbackBase>&&);
177     void respondToPendingPageNavigationCallbacksWithTimeout(HashMap<uint64_t, RefPtr<Inspector::BackendDispatcher::CallbackBase>>&);
178     void respondToPendingFrameNavigationCallbacksWithTimeout(HashMap<uint64_t, RefPtr<Inspector::BackendDispatcher::CallbackBase>>&);
179     void loadTimerFired();
180
181     // Implemented in generated WebAutomationSessionMessageReceiver.cpp.
182     void didReceiveMessage(IPC::Connection&, IPC::Decoder&) override;
183
184     // Called by WebAutomationSession messages.
185     void didEvaluateJavaScriptFunction(uint64_t callbackID, const String& result, const String& errorType);
186     void didResolveChildFrame(uint64_t callbackID, uint64_t frameID, const String& errorType);
187     void didResolveParentFrame(uint64_t callbackID, uint64_t frameID, const String& errorType);
188     void didComputeElementLayout(uint64_t callbackID, WebCore::IntRect, std::optional<WebCore::IntPoint>, bool isObscured, const String& errorType);
189     void didSelectOptionElement(uint64_t callbackID, const String& errorType);
190     void didTakeScreenshot(uint64_t callbackID, const ShareableBitmap::Handle&, const String& errorType);
191     void didGetCookiesForFrame(uint64_t callbackID, Vector<WebCore::Cookie>, const String& errorType);
192     void didDeleteCookie(uint64_t callbackID, const String& errorType);
193
194     // Platform-dependent implementations.
195     void platformSimulateMouseInteraction(WebPageProxy&, const WebCore::IntPoint& viewPosition, Inspector::Protocol::Automation::MouseInteraction, Inspector::Protocol::Automation::MouseButton, WebEvent::Modifiers);
196     // Simulates a single virtual key being pressed, such as Control, F-keys, Numpad keys, etc. as allowed by the protocol.
197     void platformSimulateKeyStroke(WebPageProxy&, Inspector::Protocol::Automation::KeyboardInteractionType, Inspector::Protocol::Automation::VirtualKey);
198     // Simulates key presses to produce the codepoints in a string. One or more code points are delivered atomically at grapheme cluster boundaries.
199     void platformSimulateKeySequence(WebPageProxy&, const String&);
200     // Get base64 encoded PNG data from a bitmap.
201     std::optional<String> platformGetBase64EncodedPNGData(const ShareableBitmap::Handle&);
202
203 #if PLATFORM(COCOA)
204     // The type parameter of the NSArray argument is platform-dependent.
205     void sendSynthesizedEventsToPage(WebPageProxy&, NSArray *eventsToSend);
206
207     std::optional<unichar> charCodeForVirtualKey(Inspector::Protocol::Automation::VirtualKey) const;
208     std::optional<unichar> charCodeIgnoringModifiersForVirtualKey(Inspector::Protocol::Automation::VirtualKey) const;
209 #endif
210
211     WebProcessPool* m_processPool { nullptr };
212
213     std::unique_ptr<API::AutomationSessionClient> m_client;
214     String m_sessionIdentifier { ASCIILiteral("Untitled Session") };
215     Ref<Inspector::FrontendRouter> m_frontendRouter;
216     Ref<Inspector::BackendDispatcher> m_backendDispatcher;
217     Ref<Inspector::AutomationBackendDispatcher> m_domainDispatcher;
218     std::unique_ptr<Inspector::AutomationFrontendDispatcher> m_domainNotifier;
219
220     HashMap<uint64_t, String> m_webPageHandleMap;
221     HashMap<String, uint64_t> m_handleWebPageMap;
222     String m_activeBrowsingContextHandle;
223
224     HashMap<uint64_t, String> m_webFrameHandleMap;
225     HashMap<String, uint64_t> m_handleWebFrameMap;
226
227     HashMap<uint64_t, RefPtr<Inspector::BackendDispatcher::CallbackBase>> m_pendingNormalNavigationInBrowsingContextCallbacksPerPage;
228     HashMap<uint64_t, RefPtr<Inspector::BackendDispatcher::CallbackBase>> m_pendingEagerNavigationInBrowsingContextCallbacksPerPage;
229     HashMap<uint64_t, RefPtr<Inspector::BackendDispatcher::CallbackBase>> m_pendingNormalNavigationInBrowsingContextCallbacksPerFrame;
230     HashMap<uint64_t, RefPtr<Inspector::BackendDispatcher::CallbackBase>> m_pendingEagerNavigationInBrowsingContextCallbacksPerFrame;
231     HashMap<uint64_t, RefPtr<Inspector::BackendDispatcher::CallbackBase>> m_pendingInspectorCallbacksPerPage;
232     HashMap<uint64_t, RefPtr<Inspector::BackendDispatcher::CallbackBase>> m_pendingKeyboardEventsFlushedCallbacksPerPage;
233
234     uint64_t m_nextEvaluateJavaScriptCallbackID { 1 };
235     HashMap<uint64_t, RefPtr<Inspector::AutomationBackendDispatcherHandler::EvaluateJavaScriptFunctionCallback>> m_evaluateJavaScriptFunctionCallbacks;
236
237     uint64_t m_nextResolveFrameCallbackID { 1 };
238     HashMap<uint64_t, RefPtr<Inspector::AutomationBackendDispatcherHandler::ResolveChildFrameHandleCallback>> m_resolveChildFrameHandleCallbacks;
239
240     uint64_t m_nextResolveParentFrameCallbackID { 1 };
241     HashMap<uint64_t, RefPtr<Inspector::AutomationBackendDispatcherHandler::ResolveParentFrameHandleCallback>> m_resolveParentFrameHandleCallbacks;
242
243     uint64_t m_nextComputeElementLayoutCallbackID { 1 };
244     HashMap<uint64_t, RefPtr<Inspector::AutomationBackendDispatcherHandler::ComputeElementLayoutCallback>> m_computeElementLayoutCallbacks;
245
246     uint64_t m_nextScreenshotCallbackID { 1 };
247     HashMap<uint64_t, RefPtr<Inspector::AutomationBackendDispatcherHandler::TakeScreenshotCallback>> m_screenshotCallbacks;
248
249     uint64_t m_nextGetCookiesCallbackID { 1 };
250     HashMap<uint64_t, RefPtr<Inspector::AutomationBackendDispatcherHandler::GetAllCookiesCallback>> m_getCookieCallbacks;
251
252     uint64_t m_nextDeleteCookieCallbackID { 1 };
253     HashMap<uint64_t, RefPtr<Inspector::AutomationBackendDispatcherHandler::DeleteSingleCookieCallback>> m_deleteCookieCallbacks;
254
255     uint64_t m_nextSelectOptionElementCallbackID { 1 };
256     HashMap<uint64_t, RefPtr<Inspector::AutomationBackendDispatcherHandler::SelectOptionElementCallback>> m_selectOptionElementCallbacks;
257
258     RunLoop::Timer<WebAutomationSession> m_loadTimer;
259     Vector<String> m_filesToSelectForFileUpload;
260
261     bool m_permissionForGetUserMedia { true };
262
263     // True if a synthesized key event is still being processed.
264     bool m_simulatingUserInteraction { false };
265
266     // Keep track of currently active modifiers across multiple keystrokes.
267     // Most platforms do not track current modifiers from synthesized events.
268     unsigned m_currentModifiers { 0 };
269
270 #if ENABLE(REMOTE_INSPECTOR)
271     Inspector::FrontendChannel* m_remoteChannel { nullptr };
272 #endif
273
274 };
275
276 } // namespace WebKit