WebDriver: implement maximize, minimize and fullscreen window commands
[WebKit-https.git] / Source / WebKit / UIProcess / Automation / WebAutomationSession.cpp
1
2 /*
3  * Copyright (C) 2016, 2017 Apple Inc. All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions
7  * are met:
8  * 1. Redistributions of source code must retain the above copyright
9  *    notice, this list of conditions and the following disclaimer.
10  * 2. Redistributions in binary form must reproduce the above copyright
11  *    notice, this list of conditions and the following disclaimer in the
12  *    documentation and/or other materials provided with the distribution.
13  *
14  * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
15  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
16  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
17  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
18  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
19  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
20  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
21  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
22  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
23  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
24  * THE POSSIBILITY OF SUCH DAMAGE.
25  */
26
27 #include "config.h"
28 #include "WebAutomationSession.h"
29
30 #include "APIArray.h"
31 #include "APIAutomationSessionClient.h"
32 #include "APIOpenPanelParameters.h"
33 #include "AutomationProtocolObjects.h"
34 #include "CoordinateSystem.h"
35 #include "WebAutomationSessionMacros.h"
36 #include "WebAutomationSessionMessages.h"
37 #include "WebAutomationSessionProxyMessages.h"
38 #include "WebCookieManagerProxy.h"
39 #include "WebFullScreenManagerProxy.h"
40 #include "WebInspectorProxy.h"
41 #include "WebOpenPanelResultListenerProxy.h"
42 #include "WebProcessPool.h"
43 #include <JavaScriptCore/InspectorBackendDispatcher.h>
44 #include <JavaScriptCore/InspectorFrontendRouter.h>
45 #include <WebCore/MIMETypeRegistry.h>
46 #include <WebCore/URL.h>
47 #include <algorithm>
48 #include <wtf/HashMap.h>
49 #include <wtf/Optional.h>
50 #include <wtf/UUID.h>
51 #include <wtf/text/StringConcatenate.h>
52
53 using namespace Inspector;
54
55 namespace WebKit {
56
57 String AutomationCommandError::toProtocolString()
58 {
59     String protocolErrorName = Inspector::Protocol::AutomationHelpers::getEnumConstantValue(type);
60     if (!message.has_value())
61         return protocolErrorName;
62
63     return makeString(protocolErrorName, errorNameAndDetailsSeparator, message.value());
64 }
65     
66 // §8. Sessions
67 // https://www.w3.org/TR/webdriver/#dfn-session-page-load-timeout
68 static const Seconds defaultPageLoadTimeout = 300_s;
69 // https://www.w3.org/TR/webdriver/#dfn-page-loading-strategy
70 static const Inspector::Protocol::Automation::PageLoadStrategy defaultPageLoadStrategy = Inspector::Protocol::Automation::PageLoadStrategy::Normal;
71
72 WebAutomationSession::WebAutomationSession()
73     : m_client(std::make_unique<API::AutomationSessionClient>())
74     , m_frontendRouter(FrontendRouter::create())
75     , m_backendDispatcher(BackendDispatcher::create(m_frontendRouter.copyRef()))
76     , m_domainDispatcher(AutomationBackendDispatcher::create(m_backendDispatcher, this))
77     , m_domainNotifier(std::make_unique<AutomationFrontendDispatcher>(m_frontendRouter))
78     , m_loadTimer(RunLoop::main(), this, &WebAutomationSession::loadTimerFired)
79 {
80     // Set up canonical input sources to be used for 'performInteractionSequence' and 'cancelInteractionSequence'.
81     m_inputSources.add(SimulatedInputSource::create(SimulatedInputSourceType::Mouse));
82     m_inputSources.add(SimulatedInputSource::create(SimulatedInputSourceType::Keyboard));
83     m_inputSources.add(SimulatedInputSource::create(SimulatedInputSourceType::Null));
84 }
85
86 WebAutomationSession::~WebAutomationSession()
87 {
88     ASSERT(!m_client);
89     ASSERT(!m_processPool);
90 }
91
92 void WebAutomationSession::setClient(std::unique_ptr<API::AutomationSessionClient>&& client)
93 {
94     m_client = WTFMove(client);
95 }
96
97 void WebAutomationSession::setProcessPool(WebKit::WebProcessPool* processPool)
98 {
99     if (m_processPool)
100         m_processPool->removeMessageReceiver(Messages::WebAutomationSession::messageReceiverName());
101
102     m_processPool = processPool;
103
104     if (m_processPool)
105         m_processPool->addMessageReceiver(Messages::WebAutomationSession::messageReceiverName(), *this);
106 }
107
108 // NOTE: this class could be split at some point to support local and remote automation sessions.
109 // For now, it only works with a remote automation driver over a RemoteInspector connection.
110
111 #if ENABLE(REMOTE_INSPECTOR)
112
113 // Inspector::RemoteAutomationTarget API
114
115 void WebAutomationSession::dispatchMessageFromRemote(const String& message)
116 {
117     m_backendDispatcher->dispatch(message);
118 }
119
120 void WebAutomationSession::connect(Inspector::FrontendChannel* channel, bool isAutomaticConnection, bool immediatelyPause)
121 {
122     UNUSED_PARAM(isAutomaticConnection);
123     UNUSED_PARAM(immediatelyPause);
124
125     m_remoteChannel = channel;
126     m_frontendRouter->connectFrontend(channel);
127
128     setIsPaired(true);
129 }
130
131 void WebAutomationSession::disconnect(Inspector::FrontendChannel* channel)
132 {
133     ASSERT(channel == m_remoteChannel);
134     terminate();
135 }
136
137 #endif // ENABLE(REMOTE_INSPECTOR)
138
139 void WebAutomationSession::terminate()
140 {
141 #if ENABLE(REMOTE_INSPECTOR)
142     if (Inspector::FrontendChannel* channel = m_remoteChannel) {
143         m_remoteChannel = nullptr;
144         m_frontendRouter->disconnectFrontend(channel);
145     }
146
147     setIsPaired(false);
148 #endif
149
150     if (m_client)
151         m_client->didDisconnectFromRemote(*this);
152 }
153
154 WebPageProxy* WebAutomationSession::webPageProxyForHandle(const String& handle)
155 {
156     auto iter = m_handleWebPageMap.find(handle);
157     if (iter == m_handleWebPageMap.end())
158         return nullptr;
159     return WebProcessProxy::webPage(iter->value);
160 }
161
162 String WebAutomationSession::handleForWebPageProxy(const WebPageProxy& webPageProxy)
163 {
164     auto iter = m_webPageHandleMap.find(webPageProxy.pageID());
165     if (iter != m_webPageHandleMap.end())
166         return iter->value;
167
168     String handle = "page-" + createCanonicalUUIDString().convertToASCIIUppercase();
169
170     auto firstAddResult = m_webPageHandleMap.add(webPageProxy.pageID(), handle);
171     RELEASE_ASSERT(firstAddResult.isNewEntry);
172
173     auto secondAddResult = m_handleWebPageMap.add(handle, webPageProxy.pageID());
174     RELEASE_ASSERT(secondAddResult.isNewEntry);
175
176     return handle;
177 }
178
179 std::optional<uint64_t> WebAutomationSession::webFrameIDForHandle(const String& handle)
180 {
181     if (handle.isEmpty())
182         return 0;
183
184     auto iter = m_handleWebFrameMap.find(handle);
185     if (iter == m_handleWebFrameMap.end())
186         return std::nullopt;
187
188     return iter->value;
189 }
190
191 String WebAutomationSession::handleForWebFrameID(uint64_t frameID)
192 {
193     if (!frameID)
194         return emptyString();
195
196     for (auto& process : m_processPool->processes()) {
197         if (WebFrameProxy* frame = process->webFrame(frameID)) {
198             if (frame->isMainFrame())
199                 return emptyString();
200             break;
201         }
202     }
203
204     auto iter = m_webFrameHandleMap.find(frameID);
205     if (iter != m_webFrameHandleMap.end())
206         return iter->value;
207
208     String handle = "frame-" + createCanonicalUUIDString().convertToASCIIUppercase();
209
210     auto firstAddResult = m_webFrameHandleMap.add(frameID, handle);
211     RELEASE_ASSERT(firstAddResult.isNewEntry);
212
213     auto secondAddResult = m_handleWebFrameMap.add(handle, frameID);
214     RELEASE_ASSERT(secondAddResult.isNewEntry);
215
216     return handle;
217 }
218
219 String WebAutomationSession::handleForWebFrameProxy(const WebFrameProxy& webFrameProxy)
220 {
221     return handleForWebFrameID(webFrameProxy.frameID());
222 }
223
224 Ref<Inspector::Protocol::Automation::BrowsingContext> WebAutomationSession::buildBrowsingContextForPage(WebPageProxy& page, WebCore::FloatRect windowFrame)
225 {
226     auto originObject = Inspector::Protocol::Automation::Point::create()
227         .setX(windowFrame.x())
228         .setY(windowFrame.y())
229         .release();
230
231     auto sizeObject = Inspector::Protocol::Automation::Size::create()
232         .setWidth(windowFrame.width())
233         .setHeight(windowFrame.height())
234         .release();
235
236     bool isActive = page.isViewVisible() && page.isViewFocused() && page.isViewWindowActive();
237     String handle = handleForWebPageProxy(page);
238
239     return Inspector::Protocol::Automation::BrowsingContext::create()
240         .setHandle(handle)
241         .setActive(isActive)
242         .setUrl(page.pageLoadState().activeURL())
243         .setWindowOrigin(WTFMove(originObject))
244         .setWindowSize(WTFMove(sizeObject))
245         .release();
246 }
247
248 // Platform-independent Commands.
249
250 void WebAutomationSession::getNextContext(Ref<WebAutomationSession>&& protectedThis, Vector<Ref<WebPageProxy>>&& pages, Ref<JSON::ArrayOf<Inspector::Protocol::Automation::BrowsingContext>> contexts, Ref<WebAutomationSession::GetBrowsingContextsCallback>&& callback)
251 {
252     if (pages.isEmpty()) {
253         callback->sendSuccess(WTFMove(contexts));
254         return;
255     }
256     auto page = pages.takeLast();
257     auto& webPageProxy = page.get();
258     webPageProxy.getWindowFrameWithCallback([this, protectedThis = WTFMove(protectedThis), callback = WTFMove(callback), pages = WTFMove(pages), contexts = WTFMove(contexts), page = WTFMove(page)](WebCore::FloatRect windowFrame) mutable {
259         contexts->addItem(protectedThis->buildBrowsingContextForPage(page.get(), windowFrame));
260         getNextContext(WTFMove(protectedThis), WTFMove(pages), WTFMove(contexts), WTFMove(callback));
261     });
262 }
263     
264 void WebAutomationSession::getBrowsingContexts(Ref<GetBrowsingContextsCallback>&& callback)
265 {
266     Vector<Ref<WebPageProxy>> pages;
267     for (auto& process : m_processPool->processes()) {
268         for (auto* page : process->pages()) {
269             ASSERT(page);
270             if (!page->isControlledByAutomation())
271                 continue;
272             pages.append(*page);
273         }
274     }
275     
276     getNextContext(makeRef(*this), WTFMove(pages), JSON::ArrayOf<Inspector::Protocol::Automation::BrowsingContext>::create(), WTFMove(callback));
277 }
278
279 void WebAutomationSession::getBrowsingContext(const String& handle, Ref<GetBrowsingContextCallback>&& callback)
280 {
281     WebPageProxy* page = webPageProxyForHandle(handle);
282     if (!page)
283         ASYNC_FAIL_WITH_PREDEFINED_ERROR(WindowNotFound);
284
285     page->getWindowFrameWithCallback([protectedThis = makeRef(*this), page = makeRef(*page), callback = WTFMove(callback)](WebCore::FloatRect windowFrame) mutable {
286         callback->sendSuccess(protectedThis->buildBrowsingContextForPage(page.get(), windowFrame));
287     });
288 }
289
290 void WebAutomationSession::createBrowsingContext(const bool* preferNewTab, Ref<CreateBrowsingContextCallback>&& callback)
291 {
292     ASSERT(m_client);
293     if (!m_client)
294         ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(InternalError, "The remote session could not request a new browsing context.");
295
296     uint16_t options = 0;
297     if (preferNewTab && *preferNewTab)
298         options |= API::AutomationSessionBrowsingContextOptionsPreferNewTab;
299
300     m_client->requestNewPageWithOptions(*this, static_cast<API::AutomationSessionBrowsingContextOptions>(options), [protectedThis = makeRef(*this), callback = WTFMove(callback)](WebPageProxy* page) {
301         if (page)
302             callback->sendSuccess(protectedThis->handleForWebPageProxy(*page));
303         else
304             ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(InternalError, "The remote session failed to create a new browsing context.");
305     });
306 }
307
308 void WebAutomationSession::closeBrowsingContext(Inspector::ErrorString& errorString, const String& handle)
309 {
310     WebPageProxy* page = webPageProxyForHandle(handle);
311     if (!page)
312         SYNC_FAIL_WITH_PREDEFINED_ERROR(WindowNotFound);
313
314     page->closePage(false);
315 }
316
317 void WebAutomationSession::switchToBrowsingContext(const String& browsingContextHandle, const String* optionalFrameHandle, Ref<SwitchToBrowsingContextCallback>&& callback)
318 {
319     WebPageProxy* page = webPageProxyForHandle(browsingContextHandle);
320     if (!page)
321         ASYNC_FAIL_WITH_PREDEFINED_ERROR(WindowNotFound);
322
323     std::optional<uint64_t> frameID = webFrameIDForHandle(optionalFrameHandle ? *optionalFrameHandle : emptyString());
324     if (!frameID)
325         ASYNC_FAIL_WITH_PREDEFINED_ERROR(FrameNotFound);
326
327
328     m_client->requestSwitchToPage(*this, *page, [frameID, page = makeRef(*page), callback = WTFMove(callback)]() {
329         page->setFocus(true);
330         page->process().send(Messages::WebAutomationSessionProxy::FocusFrame(page->pageID(), frameID.value()), 0);
331
332         callback->sendSuccess();
333     });
334 }
335
336 void WebAutomationSession::setWindowFrameOfBrowsingContext(const String& handle, const JSON::Object* optionalOriginObject, const JSON::Object* optionalSizeObject, Ref<SetWindowFrameOfBrowsingContextCallback>&& callback)
337 {
338     std::optional<float> x;
339     std::optional<float> y;
340     if (optionalOriginObject) {
341         if (!(x = optionalOriginObject->getNumber<float>(WTF::ASCIILiteral("x"))))
342             ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(MissingParameter, "The 'x' parameter was not found or invalid.");
343
344         if (!(y = optionalOriginObject->getNumber<float>(WTF::ASCIILiteral("y"))))
345             ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(MissingParameter, "The 'y' parameter was not found or invalid.");
346
347         if (x.value() < 0)
348             ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(InvalidParameter, "The 'x' parameter had an invalid value.");
349
350         if (y.value() < 0)
351             ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(InvalidParameter, "The 'y' parameter had an invalid value.");
352     }
353
354     std::optional<float> width;
355     std::optional<float> height;
356     if (optionalSizeObject) {
357         if (!(width = optionalSizeObject->getNumber<float>(WTF::ASCIILiteral("width"))))
358             ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(MissingParameter, "The 'width' parameter was not found or invalid.");
359
360         if (!(height = optionalSizeObject->getNumber<float>(WTF::ASCIILiteral("height"))))
361             ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(MissingParameter, "The 'height' parameter was not found or invalid.");
362
363         if (width.value() < 0)
364             ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(InvalidParameter, "The 'width' parameter had an invalid value.");
365
366         if (height.value() < 0)
367             ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(InvalidParameter, "The 'height' parameter had an invalid value.");
368     }
369
370     WebPageProxy* page = webPageProxyForHandle(handle);
371     if (!page)
372         ASYNC_FAIL_WITH_PREDEFINED_ERROR(WindowNotFound);
373
374     exitFullscreenWindowForPage(*page, [this, protectedThis = makeRef(*this), callback = WTFMove(callback), page = makeRefPtr(page), width, height, x, y]() mutable {
375         auto& webPage = *page;
376         this->restoreWindowForPage(webPage, [callback = WTFMove(callback), page = WTFMove(page), width, height, x, y]() mutable {
377             auto& webPage = *page;
378             webPage.getWindowFrameWithCallback([callback = WTFMove(callback), page = WTFMove(page), width, height, x, y](WebCore::FloatRect originalFrame) mutable {
379                 WebCore::FloatRect newFrame = WebCore::FloatRect(WebCore::FloatPoint(x.value_or(originalFrame.location().x()), y.value_or(originalFrame.location().y())), WebCore::FloatSize(width.value_or(originalFrame.size().width()), height.value_or(originalFrame.size().height())));
380                 if (newFrame != originalFrame)
381                     page->setWindowFrame(newFrame);
382                 
383                 callback->sendSuccess();
384             });
385         });
386     });
387 }
388
389 static std::optional<Inspector::Protocol::Automation::PageLoadStrategy> pageLoadStrategyFromStringParameter(const String* optionalPageLoadStrategyString)
390 {
391     if (!optionalPageLoadStrategyString)
392         return defaultPageLoadStrategy;
393
394     auto parsedPageLoadStrategy = Inspector::Protocol::AutomationHelpers::parseEnumValueFromString<Inspector::Protocol::Automation::PageLoadStrategy>(*optionalPageLoadStrategyString);
395     if (!parsedPageLoadStrategy)
396         return std::nullopt;
397     return parsedPageLoadStrategy;
398 }
399
400 void WebAutomationSession::waitForNavigationToComplete(const String& browsingContextHandle, const String* optionalFrameHandle, const String* optionalPageLoadStrategyString, const int* optionalPageLoadTimeout, Ref<WaitForNavigationToCompleteCallback>&& callback)
401 {
402     WebPageProxy* page = webPageProxyForHandle(browsingContextHandle);
403     if (!page)
404         ASYNC_FAIL_WITH_PREDEFINED_ERROR(WindowNotFound);
405
406     auto pageLoadStrategy = pageLoadStrategyFromStringParameter(optionalPageLoadStrategyString);
407     if (!pageLoadStrategy)
408         ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(InvalidParameter, "The parameter 'pageLoadStrategy' is invalid.");
409     auto pageLoadTimeout = optionalPageLoadTimeout ? Seconds::fromMilliseconds(*optionalPageLoadTimeout) : defaultPageLoadTimeout;
410
411     // If page is loading and there's an active JavaScript dialog is probably because the
412     // dialog was started in an onload handler, so in case of normal page load strategy the
413     // load will not finish until the dialog is dismissed. Instead of waiting for the timeout,
414     // we return without waiting since we know it will timeout for sure. We want to check
415     // arguments first, though.
416     bool shouldTimeoutDueToUnexpectedAlert = pageLoadStrategy.value() == Inspector::Protocol::Automation::PageLoadStrategy::Normal
417         && page->pageLoadState().isLoading() && m_client->isShowingJavaScriptDialogOnPage(*this, *page);
418
419     if (optionalFrameHandle && !optionalFrameHandle->isEmpty()) {
420         std::optional<uint64_t> frameID = webFrameIDForHandle(*optionalFrameHandle);
421         if (!frameID)
422             ASYNC_FAIL_WITH_PREDEFINED_ERROR(FrameNotFound);
423         WebFrameProxy* frame = page->process().webFrame(frameID.value());
424         if (!frame)
425             ASYNC_FAIL_WITH_PREDEFINED_ERROR(FrameNotFound);
426         if (!shouldTimeoutDueToUnexpectedAlert)
427             waitForNavigationToCompleteOnFrame(*frame, pageLoadStrategy.value(), pageLoadTimeout, WTFMove(callback));
428     } else {
429         if (!shouldTimeoutDueToUnexpectedAlert)
430             waitForNavigationToCompleteOnPage(*page, pageLoadStrategy.value(), pageLoadTimeout, WTFMove(callback));
431     }
432
433     if (shouldTimeoutDueToUnexpectedAlert) {
434         // §9 Navigation.
435         // 7. If the previous step completed by the session page load timeout being reached and the browser does not
436         // have an active user prompt, return error with error code timeout.
437         // 8. Return success with data null.
438         // https://w3c.github.io/webdriver/webdriver-spec.html#dfn-wait-for-navigation-to-complete
439         callback->sendSuccess();
440     }
441 }
442
443 void WebAutomationSession::waitForNavigationToCompleteOnPage(WebPageProxy& page, Inspector::Protocol::Automation::PageLoadStrategy loadStrategy, Seconds timeout, Ref<Inspector::BackendDispatcher::CallbackBase>&& callback)
444 {
445     ASSERT(!m_loadTimer.isActive());
446     if (loadStrategy == Inspector::Protocol::Automation::PageLoadStrategy::None || !page.pageLoadState().isLoading()) {
447         callback->sendSuccess(JSON::Object::create());
448         return;
449     }
450
451     m_loadTimer.startOneShot(timeout);
452     switch (loadStrategy) {
453     case Inspector::Protocol::Automation::PageLoadStrategy::Normal:
454         m_pendingNormalNavigationInBrowsingContextCallbacksPerPage.set(page.pageID(), WTFMove(callback));
455         break;
456     case Inspector::Protocol::Automation::PageLoadStrategy::Eager:
457         m_pendingEagerNavigationInBrowsingContextCallbacksPerPage.set(page.pageID(), WTFMove(callback));
458         break;
459     case Inspector::Protocol::Automation::PageLoadStrategy::None:
460         ASSERT_NOT_REACHED();
461     }
462 }
463
464 void WebAutomationSession::waitForNavigationToCompleteOnFrame(WebFrameProxy& frame, Inspector::Protocol::Automation::PageLoadStrategy loadStrategy, Seconds timeout, Ref<Inspector::BackendDispatcher::CallbackBase>&& callback)
465 {
466     ASSERT(!m_loadTimer.isActive());
467     if (loadStrategy == Inspector::Protocol::Automation::PageLoadStrategy::None || frame.frameLoadState().state() == FrameLoadState::State::Finished) {
468         callback->sendSuccess(JSON::Object::create());
469         return;
470     }
471
472     m_loadTimer.startOneShot(timeout);
473     switch (loadStrategy) {
474     case Inspector::Protocol::Automation::PageLoadStrategy::Normal:
475         m_pendingNormalNavigationInBrowsingContextCallbacksPerFrame.set(frame.frameID(), WTFMove(callback));
476         break;
477     case Inspector::Protocol::Automation::PageLoadStrategy::Eager:
478         m_pendingEagerNavigationInBrowsingContextCallbacksPerFrame.set(frame.frameID(), WTFMove(callback));
479         break;
480     case Inspector::Protocol::Automation::PageLoadStrategy::None:
481         ASSERT_NOT_REACHED();
482     }
483 }
484
485 void WebAutomationSession::respondToPendingPageNavigationCallbacksWithTimeout(HashMap<uint64_t, RefPtr<Inspector::BackendDispatcher::CallbackBase>>& map)
486 {
487     Inspector::ErrorString timeoutError = STRING_FOR_PREDEFINED_ERROR_NAME(Timeout);
488     for (auto id : copyToVector(map.keys())) {
489         auto page = WebProcessProxy::webPage(id);
490         auto callback = map.take(id);
491         if (page && m_client->isShowingJavaScriptDialogOnPage(*this, *page))
492             callback->sendSuccess(JSON::Object::create());
493         else
494             callback->sendFailure(timeoutError);
495     }
496 }
497
498 static WebPageProxy* findPageForFrameID(const WebProcessPool& processPool, uint64_t frameID)
499 {
500     for (auto& process : processPool.processes()) {
501         if (auto* frame = process->webFrame(frameID))
502             return frame->page();
503     }
504     return nullptr;
505 }
506
507 void WebAutomationSession::respondToPendingFrameNavigationCallbacksWithTimeout(HashMap<uint64_t, RefPtr<Inspector::BackendDispatcher::CallbackBase>>& map)
508 {
509     Inspector::ErrorString timeoutError = STRING_FOR_PREDEFINED_ERROR_NAME(Timeout);
510     for (auto id : copyToVector(map.keys())) {
511         auto* page = findPageForFrameID(*m_processPool, id);
512         auto callback = map.take(id);
513         if (page && m_client->isShowingJavaScriptDialogOnPage(*this, *page))
514             callback->sendSuccess(JSON::Object::create());
515         else
516             callback->sendFailure(timeoutError);
517     }
518 }
519
520 void WebAutomationSession::loadTimerFired()
521 {
522     respondToPendingFrameNavigationCallbacksWithTimeout(m_pendingNormalNavigationInBrowsingContextCallbacksPerFrame);
523     respondToPendingFrameNavigationCallbacksWithTimeout(m_pendingEagerNavigationInBrowsingContextCallbacksPerFrame);
524     respondToPendingPageNavigationCallbacksWithTimeout(m_pendingNormalNavigationInBrowsingContextCallbacksPerPage);
525     respondToPendingPageNavigationCallbacksWithTimeout(m_pendingEagerNavigationInBrowsingContextCallbacksPerPage);
526 }
527
528 void WebAutomationSession::maximizeWindowOfBrowsingContext(const String& browsingContextHandle, Ref<MaximizeWindowOfBrowsingContextCallback>&& callback)
529 {
530     auto* page = webPageProxyForHandle(browsingContextHandle);
531     if (!page)
532         ASYNC_FAIL_WITH_PREDEFINED_ERROR(WindowNotFound);
533
534     exitFullscreenWindowForPage(*page, [this, protectedThis = makeRef(*this), callback = WTFMove(callback), page = makeRefPtr(page)]() mutable {
535         auto& webPage = *page;
536         restoreWindowForPage(webPage, [this, callback = WTFMove(callback), page = WTFMove(page)]() mutable {
537             maximizeWindowForPage(*page, [callback = WTFMove(callback)]() {
538                 callback->sendSuccess();
539             });
540         });
541     });
542 }
543
544 void WebAutomationSession::hideWindowOfBrowsingContext(const String& browsingContextHandle, Ref<HideWindowOfBrowsingContextCallback>&& callback)
545 {
546     WebPageProxy* page = webPageProxyForHandle(browsingContextHandle);
547     if (!page)
548         ASYNC_FAIL_WITH_PREDEFINED_ERROR(WindowNotFound);
549     
550     exitFullscreenWindowForPage(*page, [protectedThis = makeRef(*this), callback = WTFMove(callback), page = makeRefPtr(page)]() mutable {
551         protectedThis->hideWindowForPage(*page, [callback = WTFMove(callback)]() mutable {
552             callback->sendSuccess();
553         });
554     });
555 }
556
557 void WebAutomationSession::exitFullscreenWindowForPage(WebPageProxy& page, WTF::CompletionHandler<void()>&& completionHandler)
558 {
559 #if ENABLE(FULLSCREEN_API)
560     ASSERT(!m_windowStateTransitionCallback);
561     if (!page.fullScreenManager()->isFullScreen()) {
562         completionHandler();
563         return;
564     }
565     
566     m_windowStateTransitionCallback = WTF::Function<void(WindowTransitionedToState)> { [this, protectedThis = makeRef(*this), completionHandler = WTFMove(completionHandler)](WindowTransitionedToState state) mutable {
567         // If fullscreen exited and we didn't request that, just ignore it.
568         if (state != WindowTransitionedToState::Unfullscreen)
569             return;
570
571         // Keep this callback in scope so completionHandler does not get destroyed before we call it.
572         auto protectedCallback = WTFMove(m_windowStateTransitionCallback);
573         completionHandler();
574     } };
575     
576     page.fullScreenManager()->requestExitFullScreen();
577 #else
578     completionHandler();
579 #endif
580 }
581
582 void WebAutomationSession::restoreWindowForPage(WebPageProxy& page, WTF::CompletionHandler<void()>&& completionHandler)
583 {
584     m_client->requestRestoreWindowOfPage(*this, page, WTFMove(completionHandler));
585 }
586
587 void WebAutomationSession::maximizeWindowForPage(WebPageProxy& page, WTF::CompletionHandler<void()>&& completionHandler)
588 {
589     m_client->requestMaximizeWindowOfPage(*this, page, WTFMove(completionHandler));
590 }
591
592 void WebAutomationSession::hideWindowForPage(WebPageProxy& page, WTF::CompletionHandler<void()>&& completionHandler)
593 {
594     m_client->requestHideWindowOfPage(*this, page, WTFMove(completionHandler));
595 }
596
597 void WebAutomationSession::willShowJavaScriptDialog(WebPageProxy& page)
598 {
599     // Wait until the next run loop iteration to give time for the client to show the dialog,
600     // then check if the dialog is still present. If the page is loading, the dialog will block
601     // the load in case of normal strategy, so we want to dispatch all pending navigation callbacks.
602     // If the dialog was shown during a script execution, we want to finish the evaluateJavaScriptFunction
603     // operation with an unexpected alert open error.
604     RunLoop::main().dispatch([this, protectedThis = makeRef(*this), page = makeRef(page)] {
605         if (!page->isValid() || !m_client || !m_client->isShowingJavaScriptDialogOnPage(*this, page))
606             return;
607
608         if (page->pageLoadState().isLoading()) {
609             m_loadTimer.stop();
610             respondToPendingFrameNavigationCallbacksWithTimeout(m_pendingNormalNavigationInBrowsingContextCallbacksPerFrame);
611             respondToPendingPageNavigationCallbacksWithTimeout(m_pendingNormalNavigationInBrowsingContextCallbacksPerPage);
612         }
613
614         if (!m_evaluateJavaScriptFunctionCallbacks.isEmpty()) {
615             Inspector::ErrorString unexpectedAlertOpenError = STRING_FOR_PREDEFINED_ERROR_NAME(UnexpectedAlertOpen);
616             for (auto key : copyToVector(m_evaluateJavaScriptFunctionCallbacks.keys())) {
617                 auto callback = m_evaluateJavaScriptFunctionCallbacks.take(key);
618                 callback->sendFailure(unexpectedAlertOpenError);
619             }
620         }
621         
622         if (!m_pendingMouseEventsFlushedCallbacksPerPage.isEmpty()) {
623             for (auto key : copyToVector(m_pendingMouseEventsFlushedCallbacksPerPage.keys())) {
624                 auto callback = m_pendingMouseEventsFlushedCallbacksPerPage.take(key);
625                 callback(std::nullopt);
626             }
627         }
628
629         if (!m_pendingKeyboardEventsFlushedCallbacksPerPage.isEmpty()) {
630             for (auto key : copyToVector(m_pendingKeyboardEventsFlushedCallbacksPerPage.keys())) {
631                 auto callback = m_pendingKeyboardEventsFlushedCallbacksPerPage.take(key);
632                 callback(std::nullopt);
633             }
634         }
635     });
636 }
637     
638 void WebAutomationSession::didEnterFullScreenForPage(const WebPageProxy&)
639 {
640     if (m_windowStateTransitionCallback)
641         m_windowStateTransitionCallback(WindowTransitionedToState::Fullscreen);
642 }
643
644 void WebAutomationSession::didExitFullScreenForPage(const WebPageProxy&)
645 {
646     if (m_windowStateTransitionCallback)
647         m_windowStateTransitionCallback(WindowTransitionedToState::Unfullscreen);
648 }
649
650 void WebAutomationSession::navigateBrowsingContext(const String& handle, const String& url, const String* optionalPageLoadStrategyString, const int* optionalPageLoadTimeout, Ref<NavigateBrowsingContextCallback>&& callback)
651 {
652     WebPageProxy* page = webPageProxyForHandle(handle);
653     if (!page)
654         ASYNC_FAIL_WITH_PREDEFINED_ERROR(WindowNotFound);
655
656     auto pageLoadStrategy = pageLoadStrategyFromStringParameter(optionalPageLoadStrategyString);
657     if (!pageLoadStrategy)
658         ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(InvalidParameter, "The parameter 'pageLoadStrategy' is invalid.");
659     auto pageLoadTimeout = optionalPageLoadTimeout ? Seconds::fromMilliseconds(*optionalPageLoadTimeout) : defaultPageLoadTimeout;
660
661     page->loadRequest(WebCore::URL(WebCore::URL(), url));
662     waitForNavigationToCompleteOnPage(*page, pageLoadStrategy.value(), pageLoadTimeout, WTFMove(callback));
663 }
664
665 void WebAutomationSession::goBackInBrowsingContext(const String& handle, const String* optionalPageLoadStrategyString, const int* optionalPageLoadTimeout, Ref<GoBackInBrowsingContextCallback>&& callback)
666 {
667     WebPageProxy* page = webPageProxyForHandle(handle);
668     if (!page)
669         ASYNC_FAIL_WITH_PREDEFINED_ERROR(WindowNotFound);
670
671     auto pageLoadStrategy = pageLoadStrategyFromStringParameter(optionalPageLoadStrategyString);
672     if (!pageLoadStrategy)
673         ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(InvalidParameter, "The parameter 'pageLoadStrategy' is invalid.");
674     auto pageLoadTimeout = optionalPageLoadTimeout ? Seconds::fromMilliseconds(*optionalPageLoadTimeout) : defaultPageLoadTimeout;
675
676     page->goBack();
677     waitForNavigationToCompleteOnPage(*page, pageLoadStrategy.value(), pageLoadTimeout, WTFMove(callback));
678 }
679
680 void WebAutomationSession::goForwardInBrowsingContext(const String& handle, const String* optionalPageLoadStrategyString, const int* optionalPageLoadTimeout, Ref<GoForwardInBrowsingContextCallback>&& callback)
681 {
682     WebPageProxy* page = webPageProxyForHandle(handle);
683     if (!page)
684         ASYNC_FAIL_WITH_PREDEFINED_ERROR(WindowNotFound);
685
686     auto pageLoadStrategy = pageLoadStrategyFromStringParameter(optionalPageLoadStrategyString);
687     if (!pageLoadStrategy)
688         ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(InvalidParameter, "The parameter 'pageLoadStrategy' is invalid.");
689     auto pageLoadTimeout = optionalPageLoadTimeout ? Seconds::fromMilliseconds(*optionalPageLoadTimeout) : defaultPageLoadTimeout;
690
691     page->goForward();
692     waitForNavigationToCompleteOnPage(*page, pageLoadStrategy.value(), pageLoadTimeout, WTFMove(callback));
693 }
694
695 void WebAutomationSession::reloadBrowsingContext(const String& handle, const String* optionalPageLoadStrategyString, const int* optionalPageLoadTimeout, Ref<ReloadBrowsingContextCallback>&& callback)
696 {
697     WebPageProxy* page = webPageProxyForHandle(handle);
698     if (!page)
699         ASYNC_FAIL_WITH_PREDEFINED_ERROR(WindowNotFound);
700
701     auto pageLoadStrategy = pageLoadStrategyFromStringParameter(optionalPageLoadStrategyString);
702     if (!pageLoadStrategy)
703         ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(InvalidParameter, "The parameter 'pageLoadStrategy' is invalid.");
704     auto pageLoadTimeout = optionalPageLoadTimeout ? Seconds::fromMilliseconds(*optionalPageLoadTimeout) : defaultPageLoadTimeout;
705
706     page->reload({ });
707     waitForNavigationToCompleteOnPage(*page, pageLoadStrategy.value(), pageLoadTimeout, WTFMove(callback));
708 }
709
710 void WebAutomationSession::navigationOccurredForFrame(const WebFrameProxy& frame)
711 {
712     if (frame.isMainFrame()) {
713         // New page loaded, clear frame handles previously cached.
714         m_handleWebFrameMap.clear();
715         m_webFrameHandleMap.clear();
716         if (auto callback = m_pendingNormalNavigationInBrowsingContextCallbacksPerPage.take(frame.page()->pageID())) {
717             m_loadTimer.stop();
718             callback->sendSuccess(JSON::Object::create());
719         }
720         m_domainNotifier->browsingContextCleared(handleForWebPageProxy(*frame.page()));
721     } else {
722         if (auto callback = m_pendingNormalNavigationInBrowsingContextCallbacksPerFrame.take(frame.frameID())) {
723             m_loadTimer.stop();
724             callback->sendSuccess(JSON::Object::create());
725         }
726     }
727 }
728
729 void WebAutomationSession::documentLoadedForFrame(const WebFrameProxy& frame)
730 {
731     if (frame.isMainFrame()) {
732         if (auto callback = m_pendingEagerNavigationInBrowsingContextCallbacksPerPage.take(frame.page()->pageID())) {
733             m_loadTimer.stop();
734             callback->sendSuccess(JSON::Object::create());
735         }
736     } else {
737         if (auto callback = m_pendingEagerNavigationInBrowsingContextCallbacksPerFrame.take(frame.frameID())) {
738             m_loadTimer.stop();
739             callback->sendSuccess(JSON::Object::create());
740         }
741     }
742 }
743
744 void WebAutomationSession::inspectorFrontendLoaded(const WebPageProxy& page)
745 {
746     if (auto callback = m_pendingInspectorCallbacksPerPage.take(page.pageID()))
747         callback->sendSuccess(JSON::Object::create());
748 }
749
750 void WebAutomationSession::mouseEventsFlushedForPage(const WebPageProxy& page)
751 {
752     if (auto callback = m_pendingMouseEventsFlushedCallbacksPerPage.take(page.pageID()))
753         callback(std::nullopt);
754 }
755
756 void WebAutomationSession::keyboardEventsFlushedForPage(const WebPageProxy& page)
757 {
758     if (auto callback = m_pendingKeyboardEventsFlushedCallbacksPerPage.take(page.pageID()))
759         callback(std::nullopt);
760 }
761
762 void WebAutomationSession::willClosePage(const WebPageProxy& page)
763 {
764     String handle = handleForWebPageProxy(page);
765     m_domainNotifier->browsingContextCleared(handle);
766
767     // Cancel pending interactions on this page. By providing an error, this will cause subsequent
768     // actions to be aborted and the SimulatedInputDispatcher::run() call will unwind and fail.
769     if (auto callback = m_pendingMouseEventsFlushedCallbacksPerPage.take(page.pageID()))
770         callback(AUTOMATION_COMMAND_ERROR_WITH_NAME(WindowNotFound));
771     if (auto callback = m_pendingMouseEventsFlushedCallbacksPerPage.take(page.pageID()))
772         callback(AUTOMATION_COMMAND_ERROR_WITH_NAME(WindowNotFound));
773
774     // Then tell the input dispatcher to cancel so timers are stopped, and let it go out of scope.
775     std::optional<Ref<SimulatedInputDispatcher>> inputDispatcher = m_inputDispatchersByPage.take(page.pageID());
776     if (inputDispatcher.has_value())
777         inputDispatcher.value()->cancel();
778 }
779
780 static bool fileCanBeAcceptedForUpload(const String& filename, const HashSet<String>& allowedMIMETypes, const HashSet<String>& allowedFileExtensions)
781 {
782     if (!WebCore::FileSystem::fileExists(filename))
783         return false;
784
785     if (allowedMIMETypes.isEmpty() && allowedFileExtensions.isEmpty())
786         return true;
787
788     // We can't infer a MIME type from a file without an extension, just give up.
789     auto dotOffset = filename.reverseFind('.');
790     if (dotOffset == notFound)
791         return false;
792
793     String extension = filename.substring(dotOffset + 1).convertToASCIILowercase();
794     if (extension.isEmpty())
795         return false;
796
797     if (allowedFileExtensions.contains(extension))
798         return true;
799
800     String mappedMIMEType = WebCore::MIMETypeRegistry::getMIMETypeForExtension(extension).convertToASCIILowercase();
801     if (mappedMIMEType.isEmpty())
802         return false;
803     
804     if (allowedMIMETypes.contains(mappedMIMEType))
805         return true;
806
807     // Fall back to checking for a MIME type wildcard if an exact match is not found.
808     Vector<String> components;
809     mappedMIMEType.split('/', false, components);
810     if (components.size() != 2)
811         return false;
812
813     String wildcardedMIMEType = makeString(components[0], "/*");
814     if (allowedMIMETypes.contains(wildcardedMIMEType))
815         return true;
816
817     return false;
818 }
819
820 void WebAutomationSession::handleRunOpenPanel(const WebPageProxy& page, const WebFrameProxy&, const API::OpenPanelParameters& parameters, WebOpenPanelResultListenerProxy& resultListener)
821 {
822     String browsingContextHandle = handleForWebPageProxy(page);
823     if (!m_filesToSelectForFileUpload.size()) {
824         resultListener.cancel();
825         m_domainNotifier->fileChooserDismissed(browsingContextHandle, true);
826         return;
827     }
828
829     if (m_filesToSelectForFileUpload.size() > 1 && !parameters.allowMultipleFiles()) {
830         resultListener.cancel();
831         m_domainNotifier->fileChooserDismissed(browsingContextHandle, true);
832         return;
833     }
834
835     HashSet<String> allowedMIMETypes;
836     auto acceptMIMETypes = parameters.acceptMIMETypes();
837     for (auto type : acceptMIMETypes->elementsOfType<API::String>())
838         allowedMIMETypes.add(type->string());
839
840     HashSet<String> allowedFileExtensions;
841     auto acceptFileExtensions = parameters.acceptFileExtensions();
842     for (auto type : acceptFileExtensions->elementsOfType<API::String>()) {
843         // WebCore vends extensions with leading periods. Strip these to simplify matching later.
844         String extension = type->string();
845         ASSERT(extension.characterAt(0) == '.');
846         allowedFileExtensions.add(extension.substring(1));
847     }
848
849     // Per §14.3.10.5 in the W3C spec, if at least one file cannot be accepted, the command should fail.
850     // The REST API service can tell that this failed by checking the "files" attribute of the input element.
851     for (const String& filename : m_filesToSelectForFileUpload) {
852         if (!fileCanBeAcceptedForUpload(filename, allowedMIMETypes, allowedFileExtensions)) {
853             resultListener.cancel();
854             m_domainNotifier->fileChooserDismissed(browsingContextHandle, true);
855             return;
856         }
857     }
858
859     resultListener.chooseFiles(m_filesToSelectForFileUpload);
860     m_domainNotifier->fileChooserDismissed(browsingContextHandle, false);
861 }
862
863 void WebAutomationSession::evaluateJavaScriptFunction(const String& browsingContextHandle, const String* optionalFrameHandle, const String& function, const JSON::Array& arguments, const bool* optionalExpectsImplicitCallbackArgument, const int* optionalCallbackTimeout, Ref<EvaluateJavaScriptFunctionCallback>&& callback)
864 {
865     WebPageProxy* page = webPageProxyForHandle(browsingContextHandle);
866     if (!page)
867         ASYNC_FAIL_WITH_PREDEFINED_ERROR(WindowNotFound);
868
869     std::optional<uint64_t> frameID = webFrameIDForHandle(optionalFrameHandle ? *optionalFrameHandle : emptyString());
870     if (!frameID)
871         ASYNC_FAIL_WITH_PREDEFINED_ERROR(FrameNotFound);
872
873     Vector<String> argumentsVector;
874     argumentsVector.reserveCapacity(arguments.length());
875
876     for (auto& argument : arguments) {
877         String argumentString;
878         argument->asString(argumentString);
879         argumentsVector.uncheckedAppend(argumentString);
880     }
881
882     bool expectsImplicitCallbackArgument = optionalExpectsImplicitCallbackArgument ? *optionalExpectsImplicitCallbackArgument : false;
883     int callbackTimeout = optionalCallbackTimeout ? *optionalCallbackTimeout : 0;
884
885     uint64_t callbackID = m_nextEvaluateJavaScriptCallbackID++;
886     m_evaluateJavaScriptFunctionCallbacks.set(callbackID, WTFMove(callback));
887
888     page->process().send(Messages::WebAutomationSessionProxy::EvaluateJavaScriptFunction(page->pageID(), frameID.value(), function, argumentsVector, expectsImplicitCallbackArgument, callbackTimeout, callbackID), 0);
889 }
890
891 void WebAutomationSession::didEvaluateJavaScriptFunction(uint64_t callbackID, const String& result, const String& errorType)
892 {
893     auto callback = m_evaluateJavaScriptFunctionCallbacks.take(callbackID);
894     if (!callback)
895         return;
896
897     if (!errorType.isEmpty())
898         callback->sendFailure(STRING_FOR_PREDEFINED_ERROR_MESSAGE_AND_DETAILS(errorType, result));
899     else
900         callback->sendSuccess(result);
901 }
902
903 void WebAutomationSession::resolveChildFrameHandle(const String& browsingContextHandle, const String* optionalFrameHandle, const int* optionalOrdinal, const String* optionalName, const String* optionalNodeHandle, Ref<ResolveChildFrameHandleCallback>&& callback)
904 {
905     if (!optionalOrdinal && !optionalName && !optionalNodeHandle)
906         ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(MissingParameter, "Command must specify a child frame by ordinal, name, or element handle.");
907
908     WebPageProxy* page = webPageProxyForHandle(browsingContextHandle);
909     if (!page)
910         ASYNC_FAIL_WITH_PREDEFINED_ERROR(WindowNotFound);
911
912     std::optional<uint64_t> frameID = webFrameIDForHandle(optionalFrameHandle ? *optionalFrameHandle : emptyString());
913     if (!frameID)
914         ASYNC_FAIL_WITH_PREDEFINED_ERROR(FrameNotFound);
915
916     uint64_t callbackID = m_nextResolveFrameCallbackID++;
917     m_resolveChildFrameHandleCallbacks.set(callbackID, WTFMove(callback));
918
919     if (optionalNodeHandle) {
920         page->process().send(Messages::WebAutomationSessionProxy::ResolveChildFrameWithNodeHandle(page->pageID(), frameID.value(), *optionalNodeHandle, callbackID), 0);
921         return;
922     }
923
924     if (optionalName) {
925         page->process().send(Messages::WebAutomationSessionProxy::ResolveChildFrameWithName(page->pageID(), frameID.value(), *optionalName, callbackID), 0);
926         return;
927     }
928
929     if (optionalOrdinal) {
930         page->process().send(Messages::WebAutomationSessionProxy::ResolveChildFrameWithOrdinal(page->pageID(), frameID.value(), *optionalOrdinal, callbackID), 0);
931         return;
932     }
933
934     ASSERT_NOT_REACHED();
935 }
936
937 void WebAutomationSession::didResolveChildFrame(uint64_t callbackID, uint64_t frameID, const String& errorType)
938 {
939     auto callback = m_resolveChildFrameHandleCallbacks.take(callbackID);
940     if (!callback)
941         return;
942
943     if (!errorType.isEmpty())
944         callback->sendFailure(STRING_FOR_PREDEFINED_ERROR_MESSAGE(errorType));
945     else
946         callback->sendSuccess(handleForWebFrameID(frameID));
947 }
948
949 void WebAutomationSession::resolveParentFrameHandle(const String& browsingContextHandle, const String& frameHandle, Ref<ResolveParentFrameHandleCallback>&& callback)
950 {
951     WebPageProxy* page = webPageProxyForHandle(browsingContextHandle);
952     if (!page)
953         ASYNC_FAIL_WITH_PREDEFINED_ERROR(WindowNotFound);
954
955     std::optional<uint64_t> frameID = webFrameIDForHandle(frameHandle);
956     if (!frameID)
957         ASYNC_FAIL_WITH_PREDEFINED_ERROR(FrameNotFound);
958
959     uint64_t callbackID = m_nextResolveParentFrameCallbackID++;
960     m_resolveParentFrameHandleCallbacks.set(callbackID, WTFMove(callback));
961
962     page->process().send(Messages::WebAutomationSessionProxy::ResolveParentFrame(page->pageID(), frameID.value(), callbackID), 0);
963 }
964
965 void WebAutomationSession::didResolveParentFrame(uint64_t callbackID, uint64_t frameID, const String& errorType)
966 {
967     auto callback = m_resolveParentFrameHandleCallbacks.take(callbackID);
968     if (!callback)
969         return;
970
971     if (!errorType.isEmpty())
972         callback->sendFailure(STRING_FOR_PREDEFINED_ERROR_MESSAGE(errorType));
973     else
974         callback->sendSuccess(handleForWebFrameID(frameID));
975 }
976
977 static std::optional<CoordinateSystem> protocolStringToCoordinateSystem(const String& coordinateSystemString)
978 {
979     if (coordinateSystemString == "Page")
980         return CoordinateSystem::Page;
981     if (coordinateSystemString == "LayoutViewport")
982         return CoordinateSystem::LayoutViewport;
983     if (coordinateSystemString == "VisualViewport")
984         return CoordinateSystem::VisualViewport;
985     return std::nullopt;
986 }
987
988 void WebAutomationSession::computeElementLayout(const String& browsingContextHandle, const String& frameHandle, const String& nodeHandle, const bool* optionalScrollIntoViewIfNeeded, const String& coordinateSystemString, Ref<ComputeElementLayoutCallback>&& callback)
989 {
990     WebPageProxy* page = webPageProxyForHandle(browsingContextHandle);
991     if (!page)
992         ASYNC_FAIL_WITH_PREDEFINED_ERROR(WindowNotFound);
993
994     std::optional<uint64_t> frameID = webFrameIDForHandle(frameHandle);
995     if (!frameID)
996         ASYNC_FAIL_WITH_PREDEFINED_ERROR(FrameNotFound);
997
998     std::optional<CoordinateSystem> coordinateSystem = protocolStringToCoordinateSystem(coordinateSystemString);
999     if (!coordinateSystem)
1000         ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(InvalidParameter, "The parameter 'coordinateSystem' is invalid.");
1001
1002     // Start at 2 and use only even numbers to not conflict with m_nextViewportInViewCenterPointOfElementCallbackID.
1003     uint64_t callbackID = m_nextComputeElementLayoutCallbackID += 2;
1004     m_computeElementLayoutCallbacks.set(callbackID, WTFMove(callback));
1005
1006     bool scrollIntoViewIfNeeded = optionalScrollIntoViewIfNeeded ? *optionalScrollIntoViewIfNeeded : false;
1007     page->process().send(Messages::WebAutomationSessionProxy::ComputeElementLayout(page->pageID(), frameID.value(), nodeHandle, scrollIntoViewIfNeeded, coordinateSystem.value(), callbackID), 0);
1008 }
1009
1010 void WebAutomationSession::didComputeElementLayout(uint64_t callbackID, WebCore::IntRect rect, std::optional<WebCore::IntPoint> inViewCenterPoint, bool isObscured, const String& errorType)
1011 {
1012     if (callbackID % 2 == 1) {
1013         ASSERT(inViewCenterPoint);
1014         if (auto callback = m_viewportInViewCenterPointOfElementCallbacks.take(callbackID)) {
1015             std::optional<AutomationCommandError> error;
1016             if (!errorType.isEmpty())
1017                 error = AUTOMATION_COMMAND_ERROR_WITH_MESSAGE(errorType);
1018             callback(inViewCenterPoint, error);
1019         }
1020         return;
1021     }
1022
1023     auto callback = m_computeElementLayoutCallbacks.take(callbackID);
1024     if (!callback)
1025         return;
1026
1027     if (!errorType.isEmpty()) {
1028         callback->sendFailure(STRING_FOR_PREDEFINED_ERROR_MESSAGE(errorType));
1029         return;
1030     }
1031
1032     auto originObject = Inspector::Protocol::Automation::Point::create()
1033         .setX(rect.x())
1034         .setY(rect.y())
1035         .release();
1036
1037     auto sizeObject = Inspector::Protocol::Automation::Size::create()
1038         .setWidth(rect.width())
1039         .setHeight(rect.height())
1040         .release();
1041
1042     auto rectObject = Inspector::Protocol::Automation::Rect::create()
1043         .setOrigin(WTFMove(originObject))
1044         .setSize(WTFMove(sizeObject))
1045         .release();
1046
1047     if (!inViewCenterPoint) {
1048         callback->sendSuccess(WTFMove(rectObject), nullptr, isObscured);
1049         return;
1050     }
1051
1052     auto inViewCenterPointObject = Inspector::Protocol::Automation::Point::create()
1053         .setX(inViewCenterPoint.value().x())
1054         .setY(inViewCenterPoint.value().y())
1055         .release();
1056
1057     callback->sendSuccess(WTFMove(rectObject), WTFMove(inViewCenterPointObject), isObscured);
1058 }
1059
1060 void WebAutomationSession::selectOptionElement(const String& browsingContextHandle, const String& frameHandle, const String& nodeHandle, Ref<SelectOptionElementCallback>&& callback)
1061 {
1062     WebPageProxy* page = webPageProxyForHandle(browsingContextHandle);
1063     if (!page)
1064         ASYNC_FAIL_WITH_PREDEFINED_ERROR(WindowNotFound);
1065
1066     std::optional<uint64_t> frameID = webFrameIDForHandle(frameHandle);
1067     if (!frameID)
1068         ASYNC_FAIL_WITH_PREDEFINED_ERROR(FrameNotFound);
1069
1070     uint64_t callbackID = m_nextSelectOptionElementCallbackID++;
1071     m_selectOptionElementCallbacks.set(callbackID, WTFMove(callback));
1072
1073     page->process().send(Messages::WebAutomationSessionProxy::SelectOptionElement(page->pageID(), frameID.value(), nodeHandle, callbackID), 0);
1074 }
1075
1076 void WebAutomationSession::didSelectOptionElement(uint64_t callbackID, const String& errorType)
1077 {
1078     auto callback = m_selectOptionElementCallbacks.take(callbackID);
1079     if (!callback)
1080         return;
1081
1082     if (!errorType.isEmpty()) {
1083         callback->sendFailure(STRING_FOR_PREDEFINED_ERROR_MESSAGE(errorType));
1084         return;
1085     }
1086
1087     callback->sendSuccess();
1088 }
1089
1090
1091 void WebAutomationSession::isShowingJavaScriptDialog(Inspector::ErrorString& errorString, const String& browsingContextHandle, bool* result)
1092 {
1093     ASSERT(m_client);
1094     if (!m_client)
1095         SYNC_FAIL_WITH_PREDEFINED_ERROR(InternalError);
1096
1097     WebPageProxy* page = webPageProxyForHandle(browsingContextHandle);
1098     if (!page)
1099         SYNC_FAIL_WITH_PREDEFINED_ERROR(WindowNotFound);
1100
1101     *result = m_client->isShowingJavaScriptDialogOnPage(*this, *page);
1102 }
1103
1104 void WebAutomationSession::dismissCurrentJavaScriptDialog(Inspector::ErrorString& errorString, const String& browsingContextHandle)
1105 {
1106     ASSERT(m_client);
1107     if (!m_client)
1108         SYNC_FAIL_WITH_PREDEFINED_ERROR(InternalError);
1109
1110     WebPageProxy* page = webPageProxyForHandle(browsingContextHandle);
1111     if (!page)
1112         SYNC_FAIL_WITH_PREDEFINED_ERROR(WindowNotFound);
1113
1114     if (!m_client->isShowingJavaScriptDialogOnPage(*this, *page))
1115         SYNC_FAIL_WITH_PREDEFINED_ERROR(NoJavaScriptDialog);
1116
1117     m_client->dismissCurrentJavaScriptDialogOnPage(*this, *page);
1118 }
1119
1120 void WebAutomationSession::acceptCurrentJavaScriptDialog(Inspector::ErrorString& errorString, const String& browsingContextHandle)
1121 {
1122     ASSERT(m_client);
1123     if (!m_client)
1124         SYNC_FAIL_WITH_PREDEFINED_ERROR(InternalError);
1125
1126     WebPageProxy* page = webPageProxyForHandle(browsingContextHandle);
1127     if (!page)
1128         SYNC_FAIL_WITH_PREDEFINED_ERROR(WindowNotFound);
1129
1130     if (!m_client->isShowingJavaScriptDialogOnPage(*this, *page))
1131         SYNC_FAIL_WITH_PREDEFINED_ERROR(NoJavaScriptDialog);
1132
1133     m_client->acceptCurrentJavaScriptDialogOnPage(*this, *page);
1134 }
1135
1136 void WebAutomationSession::messageOfCurrentJavaScriptDialog(Inspector::ErrorString& errorString, const String& browsingContextHandle, String* text)
1137 {
1138     ASSERT(m_client);
1139     if (!m_client)
1140         SYNC_FAIL_WITH_PREDEFINED_ERROR(InternalError);
1141
1142     WebPageProxy* page = webPageProxyForHandle(browsingContextHandle);
1143     if (!page)
1144         SYNC_FAIL_WITH_PREDEFINED_ERROR(WindowNotFound);
1145
1146     if (!m_client->isShowingJavaScriptDialogOnPage(*this, *page))
1147         SYNC_FAIL_WITH_PREDEFINED_ERROR(NoJavaScriptDialog);
1148
1149     *text = m_client->messageOfCurrentJavaScriptDialogOnPage(*this, *page);
1150 }
1151
1152 void WebAutomationSession::setUserInputForCurrentJavaScriptPrompt(Inspector::ErrorString& errorString, const String& browsingContextHandle, const String& promptValue)
1153 {
1154     ASSERT(m_client);
1155     if (!m_client)
1156         SYNC_FAIL_WITH_PREDEFINED_ERROR(InternalError);
1157
1158     WebPageProxy* page = webPageProxyForHandle(browsingContextHandle);
1159     if (!page)
1160         SYNC_FAIL_WITH_PREDEFINED_ERROR(WindowNotFound);
1161
1162     if (!m_client->isShowingJavaScriptDialogOnPage(*this, *page))
1163         SYNC_FAIL_WITH_PREDEFINED_ERROR(NoJavaScriptDialog);
1164
1165     // §18.4 Send Alert Text.
1166     // https://w3c.github.io/webdriver/webdriver-spec.html#send-alert-text
1167     // 3. Run the substeps of the first matching current user prompt:
1168     auto scriptDialogType = m_client->typeOfCurrentJavaScriptDialogOnPage(*this, *page);
1169     ASSERT(scriptDialogType);
1170     switch (scriptDialogType.value()) {
1171     case API::AutomationSessionClient::JavaScriptDialogType::Alert:
1172     case API::AutomationSessionClient::JavaScriptDialogType::Confirm:
1173         // Return error with error code element not interactable.
1174         SYNC_FAIL_WITH_PREDEFINED_ERROR(ElementNotInteractable);
1175     case API::AutomationSessionClient::JavaScriptDialogType::Prompt:
1176         // Do nothing.
1177         break;
1178     case API::AutomationSessionClient::JavaScriptDialogType::BeforeUnloadConfirm:
1179         // Return error with error code unsupported operation.
1180         SYNC_FAIL_WITH_PREDEFINED_ERROR(NotImplemented);
1181     }
1182
1183     m_client->setUserInputForCurrentJavaScriptPromptOnPage(*this, *page, promptValue);
1184 }
1185
1186 void WebAutomationSession::setFilesToSelectForFileUpload(ErrorString& errorString, const String& browsingContextHandle, const JSON::Array& filenames)
1187 {
1188     Vector<String> newFileList;
1189     newFileList.reserveInitialCapacity(filenames.length());
1190
1191     for (auto item : filenames) {
1192         String filename;
1193         if (!item->asString(filename))
1194             SYNC_FAIL_WITH_PREDEFINED_ERROR(InternalError);
1195
1196         newFileList.append(filename);
1197     }
1198
1199     m_filesToSelectForFileUpload.swap(newFileList);
1200 }
1201
1202 void WebAutomationSession::getAllCookies(const String& browsingContextHandle, Ref<GetAllCookiesCallback>&& callback)
1203 {
1204     WebPageProxy* page = webPageProxyForHandle(browsingContextHandle);
1205     if (!page)
1206         ASYNC_FAIL_WITH_PREDEFINED_ERROR(WindowNotFound);
1207
1208     // Always send the main frame ID as 0 so it is resolved on the WebProcess side. This avoids a race when page->mainFrame() is null still.
1209     const uint64_t mainFrameID = 0;
1210
1211     uint64_t callbackID = m_nextGetCookiesCallbackID++;
1212     m_getCookieCallbacks.set(callbackID, WTFMove(callback));
1213
1214     page->process().send(Messages::WebAutomationSessionProxy::GetCookiesForFrame(page->pageID(), mainFrameID, callbackID), 0);
1215 }
1216
1217 static Ref<Inspector::Protocol::Automation::Cookie> buildObjectForCookie(const WebCore::Cookie& cookie)
1218 {
1219     return Inspector::Protocol::Automation::Cookie::create()
1220         .setName(cookie.name)
1221         .setValue(cookie.value)
1222         .setDomain(cookie.domain)
1223         .setPath(cookie.path)
1224         .setExpires(cookie.expires ? cookie.expires / 1000 : 0)
1225         .setSize((cookie.name.length() + cookie.value.length()))
1226         .setHttpOnly(cookie.httpOnly)
1227         .setSecure(cookie.secure)
1228         .setSession(cookie.session)
1229         .release();
1230 }
1231
1232 static Ref<JSON::ArrayOf<Inspector::Protocol::Automation::Cookie>> buildArrayForCookies(Vector<WebCore::Cookie>& cookiesList)
1233 {
1234     auto cookies = JSON::ArrayOf<Inspector::Protocol::Automation::Cookie>::create();
1235
1236     for (const auto& cookie : cookiesList)
1237         cookies->addItem(buildObjectForCookie(cookie));
1238
1239     return cookies;
1240 }
1241
1242 void WebAutomationSession::didGetCookiesForFrame(uint64_t callbackID, Vector<WebCore::Cookie> cookies, const String& errorType)
1243 {
1244     auto callback = m_getCookieCallbacks.take(callbackID);
1245     if (!callback)
1246         return;
1247
1248     if (!errorType.isEmpty()) {
1249         callback->sendFailure(STRING_FOR_PREDEFINED_ERROR_MESSAGE(errorType));
1250         return;
1251     }
1252
1253     callback->sendSuccess(buildArrayForCookies(cookies));
1254 }
1255
1256 void WebAutomationSession::deleteSingleCookie(const String& browsingContextHandle, const String& cookieName, Ref<DeleteSingleCookieCallback>&& callback)
1257 {
1258     WebPageProxy* page = webPageProxyForHandle(browsingContextHandle);
1259     if (!page)
1260         ASYNC_FAIL_WITH_PREDEFINED_ERROR(WindowNotFound);
1261
1262     // Always send the main frame ID as 0 so it is resolved on the WebProcess side. This avoids a race when page->mainFrame() is null still.
1263     const uint64_t mainFrameID = 0;
1264
1265     uint64_t callbackID = m_nextDeleteCookieCallbackID++;
1266     m_deleteCookieCallbacks.set(callbackID, WTFMove(callback));
1267
1268     page->process().send(Messages::WebAutomationSessionProxy::DeleteCookie(page->pageID(), mainFrameID, cookieName, callbackID), 0);
1269 }
1270
1271 void WebAutomationSession::didDeleteCookie(uint64_t callbackID, const String& errorType)
1272 {
1273     auto callback = m_deleteCookieCallbacks.take(callbackID);
1274     if (!callback)
1275         return;
1276
1277     if (!errorType.isEmpty()) {
1278         callback->sendFailure(STRING_FOR_PREDEFINED_ERROR_MESSAGE(errorType));
1279         return;
1280     }
1281
1282     callback->sendSuccess();
1283 }
1284
1285 static String domainByAddingDotPrefixIfNeeded(String domain)
1286 {
1287     if (domain[0] != '.') {
1288         // RFC 2965: If an explicitly specified value does not start with a dot, the user agent supplies a leading dot.
1289         // Assume that any host that ends with a digit is trying to be an IP address.
1290         if (!WebCore::URL::hostIsIPAddress(domain))
1291             return makeString('.', domain);
1292     }
1293     
1294     return domain;
1295 }
1296     
1297 void WebAutomationSession::addSingleCookie(const String& browsingContextHandle, const JSON::Object& cookieObject, Ref<AddSingleCookieCallback>&& callback)
1298 {
1299     WebPageProxy* page = webPageProxyForHandle(browsingContextHandle);
1300     if (!page)
1301         ASYNC_FAIL_WITH_PREDEFINED_ERROR(WindowNotFound);
1302
1303     WebCore::URL activeURL = WebCore::URL(WebCore::URL(), page->pageLoadState().activeURL());
1304     ASSERT(activeURL.isValid());
1305
1306     WebCore::Cookie cookie;
1307
1308     if (!cookieObject.getString(WTF::ASCIILiteral("name"), cookie.name))
1309         ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(MissingParameter, "The parameter 'name' was not found.");
1310
1311     if (!cookieObject.getString(WTF::ASCIILiteral("value"), cookie.value))
1312         ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(MissingParameter, "The parameter 'value' was not found.");
1313
1314     String domain;
1315     if (!cookieObject.getString(WTF::ASCIILiteral("domain"), domain))
1316         ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(MissingParameter, "The parameter 'domain' was not found.");
1317
1318     // Inherit the domain/host from the main frame's URL if it is not explicitly set.
1319     if (domain.isEmpty())
1320         domain = activeURL.host();
1321
1322     cookie.domain = domainByAddingDotPrefixIfNeeded(domain);
1323
1324     if (!cookieObject.getString(WTF::ASCIILiteral("path"), cookie.path))
1325         ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(MissingParameter, "The parameter 'path' was not found.");
1326
1327     double expires;
1328     if (!cookieObject.getDouble(WTF::ASCIILiteral("expires"), expires))
1329         ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(MissingParameter, "The parameter 'expires' was not found.");
1330
1331     cookie.expires = expires * 1000.0;
1332
1333     if (!cookieObject.getBoolean(WTF::ASCIILiteral("secure"), cookie.secure))
1334         ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(MissingParameter, "The parameter 'secure' was not found.");
1335
1336     if (!cookieObject.getBoolean(WTF::ASCIILiteral("session"), cookie.session))
1337         ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(MissingParameter, "The parameter 'session' was not found.");
1338
1339     if (!cookieObject.getBoolean(WTF::ASCIILiteral("httpOnly"), cookie.httpOnly))
1340         ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(MissingParameter, "The parameter 'httpOnly' was not found.");
1341
1342     WebCookieManagerProxy* cookieManager = m_processPool->supplement<WebCookieManagerProxy>();
1343
1344     // FIXME: Using activeURL here twice is basically saying "this is always in the context of the main document"
1345     // which probably isn't accurate.
1346     cookieManager->setCookies(page->websiteDataStore().sessionID(), { cookie }, activeURL, activeURL, [callback = callback.copyRef()](CallbackBase::Error error) {
1347         if (error == CallbackBase::Error::None)
1348             callback->sendSuccess();
1349         else
1350             callback->sendFailure(STRING_FOR_PREDEFINED_ERROR_NAME(InternalError));
1351     });
1352 }
1353
1354 void WebAutomationSession::deleteAllCookies(ErrorString& errorString, const String& browsingContextHandle)
1355 {
1356     WebPageProxy* page = webPageProxyForHandle(browsingContextHandle);
1357     if (!page)
1358         SYNC_FAIL_WITH_PREDEFINED_ERROR(WindowNotFound);
1359
1360     WebCore::URL activeURL = WebCore::URL(WebCore::URL(), page->pageLoadState().activeURL());
1361     ASSERT(activeURL.isValid());
1362
1363     WebCookieManagerProxy* cookieManager = m_processPool->supplement<WebCookieManagerProxy>();
1364     cookieManager->deleteCookiesForHostname(page->websiteDataStore().sessionID(), domainByAddingDotPrefixIfNeeded(activeURL.host()));
1365 }
1366
1367 void WebAutomationSession::getSessionPermissions(ErrorString&, RefPtr<JSON::ArrayOf<Inspector::Protocol::Automation::SessionPermissionData>>& out_permissions)
1368 {
1369     auto permissionsObjectArray = JSON::ArrayOf<Inspector::Protocol::Automation::SessionPermissionData>::create();
1370     auto getUserMediaPermissionObject = Inspector::Protocol::Automation::SessionPermissionData::create()
1371         .setPermission(Inspector::Protocol::Automation::SessionPermission::GetUserMedia)
1372         .setValue(m_permissionForGetUserMedia)
1373         .release();
1374
1375     permissionsObjectArray->addItem(WTFMove(getUserMediaPermissionObject));
1376     out_permissions = WTFMove(permissionsObjectArray);
1377 }
1378
1379 void WebAutomationSession::setSessionPermissions(ErrorString& errorString, const JSON::Array& permissions)
1380 {
1381     for (auto it = permissions.begin(); it != permissions.end(); ++it) {
1382         RefPtr<JSON::Object> permission;
1383         if (!it->get()->asObject(permission))
1384             SYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(InvalidParameter, "The parameter 'permissions' is invalid.");
1385
1386         String permissionName;
1387         if (!permission->getString(WTF::ASCIILiteral("permission"), permissionName))
1388             SYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(InvalidParameter, "The parameter 'permission' is missing or invalid.");
1389
1390         auto parsedPermissionName = Inspector::Protocol::AutomationHelpers::parseEnumValueFromString<Inspector::Protocol::Automation::SessionPermission>(permissionName);
1391         if (!parsedPermissionName)
1392             SYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(InvalidParameter, "The parameter 'permission' has an unknown value.");
1393
1394         bool permissionValue;
1395         if (!permission->getBoolean(WTF::ASCIILiteral("value"), permissionValue))
1396             SYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(InvalidParameter, "The parameter 'value' is missing or invalid.");
1397
1398         switch (parsedPermissionName.value()) {
1399         case Inspector::Protocol::Automation::SessionPermission::GetUserMedia:
1400             m_permissionForGetUserMedia = permissionValue;
1401             break;
1402         }
1403     }
1404 }
1405
1406 bool WebAutomationSession::shouldAllowGetUserMediaForPage(const WebPageProxy&) const
1407 {
1408     return m_permissionForGetUserMedia;
1409 }
1410
1411 bool WebAutomationSession::isSimulatingUserInteraction() const
1412 {
1413     if (!m_pendingMouseEventsFlushedCallbacksPerPage.isEmpty())
1414         return false;
1415     if (!m_pendingKeyboardEventsFlushedCallbacksPerPage.isEmpty())
1416         return false;
1417
1418     return true;
1419 }
1420
1421 SimulatedInputDispatcher& WebAutomationSession::inputDispatcherForPage(WebPageProxy& page)
1422 {
1423     return m_inputDispatchersByPage.ensure(page.pageID(), [&] {
1424         return SimulatedInputDispatcher::create(page, *this);
1425     }).iterator->value;
1426 }
1427
1428 SimulatedInputSource* WebAutomationSession::inputSourceForType(SimulatedInputSourceType type) const
1429 {
1430     // FIXME: this should use something like Vector's findMatching().
1431     for (auto& inputSource : m_inputSources) {
1432         if (inputSource->type == type)
1433             return &inputSource.get();
1434     }
1435
1436     return nullptr;
1437 }
1438
1439 // SimulatedInputDispatcher::Client API
1440 void WebAutomationSession::viewportInViewCenterPointOfElement(WebPageProxy& page, uint64_t frameID, const String& nodeHandle, Function<void (std::optional<WebCore::IntPoint>, std::optional<AutomationCommandError>)>&& completionHandler)
1441 {
1442     // Start at 3 and use only odd numbers to not conflict with m_nextComputeElementLayoutCallbackID.
1443     uint64_t callbackID = m_nextViewportInViewCenterPointOfElementCallbackID += 2;
1444     m_viewportInViewCenterPointOfElementCallbacks.set(callbackID, WTFMove(completionHandler));
1445
1446     page.process().send(Messages::WebAutomationSessionProxy::ComputeElementLayout(page.pageID(), frameID, nodeHandle, false, CoordinateSystem::LayoutViewport, callbackID), 0);
1447 }
1448
1449 void WebAutomationSession::simulateMouseInteraction(WebPageProxy& page, MouseInteraction interaction, WebMouseEvent::Button mouseButton, const WebCore::IntPoint& locationInViewport, CompletionHandler<void(std::optional<AutomationCommandError>)>&& completionHandler)
1450 {
1451     WebCore::IntPoint locationInView = WebCore::IntPoint(locationInViewport.x(), locationInViewport.y() + page.topContentInset());
1452     page.getWindowFrameWithCallback([this, protectedThis = makeRef(*this), completionHandler = WTFMove(completionHandler), page = makeRef(page), interaction, mouseButton, locationInView](WebCore::FloatRect windowFrame) mutable {
1453         auto clippedX = std::min(std::max(0.0f, (float)locationInView.x()), windowFrame.size().width());
1454         auto clippedY = std::min(std::max(0.0f, (float)locationInView.y()), windowFrame.size().height());
1455         if (clippedX != locationInView.x() || clippedY != locationInView.y()) {
1456             completionHandler(AUTOMATION_COMMAND_ERROR_WITH_NAME(TargetOutOfBounds));
1457             return;
1458         }
1459
1460         // Bridge the flushed callback to our command's completion handler.
1461         auto mouseEventsFlushedCallback = [completionHandler = WTFMove(completionHandler)](std::optional<AutomationCommandError> error) {
1462             completionHandler(error);
1463         };
1464
1465         auto& callbackInMap = m_pendingMouseEventsFlushedCallbacksPerPage.add(page->pageID(), nullptr).iterator->value;
1466         if (callbackInMap)
1467             callbackInMap(AUTOMATION_COMMAND_ERROR_WITH_NAME(Timeout));
1468         callbackInMap = WTFMove(mouseEventsFlushedCallback);
1469
1470         platformSimulateMouseInteraction(page, interaction, mouseButton, locationInView, (WebEvent::Modifiers)m_currentModifiers);
1471
1472         // If the event does not hit test anything in the window, then it may not have been delivered.
1473         if (callbackInMap && !page->isProcessingMouseEvents()) {
1474             auto callbackToCancel = m_pendingMouseEventsFlushedCallbacksPerPage.take(page->pageID());
1475             callbackToCancel(std::nullopt);
1476         }
1477
1478         // Otherwise, wait for mouseEventsFlushedCallback to run when all events are handled.
1479     });
1480 }
1481
1482 void WebAutomationSession::simulateKeyboardInteraction(WebPageProxy& page, KeyboardInteraction interaction, std::optional<VirtualKey> virtualKey, std::optional<CharKey> charKey, CompletionHandler<void(std::optional<AutomationCommandError>)>&& completionHandler)
1483 {
1484     // Bridge the flushed callback to our command's completion handler.
1485     auto keyboardEventsFlushedCallback = [completionHandler = WTFMove(completionHandler)](std::optional<AutomationCommandError> error) {
1486         completionHandler(error);
1487     };
1488
1489     auto& callbackInMap = m_pendingKeyboardEventsFlushedCallbacksPerPage.add(page.pageID(), nullptr).iterator->value;
1490     if (callbackInMap)
1491         callbackInMap(AUTOMATION_COMMAND_ERROR_WITH_NAME(Timeout));
1492     callbackInMap = WTFMove(keyboardEventsFlushedCallback);
1493
1494     platformSimulateKeyboardInteraction(page, interaction, virtualKey, charKey);
1495
1496     // Wait for keyboardEventsFlushedCallback to run when all events are handled.
1497 }
1498
1499 #if USE(APPKIT) || PLATFORM(GTK)
1500 static WebEvent::Modifiers protocolModifierToWebEventModifier(Inspector::Protocol::Automation::KeyModifier modifier)
1501 {
1502     switch (modifier) {
1503     case Inspector::Protocol::Automation::KeyModifier::Alt:
1504         return WebEvent::AltKey;
1505     case Inspector::Protocol::Automation::KeyModifier::Meta:
1506         return WebEvent::MetaKey;
1507     case Inspector::Protocol::Automation::KeyModifier::Control:
1508         return WebEvent::ControlKey;
1509     case Inspector::Protocol::Automation::KeyModifier::Shift:
1510         return WebEvent::ShiftKey;
1511     case Inspector::Protocol::Automation::KeyModifier::CapsLock:
1512         return WebEvent::CapsLockKey;
1513     }
1514
1515     RELEASE_ASSERT_NOT_REACHED();
1516 }
1517
1518 static WebMouseEvent::Button protocolMouseButtonToWebMouseEventButton(Inspector::Protocol::Automation::MouseButton button)
1519 {
1520     switch (button) {
1521     case Inspector::Protocol::Automation::MouseButton::None:
1522         return WebMouseEvent::NoButton;
1523     case Inspector::Protocol::Automation::MouseButton::Left:
1524         return WebMouseEvent::LeftButton;
1525     case Inspector::Protocol::Automation::MouseButton::Middle:
1526         return WebMouseEvent::MiddleButton;
1527     case Inspector::Protocol::Automation::MouseButton::Right:
1528         return WebMouseEvent::RightButton;
1529     }
1530
1531     RELEASE_ASSERT_NOT_REACHED();
1532 }
1533 #endif // USE(APPKIT) || PLATFORM(GTK)
1534
1535 void WebAutomationSession::performMouseInteraction(const String& handle, const JSON::Object& requestedPositionObject, const String& mouseButtonString, const String& mouseInteractionString, const JSON::Array& keyModifierStrings, Ref<PerformMouseInteractionCallback>&& callback)
1536 {
1537 #if !USE(APPKIT) && !PLATFORM(GTK)
1538     ASYNC_FAIL_WITH_PREDEFINED_ERROR(NotImplemented);
1539 #else
1540     WebPageProxy* page = webPageProxyForHandle(handle);
1541     if (!page)
1542         ASYNC_FAIL_WITH_PREDEFINED_ERROR(WindowNotFound);
1543
1544     float x;
1545     if (!requestedPositionObject.getDouble(WTF::ASCIILiteral("x"), x))
1546         ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(MissingParameter, "The parameter 'x' was not found.");
1547
1548     float y;
1549     if (!requestedPositionObject.getDouble(WTF::ASCIILiteral("y"), y))
1550         ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(MissingParameter, "The parameter 'y' was not found.");
1551
1552     WebEvent::Modifiers keyModifiers = (WebEvent::Modifiers)0;
1553     for (auto it = keyModifierStrings.begin(); it != keyModifierStrings.end(); ++it) {
1554         String modifierString;
1555         if (!it->get()->asString(modifierString))
1556             ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(InvalidParameter, "The parameter 'modifiers' is invalid.");
1557
1558         auto parsedModifier = Inspector::Protocol::AutomationHelpers::parseEnumValueFromString<Inspector::Protocol::Automation::KeyModifier>(modifierString);
1559         if (!parsedModifier)
1560             ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(InvalidParameter, "A modifier in the 'modifiers' array is invalid.");
1561         WebEvent::Modifiers enumValue = protocolModifierToWebEventModifier(parsedModifier.value());
1562         keyModifiers = (WebEvent::Modifiers)(enumValue | keyModifiers);
1563     }
1564
1565     page->getWindowFrameWithCallback([this, protectedThis = makeRef(*this), callback = WTFMove(callback), page = makeRef(*page), x, y, mouseInteractionString, mouseButtonString, keyModifiers](WebCore::FloatRect windowFrame) mutable {
1566
1567         x = std::min(std::max(0.0f, x), windowFrame.size().width());
1568         y = std::min(std::max(0.0f, y + page->topContentInset()), windowFrame.size().height());
1569
1570         WebCore::IntPoint positionInView = WebCore::IntPoint(static_cast<int>(x), static_cast<int>(y));
1571
1572         auto parsedInteraction = Inspector::Protocol::AutomationHelpers::parseEnumValueFromString<Inspector::Protocol::Automation::MouseInteraction>(mouseInteractionString);
1573         if (!parsedInteraction)
1574             ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(InvalidParameter, "The parameter 'interaction' is invalid.");
1575
1576         auto parsedButton = Inspector::Protocol::AutomationHelpers::parseEnumValueFromString<Inspector::Protocol::Automation::MouseButton>(mouseButtonString);
1577         if (!parsedButton)
1578             ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(InvalidParameter, "The parameter 'button' is invalid.");
1579
1580         auto mouseEventsFlushedCallback = [protectedThis = WTFMove(protectedThis), callback = WTFMove(callback), page = page.copyRef(), x, y](std::optional<AutomationCommandError> error) {
1581             if (error)
1582                 callback->sendFailure(error.value().toProtocolString());
1583             else {
1584                 callback->sendSuccess(Inspector::Protocol::Automation::Point::create()
1585                     .setX(x)
1586                     .setY(y - page->topContentInset())
1587                     .release());
1588             }
1589         };
1590
1591         auto& callbackInMap = m_pendingMouseEventsFlushedCallbacksPerPage.add(page->pageID(), nullptr).iterator->value;
1592         if (callbackInMap)
1593             callbackInMap(AUTOMATION_COMMAND_ERROR_WITH_NAME(Timeout));
1594         callbackInMap = WTFMove(mouseEventsFlushedCallback);
1595
1596         platformSimulateMouseInteraction(page, parsedInteraction.value(), protocolMouseButtonToWebMouseEventButton(parsedButton.value()), positionInView, keyModifiers);
1597
1598         // If the event location was previously clipped and does not hit test anything in the window, then it will not be processed.
1599         // For compatibility with pre-W3C driver implementations, don't make this a hard error; just do nothing silently.
1600         // In W3C-only code paths, we can reject any pointer actions whose coordinates are outside the viewport rect.
1601         if (callbackInMap && !page->isProcessingMouseEvents()) {
1602             auto callbackToCancel = m_pendingMouseEventsFlushedCallbacksPerPage.take(page->pageID());
1603             callbackToCancel(std::nullopt);
1604         }
1605     });
1606 #endif // USE(APPKIT) || PLATFORM(GTK)
1607 }
1608
1609 void WebAutomationSession::performKeyboardInteractions(const String& handle, const JSON::Array& interactions, Ref<PerformKeyboardInteractionsCallback>&& callback)
1610 {
1611 #if !PLATFORM(COCOA) && !PLATFORM(GTK)
1612     ASYNC_FAIL_WITH_PREDEFINED_ERROR(NotImplemented);
1613 #else
1614     WebPageProxy* page = webPageProxyForHandle(handle);
1615     if (!page)
1616         ASYNC_FAIL_WITH_PREDEFINED_ERROR(WindowNotFound);
1617
1618     if (!interactions.length())
1619         ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(InvalidParameter, "The parameter 'interactions' was not found or empty.");
1620
1621     // Validate all of the parameters before performing any interactions with the browsing context under test.
1622     Vector<WTF::Function<void()>> actionsToPerform;
1623     actionsToPerform.reserveCapacity(interactions.length());
1624
1625     for (auto interaction : interactions) {
1626         RefPtr<JSON::Object> interactionObject;
1627         if (!interaction->asObject(interactionObject))
1628             ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(InvalidParameter, "An interaction in the 'interactions' parameter was invalid.");
1629
1630         String interactionTypeString;
1631         if (!interactionObject->getString(ASCIILiteral("type"), interactionTypeString))
1632             ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(InvalidParameter, "An interaction in the 'interactions' parameter is missing the 'type' key.");
1633         auto interactionType = Inspector::Protocol::AutomationHelpers::parseEnumValueFromString<Inspector::Protocol::Automation::KeyboardInteractionType>(interactionTypeString);
1634         if (!interactionType)
1635             ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(InvalidParameter, "An interaction in the 'interactions' parameter has an invalid 'type' key.");
1636
1637         String virtualKeyString;
1638         bool foundVirtualKey = interactionObject->getString(ASCIILiteral("key"), virtualKeyString);
1639         if (foundVirtualKey) {
1640             auto virtualKey = Inspector::Protocol::AutomationHelpers::parseEnumValueFromString<Inspector::Protocol::Automation::VirtualKey>(virtualKeyString);
1641             if (!virtualKey)
1642                 ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(InvalidParameter, "An interaction in the 'interactions' parameter has an invalid 'key' value.");
1643
1644             actionsToPerform.uncheckedAppend([this, page, interactionType, virtualKey] {
1645                 platformSimulateKeyboardInteraction(*page, interactionType.value(), virtualKey, std::nullopt);
1646             });
1647         }
1648
1649         String keySequence;
1650         bool foundKeySequence = interactionObject->getString(ASCIILiteral("text"), keySequence);
1651         if (foundKeySequence) {
1652             switch (interactionType.value()) {
1653             case Inspector::Protocol::Automation::KeyboardInteractionType::KeyPress:
1654             case Inspector::Protocol::Automation::KeyboardInteractionType::KeyRelease:
1655                 // 'KeyPress' and 'KeyRelease' are meant for a virtual key and are not supported for a string (sequence of codepoints).
1656                 ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(InvalidParameter, "An interaction in the 'interactions' parameter has an invalid 'key' value.");
1657
1658             case Inspector::Protocol::Automation::KeyboardInteractionType::InsertByKey:
1659                 actionsToPerform.uncheckedAppend([this, page, keySequence] {
1660                     platformSimulateKeySequence(*page, keySequence);
1661                 });
1662                 break;
1663             }
1664         }
1665
1666         if (!foundVirtualKey && !foundKeySequence)
1667             ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(MissingParameter, "An interaction in the 'interactions' parameter is missing both 'key' and 'text'. One must be provided.");
1668     }
1669
1670     ASSERT(actionsToPerform.size());
1671     if (!actionsToPerform.size())
1672         ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(InternalError, "No actions to perform.");
1673
1674     auto keyboardEventsFlushedCallback = [protectedThis = makeRef(*this), callback = WTFMove(callback), page = makeRef(*page)](std::optional<AutomationCommandError> error) {
1675         if (error)
1676             callback->sendFailure(error.value().toProtocolString());
1677         else
1678             callback->sendSuccess();
1679     };
1680
1681     auto& callbackInMap = m_pendingKeyboardEventsFlushedCallbacksPerPage.add(page->pageID(), nullptr).iterator->value;
1682     if (callbackInMap)
1683         callbackInMap(AUTOMATION_COMMAND_ERROR_WITH_NAME(Timeout));
1684     callbackInMap = WTFMove(keyboardEventsFlushedCallback);
1685
1686     for (auto& action : actionsToPerform)
1687         action();
1688 #endif // PLATFORM(COCOA) || PLATFORM(GTK)
1689 }
1690
1691 #if USE(APPKIT) || PLATFORM(GTK)
1692 static SimulatedInputSourceType simulatedInputSourceTypeFromProtocolSourceType(Inspector::Protocol::Automation::InputSourceType protocolType)
1693 {
1694     switch (protocolType) {
1695     case Inspector::Protocol::Automation::InputSourceType::Null:
1696         return SimulatedInputSourceType::Null;
1697     case Inspector::Protocol::Automation::InputSourceType::Keyboard:
1698         return SimulatedInputSourceType::Keyboard;
1699     case Inspector::Protocol::Automation::InputSourceType::Mouse:
1700         return SimulatedInputSourceType::Mouse;
1701     case Inspector::Protocol::Automation::InputSourceType::Touch:
1702         return SimulatedInputSourceType::Touch;
1703     }
1704
1705     RELEASE_ASSERT_NOT_REACHED();
1706 }
1707 #endif // USE(APPKIT) || PLATFORM(GTK)
1708
1709 void WebAutomationSession::performInteractionSequence(const String& handle, const String* optionalFrameHandle, const JSON::Array& inputSources, const JSON::Array& steps, Ref<WebAutomationSession::PerformInteractionSequenceCallback>&& callback)
1710 {
1711     // This command implements WebKit support for §17.5 Perform Actions.
1712
1713 #if !USE(APPKIT) && !PLATFORM(GTK)
1714     ASYNC_FAIL_WITH_PREDEFINED_ERROR(NotImplemented);
1715 #else
1716     WebPageProxy* page = webPageProxyForHandle(handle);
1717     if (!page)
1718         ASYNC_FAIL_WITH_PREDEFINED_ERROR(WindowNotFound);
1719
1720     auto frameID = webFrameIDForHandle(optionalFrameHandle ? *optionalFrameHandle : emptyString());
1721     if (!frameID)
1722         ASYNC_FAIL_WITH_PREDEFINED_ERROR(FrameNotFound);
1723
1724     HashMap<String, Ref<SimulatedInputSource>> sourceIdToInputSourceMap;
1725     HashMap<SimulatedInputSourceType, String, WTF::IntHash<SimulatedInputSourceType>, WTF::StrongEnumHashTraits<SimulatedInputSourceType>> typeToSourceIdMap;
1726
1727     // Parse and validate Automation protocol arguments. By this point, the driver has
1728     // already performed the steps in §17.3 Processing Actions Requests.
1729     if (!inputSources.length())
1730         ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(InvalidParameter, "The parameter 'inputSources' was not found or empty.");
1731
1732     for (auto inputSource : inputSources) {
1733         RefPtr<JSON::Object> inputSourceObject;
1734         if (!inputSource->asObject(inputSourceObject))
1735             ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(InvalidParameter, "An input source in the 'inputSources' parameter was invalid.");
1736         
1737         String sourceId;
1738         if (!inputSourceObject->getString(ASCIILiteral("sourceId"), sourceId))
1739             ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(InvalidParameter, "An input source in the 'inputSources' parameter is missing a 'sourceId'.");
1740
1741         String sourceType;
1742         if (!inputSourceObject->getString(ASCIILiteral("sourceType"), sourceType))
1743             ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(InvalidParameter, "An input source in the 'inputSources' parameter is missing a 'sourceType'.");
1744
1745         auto parsedInputSourceType = Inspector::Protocol::AutomationHelpers::parseEnumValueFromString<Inspector::Protocol::Automation::InputSourceType>(sourceType);
1746         if (!parsedInputSourceType)
1747             ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(InvalidParameter, "An input source in the 'inputSources' parameter has an invalid 'sourceType'.");
1748
1749         SimulatedInputSourceType inputSourceType = simulatedInputSourceTypeFromProtocolSourceType(*parsedInputSourceType);
1750         if (inputSourceType == SimulatedInputSourceType::Touch)
1751             ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(NotImplemented, "Touch input sources are not yet supported.");
1752
1753         if (typeToSourceIdMap.contains(inputSourceType))
1754             ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(InvalidParameter, "Two input sources with the same type were specified.");
1755         if (sourceIdToInputSourceMap.contains(sourceId))
1756             ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(InvalidParameter, "Two input sources with the same sourceId were specified.");
1757
1758         typeToSourceIdMap.add(inputSourceType, sourceId);
1759         sourceIdToInputSourceMap.add(sourceId, *inputSourceForType(inputSourceType));
1760     }
1761
1762     Vector<SimulatedInputKeyFrame> keyFrames;
1763
1764     if (!steps.length())
1765         ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(InvalidParameter, "The parameter 'steps' was not found or empty.");
1766
1767     for (auto step : steps) {
1768         RefPtr<JSON::Object> stepObject;
1769         if (!step->asObject(stepObject))
1770             ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(InvalidParameter, "A step in the 'steps' parameter was not an object.");
1771
1772         RefPtr<JSON::Array> stepStates;
1773         if (!stepObject->getArray(ASCIILiteral("states"), stepStates))
1774             ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(InvalidParameter, "A step is missing the 'states' property.");
1775
1776         Vector<SimulatedInputKeyFrame::StateEntry> entries;
1777         entries.reserveCapacity(stepStates->length());
1778
1779         for (auto state : *stepStates) {
1780             RefPtr<JSON::Object> stateObject;
1781             if (!state->asObject(stateObject))
1782                 ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(InvalidParameter, "Encountered a non-object step state.");
1783
1784             String sourceId;
1785             if (!stateObject->getString(ASCIILiteral("sourceId"), sourceId))
1786                 ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(InvalidParameter, "Step state lacks required 'sourceId' property.");
1787
1788             if (!sourceIdToInputSourceMap.contains(sourceId))
1789                 ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(InvalidParameter, "Unknown 'sourceId' specified.");
1790
1791             SimulatedInputSource& inputSource = *sourceIdToInputSourceMap.get(sourceId);
1792             SimulatedInputSourceState sourceState { };
1793
1794             String pressedCharKeyString;
1795             if (stateObject->getString(ASCIILiteral("pressedCharKey"), pressedCharKeyString))
1796                 sourceState.pressedCharKey = pressedCharKeyString.characterAt(0);
1797
1798             String pressedVirtualKeyString;
1799             if (stateObject->getString(ASCIILiteral("pressedVirtualKey"), pressedVirtualKeyString))
1800                 sourceState.pressedVirtualKey = Inspector::Protocol::AutomationHelpers::parseEnumValueFromString<Inspector::Protocol::Automation::VirtualKey>(pressedVirtualKeyString);
1801
1802             String pressedButtonString;
1803             if (stateObject->getString(ASCIILiteral("pressedButton"), pressedButtonString)) {
1804                 auto protocolButton = Inspector::Protocol::AutomationHelpers::parseEnumValueFromString<Inspector::Protocol::Automation::MouseButton>(pressedButtonString);
1805                 sourceState.pressedMouseButton = protocolMouseButtonToWebMouseEventButton(protocolButton.value_or(Inspector::Protocol::Automation::MouseButton::None));
1806             }
1807
1808             String originString;
1809             if (stateObject->getString(ASCIILiteral("origin"), originString))
1810                 sourceState.origin = Inspector::Protocol::AutomationHelpers::parseEnumValueFromString<Inspector::Protocol::Automation::MouseMoveOrigin>(originString);
1811
1812             if (sourceState.origin && sourceState.origin.value() == Inspector::Protocol::Automation::MouseMoveOrigin::Element) {
1813                 String nodeHandleString;
1814                 if (!stateObject->getString(ASCIILiteral("nodeHandle"), nodeHandleString))
1815                     ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(InvalidParameter, "Node handle not provided for 'Element' origin");
1816                 sourceState.nodeHandle = nodeHandleString;
1817             }
1818
1819             RefPtr<JSON::Object> locationObject;
1820             if (stateObject->getObject(ASCIILiteral("location"), locationObject)) {
1821                 int x, y;
1822                 if (locationObject->getInteger(ASCIILiteral("x"), x) && locationObject->getInteger(ASCIILiteral("y"), y))
1823                     sourceState.location = WebCore::IntPoint(x, y);
1824             }
1825
1826             int parsedDuration;
1827             if (stateObject->getInteger(ASCIILiteral("duration"), parsedDuration))
1828                 sourceState.duration = Seconds::fromMilliseconds(parsedDuration);
1829
1830             entries.uncheckedAppend(std::pair<SimulatedInputSource&, SimulatedInputSourceState> { inputSource, sourceState });
1831         }
1832         
1833         keyFrames.append(SimulatedInputKeyFrame(WTFMove(entries)));
1834     }
1835
1836     SimulatedInputDispatcher& inputDispatcher = inputDispatcherForPage(*page);
1837     if (inputDispatcher.isActive()) {
1838         ASSERT_NOT_REACHED();
1839         ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(InternalError, "A previous interaction is still underway.");
1840     }
1841
1842     // Delegate the rest of §17.4 Dispatching Actions to the dispatcher.
1843     inputDispatcher.run(frameID.value(), WTFMove(keyFrames), m_inputSources, [protectedThis = makeRef(*this), callback = WTFMove(callback)](std::optional<AutomationCommandError> error) {
1844         if (error)
1845             callback->sendFailure(error.value().toProtocolString());
1846         else
1847             callback->sendSuccess();
1848     });
1849 #endif // PLATFORM(COCOA) || PLATFORM(GTK)
1850 }
1851
1852 void WebAutomationSession::cancelInteractionSequence(const String& handle, const String* optionalFrameHandle, Ref<CancelInteractionSequenceCallback>&& callback)
1853 {
1854     // This command implements WebKit support for §17.6 Release Actions.
1855
1856 #if !USE(APPKIT) && !PLATFORM(GTK)
1857     ASYNC_FAIL_WITH_PREDEFINED_ERROR(NotImplemented);
1858 #else
1859     WebPageProxy* page = webPageProxyForHandle(handle);
1860     if (!page)
1861         ASYNC_FAIL_WITH_PREDEFINED_ERROR(WindowNotFound);
1862
1863     auto frameID = webFrameIDForHandle(optionalFrameHandle ? *optionalFrameHandle : emptyString());
1864     if (!frameID)
1865         ASYNC_FAIL_WITH_PREDEFINED_ERROR(FrameNotFound);
1866
1867     Vector<SimulatedInputKeyFrame> keyFrames({ SimulatedInputKeyFrame::keyFrameToResetInputSources(m_inputSources) });
1868     SimulatedInputDispatcher& inputDispatcher = inputDispatcherForPage(*page);
1869     inputDispatcher.cancel();
1870     
1871     inputDispatcher.run(frameID.value(), WTFMove(keyFrames), m_inputSources, [protectedThis = makeRef(*this), callback = WTFMove(callback)](std::optional<AutomationCommandError> error) {
1872         if (error)
1873             callback->sendFailure(error.value().toProtocolString());
1874         else
1875             callback->sendSuccess();
1876     });
1877 #endif // PLATFORM(COCOA) || PLATFORM(GTK)
1878 }
1879
1880 void WebAutomationSession::takeScreenshot(const String& handle, const String* optionalFrameHandle, const String* optionalNodeHandle, const bool* optionalScrollIntoViewIfNeeded, const bool* optionalClipToViewport, Ref<TakeScreenshotCallback>&& callback)
1881 {
1882     WebPageProxy* page = webPageProxyForHandle(handle);
1883     if (!page)
1884         ASYNC_FAIL_WITH_PREDEFINED_ERROR(WindowNotFound);
1885
1886     std::optional<uint64_t> frameID = webFrameIDForHandle(optionalFrameHandle ? *optionalFrameHandle : emptyString());
1887     if (!frameID)
1888         ASYNC_FAIL_WITH_PREDEFINED_ERROR(FrameNotFound);
1889
1890     bool scrollIntoViewIfNeeded = optionalScrollIntoViewIfNeeded ? *optionalScrollIntoViewIfNeeded : false;
1891     String nodeHandle = optionalNodeHandle ? *optionalNodeHandle : emptyString();
1892     bool clipToViewport = optionalClipToViewport ? *optionalClipToViewport : false;
1893
1894     uint64_t callbackID = m_nextScreenshotCallbackID++;
1895     m_screenshotCallbacks.set(callbackID, WTFMove(callback));
1896
1897     page->process().send(Messages::WebAutomationSessionProxy::TakeScreenshot(page->pageID(), frameID.value(), nodeHandle, scrollIntoViewIfNeeded, clipToViewport, callbackID), 0);
1898 }
1899
1900 void WebAutomationSession::didTakeScreenshot(uint64_t callbackID, const ShareableBitmap::Handle& imageDataHandle, const String& errorType)
1901 {
1902     auto callback = m_screenshotCallbacks.take(callbackID);
1903     if (!callback)
1904         return;
1905
1906     if (!errorType.isEmpty()) {
1907         callback->sendFailure(STRING_FOR_PREDEFINED_ERROR_MESSAGE(errorType));
1908         return;
1909     }
1910
1911     std::optional<String> base64EncodedData = platformGetBase64EncodedPNGData(imageDataHandle);
1912     if (!base64EncodedData)
1913         ASYNC_FAIL_WITH_PREDEFINED_ERROR(InternalError);
1914
1915     callback->sendSuccess(base64EncodedData.value());
1916 }
1917
1918 // Platform-dependent Implementation Stubs.
1919
1920 #if !PLATFORM(MAC) && !PLATFORM(GTK)
1921 void WebAutomationSession::platformSimulateMouseInteraction(WebPageProxy&, MouseInteraction, WebMouseEvent::Button, const WebCore::IntPoint&, WebEvent::Modifiers)
1922 {
1923 }
1924 #endif // !PLATFORM(MAC) && !PLATFORM(GTK)
1925
1926 #if !PLATFORM(COCOA) && !PLATFORM(GTK)
1927
1928
1929 void WebAutomationSession::platformSimulateKeyboardInteraction(WebPageProxy&, KeyboardInteraction, std::optional<VirtualKey>, std::optional<CharKey>)
1930 {
1931 }
1932 #endif // !PLATFORM(COCOA) && !PLATFORM(GTK)
1933
1934 #if !PLATFORM(COCOA) && !USE(CAIRO)
1935 std::optional<String> WebAutomationSession::platformGetBase64EncodedPNGData(const ShareableBitmap::Handle&)
1936 {
1937     return std::nullopt;
1938 }
1939 #endif // !PLATFORM(COCOA) && !USE(CAIRO)
1940
1941 } // namespace WebKit