Move URL from WebCore to WTF
[WebKit-https.git] / Source / WebKit / UIProcess / Automation / WebAutomationSession.cpp
1
2 /*
3  * Copyright (C) 2016, 2017 Apple Inc. All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions
7  * are met:
8  * 1. Redistributions of source code must retain the above copyright
9  *    notice, this list of conditions and the following disclaimer.
10  * 2. Redistributions in binary form must reproduce the above copyright
11  *    notice, this list of conditions and the following disclaimer in the
12  *    documentation and/or other materials provided with the distribution.
13  *
14  * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
15  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
16  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
17  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
18  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
19  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
20  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
21  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
22  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
23  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
24  * THE POSSIBILITY OF SUCH DAMAGE.
25  */
26
27 #include "config.h"
28 #include "WebAutomationSession.h"
29
30 #include "APIArray.h"
31 #include "APIAutomationSessionClient.h"
32 #include "APINavigation.h"
33 #include "APIOpenPanelParameters.h"
34 #include "AutomationProtocolObjects.h"
35 #include "CoordinateSystem.h"
36 #include "WebAutomationSessionMacros.h"
37 #include "WebAutomationSessionMessages.h"
38 #include "WebAutomationSessionProxyMessages.h"
39 #include "WebCookieManagerProxy.h"
40 #include "WebFullScreenManagerProxy.h"
41 #include "WebInspectorProxy.h"
42 #include "WebOpenPanelResultListenerProxy.h"
43 #include "WebProcessPool.h"
44 #include <JavaScriptCore/InspectorBackendDispatcher.h>
45 #include <JavaScriptCore/InspectorFrontendRouter.h>
46 #include <WebCore/MIMETypeRegistry.h>
47 #include <algorithm>
48 #include <wtf/HashMap.h>
49 #include <wtf/Optional.h>
50 #include <wtf/URL.h>
51 #include <wtf/UUID.h>
52 #include <wtf/text/StringConcatenate.h>
53
54 using namespace Inspector;
55
56 namespace WebKit {
57
58 String AutomationCommandError::toProtocolString()
59 {
60     String protocolErrorName = Inspector::Protocol::AutomationHelpers::getEnumConstantValue(type);
61     if (!message.has_value())
62         return protocolErrorName;
63
64     return makeString(protocolErrorName, errorNameAndDetailsSeparator, message.value());
65 }
66     
67 // §8. Sessions
68 // https://www.w3.org/TR/webdriver/#dfn-session-page-load-timeout
69 static const Seconds defaultPageLoadTimeout = 300_s;
70 // https://www.w3.org/TR/webdriver/#dfn-page-loading-strategy
71 static const Inspector::Protocol::Automation::PageLoadStrategy defaultPageLoadStrategy = Inspector::Protocol::Automation::PageLoadStrategy::Normal;
72
73 WebAutomationSession::WebAutomationSession()
74     : m_client(std::make_unique<API::AutomationSessionClient>())
75     , m_frontendRouter(FrontendRouter::create())
76     , m_backendDispatcher(BackendDispatcher::create(m_frontendRouter.copyRef()))
77     , m_domainDispatcher(AutomationBackendDispatcher::create(m_backendDispatcher, this))
78     , m_domainNotifier(std::make_unique<AutomationFrontendDispatcher>(m_frontendRouter))
79     , m_loadTimer(RunLoop::main(), this, &WebAutomationSession::loadTimerFired)
80 {
81     // Set up canonical input sources to be used for 'performInteractionSequence' and 'cancelInteractionSequence'.
82     m_inputSources.add(SimulatedInputSource::create(SimulatedInputSourceType::Mouse));
83     m_inputSources.add(SimulatedInputSource::create(SimulatedInputSourceType::Keyboard));
84     m_inputSources.add(SimulatedInputSource::create(SimulatedInputSourceType::Null));
85 }
86
87 WebAutomationSession::~WebAutomationSession()
88 {
89     ASSERT(!m_client);
90     ASSERT(!m_processPool);
91 }
92
93 void WebAutomationSession::setClient(std::unique_ptr<API::AutomationSessionClient>&& client)
94 {
95     m_client = WTFMove(client);
96 }
97
98 void WebAutomationSession::setProcessPool(WebKit::WebProcessPool* processPool)
99 {
100     if (m_processPool)
101         m_processPool->removeMessageReceiver(Messages::WebAutomationSession::messageReceiverName());
102
103     m_processPool = processPool;
104
105     if (m_processPool)
106         m_processPool->addMessageReceiver(Messages::WebAutomationSession::messageReceiverName(), *this);
107 }
108
109 // NOTE: this class could be split at some point to support local and remote automation sessions.
110 // For now, it only works with a remote automation driver over a RemoteInspector connection.
111
112 #if ENABLE(REMOTE_INSPECTOR)
113
114 // Inspector::RemoteAutomationTarget API
115
116 void WebAutomationSession::dispatchMessageFromRemote(const String& message)
117 {
118     m_backendDispatcher->dispatch(message);
119 }
120
121 void WebAutomationSession::connect(Inspector::FrontendChannel& channel, bool isAutomaticConnection, bool immediatelyPause)
122 {
123     UNUSED_PARAM(isAutomaticConnection);
124     UNUSED_PARAM(immediatelyPause);
125
126     m_remoteChannel = &channel;
127     m_frontendRouter->connectFrontend(channel);
128
129     setIsPaired(true);
130 }
131
132 void WebAutomationSession::disconnect(Inspector::FrontendChannel& channel)
133 {
134     ASSERT(&channel == m_remoteChannel);
135     terminate();
136 }
137
138 #endif // ENABLE(REMOTE_INSPECTOR)
139
140 void WebAutomationSession::terminate()
141 {
142     for (auto& identifier : copyToVector(m_pendingKeyboardEventsFlushedCallbacksPerPage.keys())) {
143         auto callback = m_pendingKeyboardEventsFlushedCallbacksPerPage.take(identifier);
144         callback(AUTOMATION_COMMAND_ERROR_WITH_NAME(InternalError));
145     }
146
147     for (auto& identifier : copyToVector(m_pendingMouseEventsFlushedCallbacksPerPage.keys())) {
148         auto callback = m_pendingMouseEventsFlushedCallbacksPerPage.take(identifier);
149         callback(AUTOMATION_COMMAND_ERROR_WITH_NAME(InternalError));
150     }
151
152 #if ENABLE(REMOTE_INSPECTOR)
153     if (Inspector::FrontendChannel* channel = m_remoteChannel) {
154         m_remoteChannel = nullptr;
155         m_frontendRouter->disconnectFrontend(*channel);
156     }
157
158     setIsPaired(false);
159 #endif
160
161     if (m_client)
162         m_client->didDisconnectFromRemote(*this);
163 }
164
165 WebPageProxy* WebAutomationSession::webPageProxyForHandle(const String& handle)
166 {
167     auto iter = m_handleWebPageMap.find(handle);
168     if (iter == m_handleWebPageMap.end())
169         return nullptr;
170     return WebProcessProxy::webPage(iter->value);
171 }
172
173 String WebAutomationSession::handleForWebPageProxy(const WebPageProxy& webPageProxy)
174 {
175     auto iter = m_webPageHandleMap.find(webPageProxy.pageID());
176     if (iter != m_webPageHandleMap.end())
177         return iter->value;
178
179     String handle = "page-" + createCanonicalUUIDString().convertToASCIIUppercase();
180
181     auto firstAddResult = m_webPageHandleMap.add(webPageProxy.pageID(), handle);
182     RELEASE_ASSERT(firstAddResult.isNewEntry);
183
184     auto secondAddResult = m_handleWebPageMap.add(handle, webPageProxy.pageID());
185     RELEASE_ASSERT(secondAddResult.isNewEntry);
186
187     return handle;
188 }
189
190 std::optional<uint64_t> WebAutomationSession::webFrameIDForHandle(const String& handle)
191 {
192     if (handle.isEmpty())
193         return 0;
194
195     auto iter = m_handleWebFrameMap.find(handle);
196     if (iter == m_handleWebFrameMap.end())
197         return std::nullopt;
198
199     return iter->value;
200 }
201
202 String WebAutomationSession::handleForWebFrameID(uint64_t frameID)
203 {
204     if (!frameID)
205         return emptyString();
206
207     for (auto& process : m_processPool->processes()) {
208         if (WebFrameProxy* frame = process->webFrame(frameID)) {
209             if (frame->isMainFrame())
210                 return emptyString();
211             break;
212         }
213     }
214
215     auto iter = m_webFrameHandleMap.find(frameID);
216     if (iter != m_webFrameHandleMap.end())
217         return iter->value;
218
219     String handle = "frame-" + createCanonicalUUIDString().convertToASCIIUppercase();
220
221     auto firstAddResult = m_webFrameHandleMap.add(frameID, handle);
222     RELEASE_ASSERT(firstAddResult.isNewEntry);
223
224     auto secondAddResult = m_handleWebFrameMap.add(handle, frameID);
225     RELEASE_ASSERT(secondAddResult.isNewEntry);
226
227     return handle;
228 }
229
230 String WebAutomationSession::handleForWebFrameProxy(const WebFrameProxy& webFrameProxy)
231 {
232     return handleForWebFrameID(webFrameProxy.frameID());
233 }
234
235 Ref<Inspector::Protocol::Automation::BrowsingContext> WebAutomationSession::buildBrowsingContextForPage(WebPageProxy& page, WebCore::FloatRect windowFrame)
236 {
237     auto originObject = Inspector::Protocol::Automation::Point::create()
238         .setX(windowFrame.x())
239         .setY(windowFrame.y())
240         .release();
241
242     auto sizeObject = Inspector::Protocol::Automation::Size::create()
243         .setWidth(windowFrame.width())
244         .setHeight(windowFrame.height())
245         .release();
246
247     bool isActive = page.isViewVisible() && page.isViewFocused() && page.isViewWindowActive();
248     String handle = handleForWebPageProxy(page);
249
250     return Inspector::Protocol::Automation::BrowsingContext::create()
251         .setHandle(handle)
252         .setActive(isActive)
253         .setUrl(page.pageLoadState().activeURL())
254         .setWindowOrigin(WTFMove(originObject))
255         .setWindowSize(WTFMove(sizeObject))
256         .release();
257 }
258
259 // Platform-independent Commands.
260
261 void WebAutomationSession::getNextContext(Ref<WebAutomationSession>&& protectedThis, Vector<Ref<WebPageProxy>>&& pages, Ref<JSON::ArrayOf<Inspector::Protocol::Automation::BrowsingContext>> contexts, Ref<WebAutomationSession::GetBrowsingContextsCallback>&& callback)
262 {
263     if (pages.isEmpty()) {
264         callback->sendSuccess(WTFMove(contexts));
265         return;
266     }
267     auto page = pages.takeLast();
268     auto& webPageProxy = page.get();
269     webPageProxy.getWindowFrameWithCallback([this, protectedThis = WTFMove(protectedThis), callback = WTFMove(callback), pages = WTFMove(pages), contexts = WTFMove(contexts), page = WTFMove(page)](WebCore::FloatRect windowFrame) mutable {
270         contexts->addItem(protectedThis->buildBrowsingContextForPage(page.get(), windowFrame));
271         getNextContext(WTFMove(protectedThis), WTFMove(pages), WTFMove(contexts), WTFMove(callback));
272     });
273 }
274     
275 void WebAutomationSession::getBrowsingContexts(Ref<GetBrowsingContextsCallback>&& callback)
276 {
277     Vector<Ref<WebPageProxy>> pages;
278     for (auto& process : m_processPool->processes()) {
279         for (auto* page : process->pages()) {
280             ASSERT(page);
281             if (!page->isControlledByAutomation())
282                 continue;
283             pages.append(*page);
284         }
285     }
286     
287     getNextContext(makeRef(*this), WTFMove(pages), JSON::ArrayOf<Inspector::Protocol::Automation::BrowsingContext>::create(), WTFMove(callback));
288 }
289
290 void WebAutomationSession::getBrowsingContext(const String& handle, Ref<GetBrowsingContextCallback>&& callback)
291 {
292     WebPageProxy* page = webPageProxyForHandle(handle);
293     if (!page)
294         ASYNC_FAIL_WITH_PREDEFINED_ERROR(WindowNotFound);
295
296     page->getWindowFrameWithCallback([protectedThis = makeRef(*this), page = makeRef(*page), callback = WTFMove(callback)](WebCore::FloatRect windowFrame) mutable {
297         callback->sendSuccess(protectedThis->buildBrowsingContextForPage(page.get(), windowFrame));
298     });
299 }
300
301 void WebAutomationSession::createBrowsingContext(const bool* preferNewTab, Ref<CreateBrowsingContextCallback>&& callback)
302 {
303     ASSERT(m_client);
304     if (!m_client)
305         ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(InternalError, "The remote session could not request a new browsing context.");
306
307     uint16_t options = 0;
308     if (preferNewTab && *preferNewTab)
309         options |= API::AutomationSessionBrowsingContextOptionsPreferNewTab;
310
311     m_client->requestNewPageWithOptions(*this, static_cast<API::AutomationSessionBrowsingContextOptions>(options), [protectedThis = makeRef(*this), callback = WTFMove(callback)](WebPageProxy* page) {
312         if (page)
313             callback->sendSuccess(protectedThis->handleForWebPageProxy(*page));
314         else
315             ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(InternalError, "The remote session failed to create a new browsing context.");
316     });
317 }
318
319 void WebAutomationSession::closeBrowsingContext(Inspector::ErrorString& errorString, const String& handle)
320 {
321     WebPageProxy* page = webPageProxyForHandle(handle);
322     if (!page)
323         SYNC_FAIL_WITH_PREDEFINED_ERROR(WindowNotFound);
324
325     page->closePage(false);
326 }
327
328 void WebAutomationSession::switchToBrowsingContext(const String& browsingContextHandle, const String* optionalFrameHandle, Ref<SwitchToBrowsingContextCallback>&& callback)
329 {
330     WebPageProxy* page = webPageProxyForHandle(browsingContextHandle);
331     if (!page)
332         ASYNC_FAIL_WITH_PREDEFINED_ERROR(WindowNotFound);
333
334     std::optional<uint64_t> frameID = webFrameIDForHandle(optionalFrameHandle ? *optionalFrameHandle : emptyString());
335     if (!frameID)
336         ASYNC_FAIL_WITH_PREDEFINED_ERROR(FrameNotFound);
337
338
339     m_client->requestSwitchToPage(*this, *page, [frameID, page = makeRef(*page), callback = WTFMove(callback)]() {
340         page->setFocus(true);
341         page->process().send(Messages::WebAutomationSessionProxy::FocusFrame(page->pageID(), frameID.value()), 0);
342
343         callback->sendSuccess();
344     });
345 }
346
347 void WebAutomationSession::setWindowFrameOfBrowsingContext(const String& handle, const JSON::Object* optionalOriginObject, const JSON::Object* optionalSizeObject, Ref<SetWindowFrameOfBrowsingContextCallback>&& callback)
348 {
349     std::optional<float> x;
350     std::optional<float> y;
351     if (optionalOriginObject) {
352         if (!(x = optionalOriginObject->getNumber<float>("x"_s)))
353             ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(MissingParameter, "The 'x' parameter was not found or invalid.");
354
355         if (!(y = optionalOriginObject->getNumber<float>("y"_s)))
356             ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(MissingParameter, "The 'y' parameter was not found or invalid.");
357
358         if (x.value() < 0)
359             ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(InvalidParameter, "The 'x' parameter had an invalid value.");
360
361         if (y.value() < 0)
362             ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(InvalidParameter, "The 'y' parameter had an invalid value.");
363     }
364
365     std::optional<float> width;
366     std::optional<float> height;
367     if (optionalSizeObject) {
368         if (!(width = optionalSizeObject->getNumber<float>("width"_s)))
369             ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(MissingParameter, "The 'width' parameter was not found or invalid.");
370
371         if (!(height = optionalSizeObject->getNumber<float>("height"_s)))
372             ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(MissingParameter, "The 'height' parameter was not found or invalid.");
373
374         if (width.value() < 0)
375             ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(InvalidParameter, "The 'width' parameter had an invalid value.");
376
377         if (height.value() < 0)
378             ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(InvalidParameter, "The 'height' parameter had an invalid value.");
379     }
380
381     WebPageProxy* page = webPageProxyForHandle(handle);
382     if (!page)
383         ASYNC_FAIL_WITH_PREDEFINED_ERROR(WindowNotFound);
384
385     exitFullscreenWindowForPage(*page, [this, protectedThis = makeRef(*this), callback = WTFMove(callback), page = makeRefPtr(page), width, height, x, y]() mutable {
386         auto& webPage = *page;
387         this->restoreWindowForPage(webPage, [callback = WTFMove(callback), page = WTFMove(page), width, height, x, y]() mutable {
388             auto& webPage = *page;
389             webPage.getWindowFrameWithCallback([callback = WTFMove(callback), page = WTFMove(page), width, height, x, y](WebCore::FloatRect originalFrame) mutable {
390                 WebCore::FloatRect newFrame = WebCore::FloatRect(WebCore::FloatPoint(x.value_or(originalFrame.location().x()), y.value_or(originalFrame.location().y())), WebCore::FloatSize(width.value_or(originalFrame.size().width()), height.value_or(originalFrame.size().height())));
391                 if (newFrame != originalFrame)
392                     page->setWindowFrame(newFrame);
393                 
394                 callback->sendSuccess();
395             });
396         });
397     });
398 }
399
400 static std::optional<Inspector::Protocol::Automation::PageLoadStrategy> pageLoadStrategyFromStringParameter(const String* optionalPageLoadStrategyString)
401 {
402     if (!optionalPageLoadStrategyString)
403         return defaultPageLoadStrategy;
404
405     auto parsedPageLoadStrategy = Inspector::Protocol::AutomationHelpers::parseEnumValueFromString<Inspector::Protocol::Automation::PageLoadStrategy>(*optionalPageLoadStrategyString);
406     if (!parsedPageLoadStrategy)
407         return std::nullopt;
408     return parsedPageLoadStrategy;
409 }
410
411 void WebAutomationSession::waitForNavigationToComplete(const String& browsingContextHandle, const String* optionalFrameHandle, const String* optionalPageLoadStrategyString, const int* optionalPageLoadTimeout, Ref<WaitForNavigationToCompleteCallback>&& callback)
412 {
413     WebPageProxy* page = webPageProxyForHandle(browsingContextHandle);
414     if (!page)
415         ASYNC_FAIL_WITH_PREDEFINED_ERROR(WindowNotFound);
416
417     auto pageLoadStrategy = pageLoadStrategyFromStringParameter(optionalPageLoadStrategyString);
418     if (!pageLoadStrategy)
419         ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(InvalidParameter, "The parameter 'pageLoadStrategy' is invalid.");
420     auto pageLoadTimeout = optionalPageLoadTimeout ? Seconds::fromMilliseconds(*optionalPageLoadTimeout) : defaultPageLoadTimeout;
421
422     // If page is loading and there's an active JavaScript dialog is probably because the
423     // dialog was started in an onload handler, so in case of normal page load strategy the
424     // load will not finish until the dialog is dismissed. Instead of waiting for the timeout,
425     // we return without waiting since we know it will timeout for sure. We want to check
426     // arguments first, though.
427     bool shouldTimeoutDueToUnexpectedAlert = pageLoadStrategy.value() == Inspector::Protocol::Automation::PageLoadStrategy::Normal
428         && page->pageLoadState().isLoading() && m_client->isShowingJavaScriptDialogOnPage(*this, *page);
429
430     if (optionalFrameHandle && !optionalFrameHandle->isEmpty()) {
431         std::optional<uint64_t> frameID = webFrameIDForHandle(*optionalFrameHandle);
432         if (!frameID)
433             ASYNC_FAIL_WITH_PREDEFINED_ERROR(FrameNotFound);
434         WebFrameProxy* frame = page->process().webFrame(frameID.value());
435         if (!frame)
436             ASYNC_FAIL_WITH_PREDEFINED_ERROR(FrameNotFound);
437         if (!shouldTimeoutDueToUnexpectedAlert)
438             waitForNavigationToCompleteOnFrame(*frame, pageLoadStrategy.value(), pageLoadTimeout, WTFMove(callback));
439     } else {
440         if (!shouldTimeoutDueToUnexpectedAlert)
441             waitForNavigationToCompleteOnPage(*page, pageLoadStrategy.value(), pageLoadTimeout, WTFMove(callback));
442     }
443
444     if (shouldTimeoutDueToUnexpectedAlert) {
445         // §9 Navigation.
446         // 7. If the previous step completed by the session page load timeout being reached and the browser does not
447         // have an active user prompt, return error with error code timeout.
448         // 8. Return success with data null.
449         // https://w3c.github.io/webdriver/webdriver-spec.html#dfn-wait-for-navigation-to-complete
450         callback->sendSuccess();
451     }
452 }
453
454 void WebAutomationSession::waitForNavigationToCompleteOnPage(WebPageProxy& page, Inspector::Protocol::Automation::PageLoadStrategy loadStrategy, Seconds timeout, Ref<Inspector::BackendDispatcher::CallbackBase>&& callback)
455 {
456     ASSERT(!m_loadTimer.isActive());
457     if (loadStrategy == Inspector::Protocol::Automation::PageLoadStrategy::None || !page.pageLoadState().isLoading()) {
458         callback->sendSuccess(JSON::Object::create());
459         return;
460     }
461
462     m_loadTimer.startOneShot(timeout);
463     switch (loadStrategy) {
464     case Inspector::Protocol::Automation::PageLoadStrategy::Normal:
465         m_pendingNormalNavigationInBrowsingContextCallbacksPerPage.set(page.pageID(), WTFMove(callback));
466         break;
467     case Inspector::Protocol::Automation::PageLoadStrategy::Eager:
468         m_pendingEagerNavigationInBrowsingContextCallbacksPerPage.set(page.pageID(), WTFMove(callback));
469         break;
470     case Inspector::Protocol::Automation::PageLoadStrategy::None:
471         ASSERT_NOT_REACHED();
472     }
473 }
474
475 void WebAutomationSession::waitForNavigationToCompleteOnFrame(WebFrameProxy& frame, Inspector::Protocol::Automation::PageLoadStrategy loadStrategy, Seconds timeout, Ref<Inspector::BackendDispatcher::CallbackBase>&& callback)
476 {
477     ASSERT(!m_loadTimer.isActive());
478     if (loadStrategy == Inspector::Protocol::Automation::PageLoadStrategy::None || frame.frameLoadState().state() == FrameLoadState::State::Finished) {
479         callback->sendSuccess(JSON::Object::create());
480         return;
481     }
482
483     m_loadTimer.startOneShot(timeout);
484     switch (loadStrategy) {
485     case Inspector::Protocol::Automation::PageLoadStrategy::Normal:
486         m_pendingNormalNavigationInBrowsingContextCallbacksPerFrame.set(frame.frameID(), WTFMove(callback));
487         break;
488     case Inspector::Protocol::Automation::PageLoadStrategy::Eager:
489         m_pendingEagerNavigationInBrowsingContextCallbacksPerFrame.set(frame.frameID(), WTFMove(callback));
490         break;
491     case Inspector::Protocol::Automation::PageLoadStrategy::None:
492         ASSERT_NOT_REACHED();
493     }
494 }
495
496 void WebAutomationSession::respondToPendingPageNavigationCallbacksWithTimeout(HashMap<uint64_t, RefPtr<Inspector::BackendDispatcher::CallbackBase>>& map)
497 {
498     Inspector::ErrorString timeoutError = STRING_FOR_PREDEFINED_ERROR_NAME(Timeout);
499     for (auto id : copyToVector(map.keys())) {
500         auto page = WebProcessProxy::webPage(id);
501         auto callback = map.take(id);
502         if (page && m_client->isShowingJavaScriptDialogOnPage(*this, *page))
503             callback->sendSuccess(JSON::Object::create());
504         else
505             callback->sendFailure(timeoutError);
506     }
507 }
508
509 static WebPageProxy* findPageForFrameID(const WebProcessPool& processPool, uint64_t frameID)
510 {
511     for (auto& process : processPool.processes()) {
512         if (auto* frame = process->webFrame(frameID))
513             return frame->page();
514     }
515     return nullptr;
516 }
517
518 void WebAutomationSession::respondToPendingFrameNavigationCallbacksWithTimeout(HashMap<uint64_t, RefPtr<Inspector::BackendDispatcher::CallbackBase>>& map)
519 {
520     Inspector::ErrorString timeoutError = STRING_FOR_PREDEFINED_ERROR_NAME(Timeout);
521     for (auto id : copyToVector(map.keys())) {
522         auto* page = findPageForFrameID(*m_processPool, id);
523         auto callback = map.take(id);
524         if (page && m_client->isShowingJavaScriptDialogOnPage(*this, *page))
525             callback->sendSuccess(JSON::Object::create());
526         else
527             callback->sendFailure(timeoutError);
528     }
529 }
530
531 void WebAutomationSession::loadTimerFired()
532 {
533     respondToPendingFrameNavigationCallbacksWithTimeout(m_pendingNormalNavigationInBrowsingContextCallbacksPerFrame);
534     respondToPendingFrameNavigationCallbacksWithTimeout(m_pendingEagerNavigationInBrowsingContextCallbacksPerFrame);
535     respondToPendingPageNavigationCallbacksWithTimeout(m_pendingNormalNavigationInBrowsingContextCallbacksPerPage);
536     respondToPendingPageNavigationCallbacksWithTimeout(m_pendingEagerNavigationInBrowsingContextCallbacksPerPage);
537 }
538
539 void WebAutomationSession::maximizeWindowOfBrowsingContext(const String& browsingContextHandle, Ref<MaximizeWindowOfBrowsingContextCallback>&& callback)
540 {
541     auto* page = webPageProxyForHandle(browsingContextHandle);
542     if (!page)
543         ASYNC_FAIL_WITH_PREDEFINED_ERROR(WindowNotFound);
544
545     exitFullscreenWindowForPage(*page, [this, protectedThis = makeRef(*this), callback = WTFMove(callback), page = makeRefPtr(page)]() mutable {
546         auto& webPage = *page;
547         restoreWindowForPage(webPage, [this, callback = WTFMove(callback), page = WTFMove(page)]() mutable {
548             maximizeWindowForPage(*page, [callback = WTFMove(callback)]() {
549                 callback->sendSuccess();
550             });
551         });
552     });
553 }
554
555 void WebAutomationSession::hideWindowOfBrowsingContext(const String& browsingContextHandle, Ref<HideWindowOfBrowsingContextCallback>&& callback)
556 {
557     WebPageProxy* page = webPageProxyForHandle(browsingContextHandle);
558     if (!page)
559         ASYNC_FAIL_WITH_PREDEFINED_ERROR(WindowNotFound);
560     
561     exitFullscreenWindowForPage(*page, [protectedThis = makeRef(*this), callback = WTFMove(callback), page = makeRefPtr(page)]() mutable {
562         protectedThis->hideWindowForPage(*page, [callback = WTFMove(callback)]() mutable {
563             callback->sendSuccess();
564         });
565     });
566 }
567
568 void WebAutomationSession::exitFullscreenWindowForPage(WebPageProxy& page, WTF::CompletionHandler<void()>&& completionHandler)
569 {
570 #if ENABLE(FULLSCREEN_API)
571     ASSERT(!m_windowStateTransitionCallback);
572     if (!page.fullScreenManager()->isFullScreen()) {
573         completionHandler();
574         return;
575     }
576     
577     m_windowStateTransitionCallback = WTF::Function<void(WindowTransitionedToState)> { [this, protectedThis = makeRef(*this), completionHandler = WTFMove(completionHandler)](WindowTransitionedToState state) mutable {
578         // If fullscreen exited and we didn't request that, just ignore it.
579         if (state != WindowTransitionedToState::Unfullscreen)
580             return;
581
582         // Keep this callback in scope so completionHandler does not get destroyed before we call it.
583         auto protectedCallback = WTFMove(m_windowStateTransitionCallback);
584         completionHandler();
585     } };
586     
587     page.fullScreenManager()->requestExitFullScreen();
588 #else
589     completionHandler();
590 #endif
591 }
592
593 void WebAutomationSession::restoreWindowForPage(WebPageProxy& page, WTF::CompletionHandler<void()>&& completionHandler)
594 {
595     m_client->requestRestoreWindowOfPage(*this, page, WTFMove(completionHandler));
596 }
597
598 void WebAutomationSession::maximizeWindowForPage(WebPageProxy& page, WTF::CompletionHandler<void()>&& completionHandler)
599 {
600     m_client->requestMaximizeWindowOfPage(*this, page, WTFMove(completionHandler));
601 }
602
603 void WebAutomationSession::hideWindowForPage(WebPageProxy& page, WTF::CompletionHandler<void()>&& completionHandler)
604 {
605     m_client->requestHideWindowOfPage(*this, page, WTFMove(completionHandler));
606 }
607
608 void WebAutomationSession::willShowJavaScriptDialog(WebPageProxy& page)
609 {
610     // Wait until the next run loop iteration to give time for the client to show the dialog,
611     // then check if the dialog is still present. If the page is loading, the dialog will block
612     // the load in case of normal strategy, so we want to dispatch all pending navigation callbacks.
613     // If the dialog was shown during a script execution, we want to finish the evaluateJavaScriptFunction
614     // operation with an unexpected alert open error.
615     RunLoop::main().dispatch([this, protectedThis = makeRef(*this), page = makeRef(page)] {
616         if (!page->isValid() || !m_client || !m_client->isShowingJavaScriptDialogOnPage(*this, page))
617             return;
618
619         if (page->pageLoadState().isLoading()) {
620             m_loadTimer.stop();
621             respondToPendingFrameNavigationCallbacksWithTimeout(m_pendingNormalNavigationInBrowsingContextCallbacksPerFrame);
622             respondToPendingPageNavigationCallbacksWithTimeout(m_pendingNormalNavigationInBrowsingContextCallbacksPerPage);
623         }
624
625         if (!m_evaluateJavaScriptFunctionCallbacks.isEmpty()) {
626             Inspector::ErrorString unexpectedAlertOpenError = STRING_FOR_PREDEFINED_ERROR_NAME(UnexpectedAlertOpen);
627             for (auto key : copyToVector(m_evaluateJavaScriptFunctionCallbacks.keys())) {
628                 auto callback = m_evaluateJavaScriptFunctionCallbacks.take(key);
629                 callback->sendFailure(unexpectedAlertOpenError);
630             }
631         }
632         
633         if (!m_pendingMouseEventsFlushedCallbacksPerPage.isEmpty()) {
634             for (auto key : copyToVector(m_pendingMouseEventsFlushedCallbacksPerPage.keys())) {
635                 auto callback = m_pendingMouseEventsFlushedCallbacksPerPage.take(key);
636                 callback(std::nullopt);
637             }
638         }
639
640         if (!m_pendingKeyboardEventsFlushedCallbacksPerPage.isEmpty()) {
641             for (auto key : copyToVector(m_pendingKeyboardEventsFlushedCallbacksPerPage.keys())) {
642                 auto callback = m_pendingKeyboardEventsFlushedCallbacksPerPage.take(key);
643                 callback(std::nullopt);
644             }
645         }
646     });
647 }
648     
649 void WebAutomationSession::didEnterFullScreenForPage(const WebPageProxy&)
650 {
651     if (m_windowStateTransitionCallback)
652         m_windowStateTransitionCallback(WindowTransitionedToState::Fullscreen);
653 }
654
655 void WebAutomationSession::didExitFullScreenForPage(const WebPageProxy&)
656 {
657     if (m_windowStateTransitionCallback)
658         m_windowStateTransitionCallback(WindowTransitionedToState::Unfullscreen);
659 }
660
661 void WebAutomationSession::navigateBrowsingContext(const String& handle, const String& url, const String* optionalPageLoadStrategyString, const int* optionalPageLoadTimeout, Ref<NavigateBrowsingContextCallback>&& callback)
662 {
663     WebPageProxy* page = webPageProxyForHandle(handle);
664     if (!page)
665         ASYNC_FAIL_WITH_PREDEFINED_ERROR(WindowNotFound);
666
667     auto pageLoadStrategy = pageLoadStrategyFromStringParameter(optionalPageLoadStrategyString);
668     if (!pageLoadStrategy)
669         ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(InvalidParameter, "The parameter 'pageLoadStrategy' is invalid.");
670     auto pageLoadTimeout = optionalPageLoadTimeout ? Seconds::fromMilliseconds(*optionalPageLoadTimeout) : defaultPageLoadTimeout;
671
672     page->loadRequest(URL(URL(), url));
673     waitForNavigationToCompleteOnPage(*page, pageLoadStrategy.value(), pageLoadTimeout, WTFMove(callback));
674 }
675
676 void WebAutomationSession::goBackInBrowsingContext(const String& handle, const String* optionalPageLoadStrategyString, const int* optionalPageLoadTimeout, Ref<GoBackInBrowsingContextCallback>&& callback)
677 {
678     WebPageProxy* page = webPageProxyForHandle(handle);
679     if (!page)
680         ASYNC_FAIL_WITH_PREDEFINED_ERROR(WindowNotFound);
681
682     auto pageLoadStrategy = pageLoadStrategyFromStringParameter(optionalPageLoadStrategyString);
683     if (!pageLoadStrategy)
684         ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(InvalidParameter, "The parameter 'pageLoadStrategy' is invalid.");
685     auto pageLoadTimeout = optionalPageLoadTimeout ? Seconds::fromMilliseconds(*optionalPageLoadTimeout) : defaultPageLoadTimeout;
686
687     page->goBack();
688     waitForNavigationToCompleteOnPage(*page, pageLoadStrategy.value(), pageLoadTimeout, WTFMove(callback));
689 }
690
691 void WebAutomationSession::goForwardInBrowsingContext(const String& handle, const String* optionalPageLoadStrategyString, const int* optionalPageLoadTimeout, Ref<GoForwardInBrowsingContextCallback>&& callback)
692 {
693     WebPageProxy* page = webPageProxyForHandle(handle);
694     if (!page)
695         ASYNC_FAIL_WITH_PREDEFINED_ERROR(WindowNotFound);
696
697     auto pageLoadStrategy = pageLoadStrategyFromStringParameter(optionalPageLoadStrategyString);
698     if (!pageLoadStrategy)
699         ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(InvalidParameter, "The parameter 'pageLoadStrategy' is invalid.");
700     auto pageLoadTimeout = optionalPageLoadTimeout ? Seconds::fromMilliseconds(*optionalPageLoadTimeout) : defaultPageLoadTimeout;
701
702     page->goForward();
703     waitForNavigationToCompleteOnPage(*page, pageLoadStrategy.value(), pageLoadTimeout, WTFMove(callback));
704 }
705
706 void WebAutomationSession::reloadBrowsingContext(const String& handle, const String* optionalPageLoadStrategyString, const int* optionalPageLoadTimeout, Ref<ReloadBrowsingContextCallback>&& callback)
707 {
708     WebPageProxy* page = webPageProxyForHandle(handle);
709     if (!page)
710         ASYNC_FAIL_WITH_PREDEFINED_ERROR(WindowNotFound);
711
712     auto pageLoadStrategy = pageLoadStrategyFromStringParameter(optionalPageLoadStrategyString);
713     if (!pageLoadStrategy)
714         ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(InvalidParameter, "The parameter 'pageLoadStrategy' is invalid.");
715     auto pageLoadTimeout = optionalPageLoadTimeout ? Seconds::fromMilliseconds(*optionalPageLoadTimeout) : defaultPageLoadTimeout;
716
717     page->reload({ });
718     waitForNavigationToCompleteOnPage(*page, pageLoadStrategy.value(), pageLoadTimeout, WTFMove(callback));
719 }
720
721 void WebAutomationSession::navigationOccurredForFrame(const WebFrameProxy& frame)
722 {
723     if (frame.isMainFrame()) {
724         // New page loaded, clear frame handles previously cached.
725         m_handleWebFrameMap.clear();
726         m_webFrameHandleMap.clear();
727         if (auto callback = m_pendingNormalNavigationInBrowsingContextCallbacksPerPage.take(frame.page()->pageID())) {
728             m_loadTimer.stop();
729             callback->sendSuccess(JSON::Object::create());
730         }
731         m_domainNotifier->browsingContextCleared(handleForWebPageProxy(*frame.page()));
732     } else {
733         if (auto callback = m_pendingNormalNavigationInBrowsingContextCallbacksPerFrame.take(frame.frameID())) {
734             m_loadTimer.stop();
735             callback->sendSuccess(JSON::Object::create());
736         }
737     }
738 }
739
740 void WebAutomationSession::documentLoadedForFrame(const WebFrameProxy& frame)
741 {
742     if (frame.isMainFrame()) {
743         if (auto callback = m_pendingEagerNavigationInBrowsingContextCallbacksPerPage.take(frame.page()->pageID())) {
744             m_loadTimer.stop();
745             callback->sendSuccess(JSON::Object::create());
746         }
747     } else {
748         if (auto callback = m_pendingEagerNavigationInBrowsingContextCallbacksPerFrame.take(frame.frameID())) {
749             m_loadTimer.stop();
750             callback->sendSuccess(JSON::Object::create());
751         }
752     }
753 }
754
755 void WebAutomationSession::inspectorFrontendLoaded(const WebPageProxy& page)
756 {
757     if (auto callback = m_pendingInspectorCallbacksPerPage.take(page.pageID()))
758         callback->sendSuccess(JSON::Object::create());
759 }
760
761 void WebAutomationSession::mouseEventsFlushedForPage(const WebPageProxy& page)
762 {
763     if (auto callback = m_pendingMouseEventsFlushedCallbacksPerPage.take(page.pageID()))
764         callback(std::nullopt);
765 }
766
767 void WebAutomationSession::keyboardEventsFlushedForPage(const WebPageProxy& page)
768 {
769     if (auto callback = m_pendingKeyboardEventsFlushedCallbacksPerPage.take(page.pageID()))
770         callback(std::nullopt);
771 }
772
773 void WebAutomationSession::willClosePage(const WebPageProxy& page)
774 {
775     String handle = handleForWebPageProxy(page);
776     m_domainNotifier->browsingContextCleared(handle);
777
778     // Cancel pending interactions on this page. By providing an error, this will cause subsequent
779     // actions to be aborted and the SimulatedInputDispatcher::run() call will unwind and fail.
780     if (auto callback = m_pendingMouseEventsFlushedCallbacksPerPage.take(page.pageID()))
781         callback(AUTOMATION_COMMAND_ERROR_WITH_NAME(WindowNotFound));
782     if (auto callback = m_pendingMouseEventsFlushedCallbacksPerPage.take(page.pageID()))
783         callback(AUTOMATION_COMMAND_ERROR_WITH_NAME(WindowNotFound));
784
785     // Then tell the input dispatcher to cancel so timers are stopped, and let it go out of scope.
786     std::optional<Ref<SimulatedInputDispatcher>> inputDispatcher = m_inputDispatchersByPage.take(page.pageID());
787     if (inputDispatcher.has_value())
788         inputDispatcher.value()->cancel();
789 }
790
791 static bool fileCanBeAcceptedForUpload(const String& filename, const HashSet<String>& allowedMIMETypes, const HashSet<String>& allowedFileExtensions)
792 {
793     if (!WebCore::FileSystem::fileExists(filename))
794         return false;
795
796     if (allowedMIMETypes.isEmpty() && allowedFileExtensions.isEmpty())
797         return true;
798
799     // We can't infer a MIME type from a file without an extension, just give up.
800     auto dotOffset = filename.reverseFind('.');
801     if (dotOffset == notFound)
802         return false;
803
804     String extension = filename.substring(dotOffset + 1).convertToASCIILowercase();
805     if (extension.isEmpty())
806         return false;
807
808     if (allowedFileExtensions.contains(extension))
809         return true;
810
811     String mappedMIMEType = WebCore::MIMETypeRegistry::getMIMETypeForExtension(extension).convertToASCIILowercase();
812     if (mappedMIMEType.isEmpty())
813         return false;
814     
815     if (allowedMIMETypes.contains(mappedMIMEType))
816         return true;
817
818     // Fall back to checking for a MIME type wildcard if an exact match is not found.
819     Vector<String> components = mappedMIMEType.split('/');
820     if (components.size() != 2)
821         return false;
822
823     String wildcardedMIMEType = makeString(components[0], "/*");
824     if (allowedMIMETypes.contains(wildcardedMIMEType))
825         return true;
826
827     return false;
828 }
829
830 void WebAutomationSession::handleRunOpenPanel(const WebPageProxy& page, const WebFrameProxy&, const API::OpenPanelParameters& parameters, WebOpenPanelResultListenerProxy& resultListener)
831 {
832     String browsingContextHandle = handleForWebPageProxy(page);
833     if (!m_filesToSelectForFileUpload.size()) {
834         resultListener.cancel();
835         m_domainNotifier->fileChooserDismissed(browsingContextHandle, true);
836         return;
837     }
838
839     if (m_filesToSelectForFileUpload.size() > 1 && !parameters.allowMultipleFiles()) {
840         resultListener.cancel();
841         m_domainNotifier->fileChooserDismissed(browsingContextHandle, true);
842         return;
843     }
844
845     HashSet<String> allowedMIMETypes;
846     auto acceptMIMETypes = parameters.acceptMIMETypes();
847     for (auto type : acceptMIMETypes->elementsOfType<API::String>())
848         allowedMIMETypes.add(type->string());
849
850     HashSet<String> allowedFileExtensions;
851     auto acceptFileExtensions = parameters.acceptFileExtensions();
852     for (auto type : acceptFileExtensions->elementsOfType<API::String>()) {
853         // WebCore vends extensions with leading periods. Strip these to simplify matching later.
854         String extension = type->string();
855         ASSERT(extension.characterAt(0) == '.');
856         allowedFileExtensions.add(extension.substring(1));
857     }
858
859     // Per §14.3.10.5 in the W3C spec, if at least one file cannot be accepted, the command should fail.
860     // The REST API service can tell that this failed by checking the "files" attribute of the input element.
861     for (const String& filename : m_filesToSelectForFileUpload) {
862         if (!fileCanBeAcceptedForUpload(filename, allowedMIMETypes, allowedFileExtensions)) {
863             resultListener.cancel();
864             m_domainNotifier->fileChooserDismissed(browsingContextHandle, true);
865             return;
866         }
867     }
868
869     resultListener.chooseFiles(m_filesToSelectForFileUpload);
870     m_domainNotifier->fileChooserDismissed(browsingContextHandle, false);
871 }
872
873 void WebAutomationSession::evaluateJavaScriptFunction(const String& browsingContextHandle, const String* optionalFrameHandle, const String& function, const JSON::Array& arguments, const bool* optionalExpectsImplicitCallbackArgument, const int* optionalCallbackTimeout, Ref<EvaluateJavaScriptFunctionCallback>&& callback)
874 {
875     WebPageProxy* page = webPageProxyForHandle(browsingContextHandle);
876     if (!page)
877         ASYNC_FAIL_WITH_PREDEFINED_ERROR(WindowNotFound);
878
879     std::optional<uint64_t> frameID = webFrameIDForHandle(optionalFrameHandle ? *optionalFrameHandle : emptyString());
880     if (!frameID)
881         ASYNC_FAIL_WITH_PREDEFINED_ERROR(FrameNotFound);
882
883     Vector<String> argumentsVector;
884     argumentsVector.reserveCapacity(arguments.length());
885
886     for (auto& argument : arguments) {
887         String argumentString;
888         argument->asString(argumentString);
889         argumentsVector.uncheckedAppend(argumentString);
890     }
891
892     bool expectsImplicitCallbackArgument = optionalExpectsImplicitCallbackArgument ? *optionalExpectsImplicitCallbackArgument : false;
893     int callbackTimeout = optionalCallbackTimeout ? *optionalCallbackTimeout : 0;
894
895     uint64_t callbackID = m_nextEvaluateJavaScriptCallbackID++;
896     m_evaluateJavaScriptFunctionCallbacks.set(callbackID, WTFMove(callback));
897
898     page->process().send(Messages::WebAutomationSessionProxy::EvaluateJavaScriptFunction(page->pageID(), frameID.value(), function, argumentsVector, expectsImplicitCallbackArgument, callbackTimeout, callbackID), 0);
899 }
900
901 void WebAutomationSession::didEvaluateJavaScriptFunction(uint64_t callbackID, const String& result, const String& errorType)
902 {
903     auto callback = m_evaluateJavaScriptFunctionCallbacks.take(callbackID);
904     if (!callback)
905         return;
906
907     if (!errorType.isEmpty())
908         callback->sendFailure(STRING_FOR_PREDEFINED_ERROR_MESSAGE_AND_DETAILS(errorType, result));
909     else
910         callback->sendSuccess(result);
911 }
912
913 void WebAutomationSession::resolveChildFrameHandle(const String& browsingContextHandle, const String* optionalFrameHandle, const int* optionalOrdinal, const String* optionalName, const String* optionalNodeHandle, Ref<ResolveChildFrameHandleCallback>&& callback)
914 {
915     if (!optionalOrdinal && !optionalName && !optionalNodeHandle)
916         ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(MissingParameter, "Command must specify a child frame by ordinal, name, or element handle.");
917
918     WebPageProxy* page = webPageProxyForHandle(browsingContextHandle);
919     if (!page)
920         ASYNC_FAIL_WITH_PREDEFINED_ERROR(WindowNotFound);
921
922     std::optional<uint64_t> frameID = webFrameIDForHandle(optionalFrameHandle ? *optionalFrameHandle : emptyString());
923     if (!frameID)
924         ASYNC_FAIL_WITH_PREDEFINED_ERROR(FrameNotFound);
925
926     uint64_t callbackID = m_nextResolveFrameCallbackID++;
927     m_resolveChildFrameHandleCallbacks.set(callbackID, WTFMove(callback));
928
929     if (optionalNodeHandle) {
930         page->process().send(Messages::WebAutomationSessionProxy::ResolveChildFrameWithNodeHandle(page->pageID(), frameID.value(), *optionalNodeHandle, callbackID), 0);
931         return;
932     }
933
934     if (optionalName) {
935         page->process().send(Messages::WebAutomationSessionProxy::ResolveChildFrameWithName(page->pageID(), frameID.value(), *optionalName, callbackID), 0);
936         return;
937     }
938
939     if (optionalOrdinal) {
940         page->process().send(Messages::WebAutomationSessionProxy::ResolveChildFrameWithOrdinal(page->pageID(), frameID.value(), *optionalOrdinal, callbackID), 0);
941         return;
942     }
943
944     ASSERT_NOT_REACHED();
945 }
946
947 void WebAutomationSession::didResolveChildFrame(uint64_t callbackID, uint64_t frameID, const String& errorType)
948 {
949     auto callback = m_resolveChildFrameHandleCallbacks.take(callbackID);
950     if (!callback)
951         return;
952
953     if (!errorType.isEmpty())
954         callback->sendFailure(STRING_FOR_PREDEFINED_ERROR_MESSAGE(errorType));
955     else
956         callback->sendSuccess(handleForWebFrameID(frameID));
957 }
958
959 void WebAutomationSession::resolveParentFrameHandle(const String& browsingContextHandle, const String& frameHandle, Ref<ResolveParentFrameHandleCallback>&& callback)
960 {
961     WebPageProxy* page = webPageProxyForHandle(browsingContextHandle);
962     if (!page)
963         ASYNC_FAIL_WITH_PREDEFINED_ERROR(WindowNotFound);
964
965     std::optional<uint64_t> frameID = webFrameIDForHandle(frameHandle);
966     if (!frameID)
967         ASYNC_FAIL_WITH_PREDEFINED_ERROR(FrameNotFound);
968
969     uint64_t callbackID = m_nextResolveParentFrameCallbackID++;
970     m_resolveParentFrameHandleCallbacks.set(callbackID, WTFMove(callback));
971
972     page->process().send(Messages::WebAutomationSessionProxy::ResolveParentFrame(page->pageID(), frameID.value(), callbackID), 0);
973 }
974
975 void WebAutomationSession::didResolveParentFrame(uint64_t callbackID, uint64_t frameID, const String& errorType)
976 {
977     auto callback = m_resolveParentFrameHandleCallbacks.take(callbackID);
978     if (!callback)
979         return;
980
981     if (!errorType.isEmpty())
982         callback->sendFailure(STRING_FOR_PREDEFINED_ERROR_MESSAGE(errorType));
983     else
984         callback->sendSuccess(handleForWebFrameID(frameID));
985 }
986
987 static std::optional<CoordinateSystem> protocolStringToCoordinateSystem(const String& coordinateSystemString)
988 {
989     if (coordinateSystemString == "Page")
990         return CoordinateSystem::Page;
991     if (coordinateSystemString == "LayoutViewport")
992         return CoordinateSystem::LayoutViewport;
993     if (coordinateSystemString == "VisualViewport")
994         return CoordinateSystem::VisualViewport;
995     return std::nullopt;
996 }
997
998 void WebAutomationSession::computeElementLayout(const String& browsingContextHandle, const String& frameHandle, const String& nodeHandle, const bool* optionalScrollIntoViewIfNeeded, const String& coordinateSystemString, Ref<ComputeElementLayoutCallback>&& callback)
999 {
1000     WebPageProxy* page = webPageProxyForHandle(browsingContextHandle);
1001     if (!page)
1002         ASYNC_FAIL_WITH_PREDEFINED_ERROR(WindowNotFound);
1003
1004     std::optional<uint64_t> frameID = webFrameIDForHandle(frameHandle);
1005     if (!frameID)
1006         ASYNC_FAIL_WITH_PREDEFINED_ERROR(FrameNotFound);
1007
1008     std::optional<CoordinateSystem> coordinateSystem = protocolStringToCoordinateSystem(coordinateSystemString);
1009     if (!coordinateSystem)
1010         ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(InvalidParameter, "The parameter 'coordinateSystem' is invalid.");
1011
1012     // Start at 2 and use only even numbers to not conflict with m_nextViewportInViewCenterPointOfElementCallbackID.
1013     uint64_t callbackID = m_nextComputeElementLayoutCallbackID += 2;
1014     m_computeElementLayoutCallbacks.set(callbackID, WTFMove(callback));
1015
1016     bool scrollIntoViewIfNeeded = optionalScrollIntoViewIfNeeded ? *optionalScrollIntoViewIfNeeded : false;
1017     page->process().send(Messages::WebAutomationSessionProxy::ComputeElementLayout(page->pageID(), frameID.value(), nodeHandle, scrollIntoViewIfNeeded, coordinateSystem.value(), callbackID), 0);
1018 }
1019
1020 void WebAutomationSession::didComputeElementLayout(uint64_t callbackID, WebCore::IntRect rect, std::optional<WebCore::IntPoint> inViewCenterPoint, bool isObscured, const String& errorType)
1021 {
1022     if (callbackID % 2 == 1) {
1023         ASSERT(inViewCenterPoint);
1024         if (auto callback = m_viewportInViewCenterPointOfElementCallbacks.take(callbackID)) {
1025             std::optional<AutomationCommandError> error;
1026             if (!errorType.isEmpty())
1027                 error = AUTOMATION_COMMAND_ERROR_WITH_MESSAGE(errorType);
1028             callback(inViewCenterPoint, error);
1029         }
1030         return;
1031     }
1032
1033     auto callback = m_computeElementLayoutCallbacks.take(callbackID);
1034     if (!callback)
1035         return;
1036
1037     if (!errorType.isEmpty()) {
1038         callback->sendFailure(STRING_FOR_PREDEFINED_ERROR_MESSAGE(errorType));
1039         return;
1040     }
1041
1042     auto originObject = Inspector::Protocol::Automation::Point::create()
1043         .setX(rect.x())
1044         .setY(rect.y())
1045         .release();
1046
1047     auto sizeObject = Inspector::Protocol::Automation::Size::create()
1048         .setWidth(rect.width())
1049         .setHeight(rect.height())
1050         .release();
1051
1052     auto rectObject = Inspector::Protocol::Automation::Rect::create()
1053         .setOrigin(WTFMove(originObject))
1054         .setSize(WTFMove(sizeObject))
1055         .release();
1056
1057     if (!inViewCenterPoint) {
1058         callback->sendSuccess(WTFMove(rectObject), nullptr, isObscured);
1059         return;
1060     }
1061
1062     auto inViewCenterPointObject = Inspector::Protocol::Automation::Point::create()
1063         .setX(inViewCenterPoint.value().x())
1064         .setY(inViewCenterPoint.value().y())
1065         .release();
1066
1067     callback->sendSuccess(WTFMove(rectObject), WTFMove(inViewCenterPointObject), isObscured);
1068 }
1069
1070 void WebAutomationSession::selectOptionElement(const String& browsingContextHandle, const String& frameHandle, const String& nodeHandle, Ref<SelectOptionElementCallback>&& callback)
1071 {
1072     WebPageProxy* page = webPageProxyForHandle(browsingContextHandle);
1073     if (!page)
1074         ASYNC_FAIL_WITH_PREDEFINED_ERROR(WindowNotFound);
1075
1076     std::optional<uint64_t> frameID = webFrameIDForHandle(frameHandle);
1077     if (!frameID)
1078         ASYNC_FAIL_WITH_PREDEFINED_ERROR(FrameNotFound);
1079
1080     uint64_t callbackID = m_nextSelectOptionElementCallbackID++;
1081     m_selectOptionElementCallbacks.set(callbackID, WTFMove(callback));
1082
1083     page->process().send(Messages::WebAutomationSessionProxy::SelectOptionElement(page->pageID(), frameID.value(), nodeHandle, callbackID), 0);
1084 }
1085
1086 void WebAutomationSession::didSelectOptionElement(uint64_t callbackID, const String& errorType)
1087 {
1088     auto callback = m_selectOptionElementCallbacks.take(callbackID);
1089     if (!callback)
1090         return;
1091
1092     if (!errorType.isEmpty()) {
1093         callback->sendFailure(STRING_FOR_PREDEFINED_ERROR_MESSAGE(errorType));
1094         return;
1095     }
1096
1097     callback->sendSuccess();
1098 }
1099
1100
1101 void WebAutomationSession::isShowingJavaScriptDialog(Inspector::ErrorString& errorString, const String& browsingContextHandle, bool* result)
1102 {
1103     ASSERT(m_client);
1104     if (!m_client)
1105         SYNC_FAIL_WITH_PREDEFINED_ERROR(InternalError);
1106
1107     WebPageProxy* page = webPageProxyForHandle(browsingContextHandle);
1108     if (!page)
1109         SYNC_FAIL_WITH_PREDEFINED_ERROR(WindowNotFound);
1110
1111     *result = m_client->isShowingJavaScriptDialogOnPage(*this, *page);
1112 }
1113
1114 void WebAutomationSession::dismissCurrentJavaScriptDialog(Inspector::ErrorString& errorString, const String& browsingContextHandle)
1115 {
1116     ASSERT(m_client);
1117     if (!m_client)
1118         SYNC_FAIL_WITH_PREDEFINED_ERROR(InternalError);
1119
1120     WebPageProxy* page = webPageProxyForHandle(browsingContextHandle);
1121     if (!page)
1122         SYNC_FAIL_WITH_PREDEFINED_ERROR(WindowNotFound);
1123
1124     if (!m_client->isShowingJavaScriptDialogOnPage(*this, *page))
1125         SYNC_FAIL_WITH_PREDEFINED_ERROR(NoJavaScriptDialog);
1126
1127     m_client->dismissCurrentJavaScriptDialogOnPage(*this, *page);
1128 }
1129
1130 void WebAutomationSession::acceptCurrentJavaScriptDialog(Inspector::ErrorString& errorString, const String& browsingContextHandle)
1131 {
1132     ASSERT(m_client);
1133     if (!m_client)
1134         SYNC_FAIL_WITH_PREDEFINED_ERROR(InternalError);
1135
1136     WebPageProxy* page = webPageProxyForHandle(browsingContextHandle);
1137     if (!page)
1138         SYNC_FAIL_WITH_PREDEFINED_ERROR(WindowNotFound);
1139
1140     if (!m_client->isShowingJavaScriptDialogOnPage(*this, *page))
1141         SYNC_FAIL_WITH_PREDEFINED_ERROR(NoJavaScriptDialog);
1142
1143     m_client->acceptCurrentJavaScriptDialogOnPage(*this, *page);
1144 }
1145
1146 void WebAutomationSession::messageOfCurrentJavaScriptDialog(Inspector::ErrorString& errorString, const String& browsingContextHandle, String* text)
1147 {
1148     ASSERT(m_client);
1149     if (!m_client)
1150         SYNC_FAIL_WITH_PREDEFINED_ERROR(InternalError);
1151
1152     WebPageProxy* page = webPageProxyForHandle(browsingContextHandle);
1153     if (!page)
1154         SYNC_FAIL_WITH_PREDEFINED_ERROR(WindowNotFound);
1155
1156     if (!m_client->isShowingJavaScriptDialogOnPage(*this, *page))
1157         SYNC_FAIL_WITH_PREDEFINED_ERROR(NoJavaScriptDialog);
1158
1159     *text = m_client->messageOfCurrentJavaScriptDialogOnPage(*this, *page);
1160 }
1161
1162 void WebAutomationSession::setUserInputForCurrentJavaScriptPrompt(Inspector::ErrorString& errorString, const String& browsingContextHandle, const String& promptValue)
1163 {
1164     ASSERT(m_client);
1165     if (!m_client)
1166         SYNC_FAIL_WITH_PREDEFINED_ERROR(InternalError);
1167
1168     WebPageProxy* page = webPageProxyForHandle(browsingContextHandle);
1169     if (!page)
1170         SYNC_FAIL_WITH_PREDEFINED_ERROR(WindowNotFound);
1171
1172     if (!m_client->isShowingJavaScriptDialogOnPage(*this, *page))
1173         SYNC_FAIL_WITH_PREDEFINED_ERROR(NoJavaScriptDialog);
1174
1175     // §18.4 Send Alert Text.
1176     // https://w3c.github.io/webdriver/webdriver-spec.html#send-alert-text
1177     // 3. Run the substeps of the first matching current user prompt:
1178     auto scriptDialogType = m_client->typeOfCurrentJavaScriptDialogOnPage(*this, *page);
1179     ASSERT(scriptDialogType);
1180     switch (scriptDialogType.value()) {
1181     case API::AutomationSessionClient::JavaScriptDialogType::Alert:
1182     case API::AutomationSessionClient::JavaScriptDialogType::Confirm:
1183         // Return error with error code element not interactable.
1184         SYNC_FAIL_WITH_PREDEFINED_ERROR(ElementNotInteractable);
1185     case API::AutomationSessionClient::JavaScriptDialogType::Prompt:
1186         // Do nothing.
1187         break;
1188     case API::AutomationSessionClient::JavaScriptDialogType::BeforeUnloadConfirm:
1189         // Return error with error code unsupported operation.
1190         SYNC_FAIL_WITH_PREDEFINED_ERROR(NotImplemented);
1191     }
1192
1193     m_client->setUserInputForCurrentJavaScriptPromptOnPage(*this, *page, promptValue);
1194 }
1195
1196 void WebAutomationSession::setFilesToSelectForFileUpload(ErrorString& errorString, const String& browsingContextHandle, const JSON::Array& filenames)
1197 {
1198     Vector<String> newFileList;
1199     newFileList.reserveInitialCapacity(filenames.length());
1200
1201     for (auto item : filenames) {
1202         String filename;
1203         if (!item->asString(filename))
1204             SYNC_FAIL_WITH_PREDEFINED_ERROR(InternalError);
1205
1206         newFileList.append(filename);
1207     }
1208
1209     m_filesToSelectForFileUpload.swap(newFileList);
1210 }
1211
1212 void WebAutomationSession::getAllCookies(const String& browsingContextHandle, Ref<GetAllCookiesCallback>&& callback)
1213 {
1214     WebPageProxy* page = webPageProxyForHandle(browsingContextHandle);
1215     if (!page)
1216         ASYNC_FAIL_WITH_PREDEFINED_ERROR(WindowNotFound);
1217
1218     // Always send the main frame ID as 0 so it is resolved on the WebProcess side. This avoids a race when page->mainFrame() is null still.
1219     const uint64_t mainFrameID = 0;
1220
1221     uint64_t callbackID = m_nextGetCookiesCallbackID++;
1222     m_getCookieCallbacks.set(callbackID, WTFMove(callback));
1223
1224     page->process().send(Messages::WebAutomationSessionProxy::GetCookiesForFrame(page->pageID(), mainFrameID, callbackID), 0);
1225 }
1226
1227 static Ref<Inspector::Protocol::Automation::Cookie> buildObjectForCookie(const WebCore::Cookie& cookie)
1228 {
1229     return Inspector::Protocol::Automation::Cookie::create()
1230         .setName(cookie.name)
1231         .setValue(cookie.value)
1232         .setDomain(cookie.domain)
1233         .setPath(cookie.path)
1234         .setExpires(cookie.expires ? cookie.expires / 1000 : 0)
1235         .setSize((cookie.name.length() + cookie.value.length()))
1236         .setHttpOnly(cookie.httpOnly)
1237         .setSecure(cookie.secure)
1238         .setSession(cookie.session)
1239         .release();
1240 }
1241
1242 static Ref<JSON::ArrayOf<Inspector::Protocol::Automation::Cookie>> buildArrayForCookies(Vector<WebCore::Cookie>& cookiesList)
1243 {
1244     auto cookies = JSON::ArrayOf<Inspector::Protocol::Automation::Cookie>::create();
1245
1246     for (const auto& cookie : cookiesList)
1247         cookies->addItem(buildObjectForCookie(cookie));
1248
1249     return cookies;
1250 }
1251
1252 void WebAutomationSession::didGetCookiesForFrame(uint64_t callbackID, Vector<WebCore::Cookie> cookies, const String& errorType)
1253 {
1254     auto callback = m_getCookieCallbacks.take(callbackID);
1255     if (!callback)
1256         return;
1257
1258     if (!errorType.isEmpty()) {
1259         callback->sendFailure(STRING_FOR_PREDEFINED_ERROR_MESSAGE(errorType));
1260         return;
1261     }
1262
1263     callback->sendSuccess(buildArrayForCookies(cookies));
1264 }
1265
1266 void WebAutomationSession::deleteSingleCookie(const String& browsingContextHandle, const String& cookieName, Ref<DeleteSingleCookieCallback>&& callback)
1267 {
1268     WebPageProxy* page = webPageProxyForHandle(browsingContextHandle);
1269     if (!page)
1270         ASYNC_FAIL_WITH_PREDEFINED_ERROR(WindowNotFound);
1271
1272     // Always send the main frame ID as 0 so it is resolved on the WebProcess side. This avoids a race when page->mainFrame() is null still.
1273     const uint64_t mainFrameID = 0;
1274
1275     uint64_t callbackID = m_nextDeleteCookieCallbackID++;
1276     m_deleteCookieCallbacks.set(callbackID, WTFMove(callback));
1277
1278     page->process().send(Messages::WebAutomationSessionProxy::DeleteCookie(page->pageID(), mainFrameID, cookieName, callbackID), 0);
1279 }
1280
1281 void WebAutomationSession::didDeleteCookie(uint64_t callbackID, const String& errorType)
1282 {
1283     auto callback = m_deleteCookieCallbacks.take(callbackID);
1284     if (!callback)
1285         return;
1286
1287     if (!errorType.isEmpty()) {
1288         callback->sendFailure(STRING_FOR_PREDEFINED_ERROR_MESSAGE(errorType));
1289         return;
1290     }
1291
1292     callback->sendSuccess();
1293 }
1294
1295 static String domainByAddingDotPrefixIfNeeded(String domain)
1296 {
1297     if (domain[0] != '.') {
1298         // RFC 2965: If an explicitly specified value does not start with a dot, the user agent supplies a leading dot.
1299         // Assume that any host that ends with a digit is trying to be an IP address.
1300         if (!URL::hostIsIPAddress(domain))
1301             return makeString('.', domain);
1302     }
1303     
1304     return domain;
1305 }
1306     
1307 void WebAutomationSession::addSingleCookie(const String& browsingContextHandle, const JSON::Object& cookieObject, Ref<AddSingleCookieCallback>&& callback)
1308 {
1309     WebPageProxy* page = webPageProxyForHandle(browsingContextHandle);
1310     if (!page)
1311         ASYNC_FAIL_WITH_PREDEFINED_ERROR(WindowNotFound);
1312
1313     URL activeURL = URL(URL(), page->pageLoadState().activeURL());
1314     ASSERT(activeURL.isValid());
1315
1316     WebCore::Cookie cookie;
1317
1318     if (!cookieObject.getString("name"_s, cookie.name))
1319         ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(MissingParameter, "The parameter 'name' was not found.");
1320
1321     if (!cookieObject.getString("value"_s, cookie.value))
1322         ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(MissingParameter, "The parameter 'value' was not found.");
1323
1324     String domain;
1325     if (!cookieObject.getString("domain"_s, domain))
1326         ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(MissingParameter, "The parameter 'domain' was not found.");
1327
1328     // Inherit the domain/host from the main frame's URL if it is not explicitly set.
1329     if (domain.isEmpty())
1330         domain = activeURL.host().toString();
1331
1332     cookie.domain = domainByAddingDotPrefixIfNeeded(domain);
1333
1334     if (!cookieObject.getString("path"_s, cookie.path))
1335         ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(MissingParameter, "The parameter 'path' was not found.");
1336
1337     double expires;
1338     if (!cookieObject.getDouble("expires"_s, expires))
1339         ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(MissingParameter, "The parameter 'expires' was not found.");
1340
1341     cookie.expires = expires * 1000.0;
1342
1343     if (!cookieObject.getBoolean("secure"_s, cookie.secure))
1344         ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(MissingParameter, "The parameter 'secure' was not found.");
1345
1346     if (!cookieObject.getBoolean("session"_s, cookie.session))
1347         ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(MissingParameter, "The parameter 'session' was not found.");
1348
1349     if (!cookieObject.getBoolean("httpOnly"_s, cookie.httpOnly))
1350         ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(MissingParameter, "The parameter 'httpOnly' was not found.");
1351
1352     WebCookieManagerProxy* cookieManager = m_processPool->supplement<WebCookieManagerProxy>();
1353
1354     // FIXME: Using activeURL here twice is basically saying "this is always in the context of the main document"
1355     // which probably isn't accurate.
1356     cookieManager->setCookies(page->websiteDataStore().sessionID(), { cookie }, activeURL, activeURL, [callback = callback.copyRef()](CallbackBase::Error error) {
1357         if (error == CallbackBase::Error::None)
1358             callback->sendSuccess();
1359         else
1360             callback->sendFailure(STRING_FOR_PREDEFINED_ERROR_NAME(InternalError));
1361     });
1362 }
1363
1364 void WebAutomationSession::deleteAllCookies(ErrorString& errorString, const String& browsingContextHandle)
1365 {
1366     WebPageProxy* page = webPageProxyForHandle(browsingContextHandle);
1367     if (!page)
1368         SYNC_FAIL_WITH_PREDEFINED_ERROR(WindowNotFound);
1369
1370     URL activeURL = URL(URL(), page->pageLoadState().activeURL());
1371     ASSERT(activeURL.isValid());
1372
1373     WebCookieManagerProxy* cookieManager = m_processPool->supplement<WebCookieManagerProxy>();
1374     cookieManager->deleteCookiesForHostname(page->websiteDataStore().sessionID(), domainByAddingDotPrefixIfNeeded(activeURL.host().toString()));
1375 }
1376
1377 void WebAutomationSession::getSessionPermissions(ErrorString&, RefPtr<JSON::ArrayOf<Inspector::Protocol::Automation::SessionPermissionData>>& out_permissions)
1378 {
1379     auto permissionsObjectArray = JSON::ArrayOf<Inspector::Protocol::Automation::SessionPermissionData>::create();
1380     auto getUserMediaPermissionObject = Inspector::Protocol::Automation::SessionPermissionData::create()
1381         .setPermission(Inspector::Protocol::Automation::SessionPermission::GetUserMedia)
1382         .setValue(m_permissionForGetUserMedia)
1383         .release();
1384
1385     permissionsObjectArray->addItem(WTFMove(getUserMediaPermissionObject));
1386     out_permissions = WTFMove(permissionsObjectArray);
1387 }
1388
1389 void WebAutomationSession::setSessionPermissions(ErrorString& errorString, const JSON::Array& permissions)
1390 {
1391     for (auto it = permissions.begin(); it != permissions.end(); ++it) {
1392         RefPtr<JSON::Object> permission;
1393         if (!it->get()->asObject(permission))
1394             SYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(InvalidParameter, "The parameter 'permissions' is invalid.");
1395
1396         String permissionName;
1397         if (!permission->getString("permission"_s, permissionName))
1398             SYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(InvalidParameter, "The parameter 'permission' is missing or invalid.");
1399
1400         auto parsedPermissionName = Inspector::Protocol::AutomationHelpers::parseEnumValueFromString<Inspector::Protocol::Automation::SessionPermission>(permissionName);
1401         if (!parsedPermissionName)
1402             SYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(InvalidParameter, "The parameter 'permission' has an unknown value.");
1403
1404         bool permissionValue;
1405         if (!permission->getBoolean("value"_s, permissionValue))
1406             SYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(InvalidParameter, "The parameter 'value' is missing or invalid.");
1407
1408         switch (parsedPermissionName.value()) {
1409         case Inspector::Protocol::Automation::SessionPermission::GetUserMedia:
1410             m_permissionForGetUserMedia = permissionValue;
1411             break;
1412         }
1413     }
1414 }
1415
1416 bool WebAutomationSession::shouldAllowGetUserMediaForPage(const WebPageProxy&) const
1417 {
1418     return m_permissionForGetUserMedia;
1419 }
1420
1421 bool WebAutomationSession::isSimulatingUserInteraction() const
1422 {
1423     if (!m_pendingMouseEventsFlushedCallbacksPerPage.isEmpty())
1424         return true;
1425     if (!m_pendingKeyboardEventsFlushedCallbacksPerPage.isEmpty())
1426         return true;
1427
1428     return false;
1429 }
1430
1431 SimulatedInputDispatcher& WebAutomationSession::inputDispatcherForPage(WebPageProxy& page)
1432 {
1433     return m_inputDispatchersByPage.ensure(page.pageID(), [&] {
1434         return SimulatedInputDispatcher::create(page, *this);
1435     }).iterator->value;
1436 }
1437
1438 SimulatedInputSource* WebAutomationSession::inputSourceForType(SimulatedInputSourceType type) const
1439 {
1440     // FIXME: this should use something like Vector's findMatching().
1441     for (auto& inputSource : m_inputSources) {
1442         if (inputSource->type == type)
1443             return &inputSource.get();
1444     }
1445
1446     return nullptr;
1447 }
1448
1449 // SimulatedInputDispatcher::Client API
1450 void WebAutomationSession::viewportInViewCenterPointOfElement(WebPageProxy& page, uint64_t frameID, const String& nodeHandle, Function<void (std::optional<WebCore::IntPoint>, std::optional<AutomationCommandError>)>&& completionHandler)
1451 {
1452     // Start at 3 and use only odd numbers to not conflict with m_nextComputeElementLayoutCallbackID.
1453     uint64_t callbackID = m_nextViewportInViewCenterPointOfElementCallbackID += 2;
1454     m_viewportInViewCenterPointOfElementCallbacks.set(callbackID, WTFMove(completionHandler));
1455
1456     page.process().send(Messages::WebAutomationSessionProxy::ComputeElementLayout(page.pageID(), frameID, nodeHandle, false, CoordinateSystem::LayoutViewport, callbackID), 0);
1457 }
1458
1459 void WebAutomationSession::simulateMouseInteraction(WebPageProxy& page, MouseInteraction interaction, WebMouseEvent::Button mouseButton, const WebCore::IntPoint& locationInViewport, CompletionHandler<void(std::optional<AutomationCommandError>)>&& completionHandler)
1460 {
1461     WebCore::IntPoint locationInView = WebCore::IntPoint(locationInViewport.x(), locationInViewport.y() + page.topContentInset());
1462     page.getWindowFrameWithCallback([this, protectedThis = makeRef(*this), completionHandler = WTFMove(completionHandler), page = makeRef(page), interaction, mouseButton, locationInView](WebCore::FloatRect windowFrame) mutable {
1463         auto clippedX = std::min(std::max(0.0f, (float)locationInView.x()), windowFrame.size().width());
1464         auto clippedY = std::min(std::max(0.0f, (float)locationInView.y()), windowFrame.size().height());
1465         if (clippedX != locationInView.x() || clippedY != locationInView.y()) {
1466             completionHandler(AUTOMATION_COMMAND_ERROR_WITH_NAME(TargetOutOfBounds));
1467             return;
1468         }
1469
1470         // Bridge the flushed callback to our command's completion handler.
1471         auto mouseEventsFlushedCallback = [completionHandler = WTFMove(completionHandler)](std::optional<AutomationCommandError> error) mutable {
1472             completionHandler(error);
1473         };
1474
1475         auto& callbackInMap = m_pendingMouseEventsFlushedCallbacksPerPage.add(page->pageID(), nullptr).iterator->value;
1476         if (callbackInMap)
1477             callbackInMap(AUTOMATION_COMMAND_ERROR_WITH_NAME(Timeout));
1478         callbackInMap = WTFMove(mouseEventsFlushedCallback);
1479
1480         platformSimulateMouseInteraction(page, interaction, mouseButton, locationInView, (WebEvent::Modifiers)m_currentModifiers);
1481
1482         // If the event does not hit test anything in the window, then it may not have been delivered.
1483         if (callbackInMap && !page->isProcessingMouseEvents()) {
1484             auto callbackToCancel = m_pendingMouseEventsFlushedCallbacksPerPage.take(page->pageID());
1485             callbackToCancel(std::nullopt);
1486         }
1487
1488         // Otherwise, wait for mouseEventsFlushedCallback to run when all events are handled.
1489     });
1490 }
1491
1492 void WebAutomationSession::simulateKeyboardInteraction(WebPageProxy& page, KeyboardInteraction interaction, WTF::Variant<VirtualKey, CharKey>&& key, CompletionHandler<void(std::optional<AutomationCommandError>)>&& completionHandler)
1493 {
1494     // Bridge the flushed callback to our command's completion handler.
1495     auto keyboardEventsFlushedCallback = [completionHandler = WTFMove(completionHandler)](std::optional<AutomationCommandError> error) mutable {
1496         completionHandler(error);
1497     };
1498
1499     auto& callbackInMap = m_pendingKeyboardEventsFlushedCallbacksPerPage.add(page.pageID(), nullptr).iterator->value;
1500     if (callbackInMap)
1501         callbackInMap(AUTOMATION_COMMAND_ERROR_WITH_NAME(Timeout));
1502     callbackInMap = WTFMove(keyboardEventsFlushedCallback);
1503
1504     platformSimulateKeyboardInteraction(page, interaction, WTFMove(key));
1505
1506     // If the interaction does not generate any events, then do not wait for events to be flushed.
1507     // This happens in some corner cases on macOS, such as releasing a key while Command is pressed.
1508     if (callbackInMap && !page.isProcessingKeyboardEvents()) {
1509         auto callbackToCancel = m_pendingKeyboardEventsFlushedCallbacksPerPage.take(page.pageID());
1510         callbackToCancel(std::nullopt);
1511     }
1512
1513     // Otherwise, wait for keyboardEventsFlushedCallback to run when all events are handled.
1514 }
1515
1516 #if USE(APPKIT) || PLATFORM(GTK) || PLATFORM(WPE)
1517 static WebEvent::Modifiers protocolModifierToWebEventModifier(Inspector::Protocol::Automation::KeyModifier modifier)
1518 {
1519     switch (modifier) {
1520     case Inspector::Protocol::Automation::KeyModifier::Alt:
1521         return WebEvent::AltKey;
1522     case Inspector::Protocol::Automation::KeyModifier::Meta:
1523         return WebEvent::MetaKey;
1524     case Inspector::Protocol::Automation::KeyModifier::Control:
1525         return WebEvent::ControlKey;
1526     case Inspector::Protocol::Automation::KeyModifier::Shift:
1527         return WebEvent::ShiftKey;
1528     case Inspector::Protocol::Automation::KeyModifier::CapsLock:
1529         return WebEvent::CapsLockKey;
1530     }
1531
1532     RELEASE_ASSERT_NOT_REACHED();
1533 }
1534
1535 static WebMouseEvent::Button protocolMouseButtonToWebMouseEventButton(Inspector::Protocol::Automation::MouseButton button)
1536 {
1537     switch (button) {
1538     case Inspector::Protocol::Automation::MouseButton::None:
1539         return WebMouseEvent::NoButton;
1540     case Inspector::Protocol::Automation::MouseButton::Left:
1541         return WebMouseEvent::LeftButton;
1542     case Inspector::Protocol::Automation::MouseButton::Middle:
1543         return WebMouseEvent::MiddleButton;
1544     case Inspector::Protocol::Automation::MouseButton::Right:
1545         return WebMouseEvent::RightButton;
1546     }
1547
1548     RELEASE_ASSERT_NOT_REACHED();
1549 }
1550 #endif // USE(APPKIT) || PLATFORM(GTK) || PLATFORM(WPE)
1551
1552 void WebAutomationSession::performMouseInteraction(const String& handle, const JSON::Object& requestedPositionObject, const String& mouseButtonString, const String& mouseInteractionString, const JSON::Array& keyModifierStrings, Ref<PerformMouseInteractionCallback>&& callback)
1553 {
1554 #if !USE(APPKIT) && !PLATFORM(GTK) && !PLATFORM(WPE)
1555     ASYNC_FAIL_WITH_PREDEFINED_ERROR(NotImplemented);
1556 #else
1557     WebPageProxy* page = webPageProxyForHandle(handle);
1558     if (!page)
1559         ASYNC_FAIL_WITH_PREDEFINED_ERROR(WindowNotFound);
1560
1561     float x;
1562     if (!requestedPositionObject.getDouble("x"_s, x))
1563         ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(MissingParameter, "The parameter 'x' was not found.");
1564
1565     float y;
1566     if (!requestedPositionObject.getDouble("y"_s, y))
1567         ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(MissingParameter, "The parameter 'y' was not found.");
1568
1569     WebEvent::Modifiers keyModifiers = (WebEvent::Modifiers)0;
1570     for (auto it = keyModifierStrings.begin(); it != keyModifierStrings.end(); ++it) {
1571         String modifierString;
1572         if (!it->get()->asString(modifierString))
1573             ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(InvalidParameter, "The parameter 'modifiers' is invalid.");
1574
1575         auto parsedModifier = Inspector::Protocol::AutomationHelpers::parseEnumValueFromString<Inspector::Protocol::Automation::KeyModifier>(modifierString);
1576         if (!parsedModifier)
1577             ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(InvalidParameter, "A modifier in the 'modifiers' array is invalid.");
1578         WebEvent::Modifiers enumValue = protocolModifierToWebEventModifier(parsedModifier.value());
1579         keyModifiers = (WebEvent::Modifiers)(enumValue | keyModifiers);
1580     }
1581
1582     page->getWindowFrameWithCallback([this, protectedThis = makeRef(*this), callback = WTFMove(callback), page = makeRef(*page), x, y, mouseInteractionString, mouseButtonString, keyModifiers](WebCore::FloatRect windowFrame) mutable {
1583
1584         x = std::min(std::max(0.0f, x), windowFrame.size().width());
1585         y = std::min(std::max(0.0f, y + page->topContentInset()), windowFrame.size().height());
1586
1587         WebCore::IntPoint positionInView = WebCore::IntPoint(static_cast<int>(x), static_cast<int>(y));
1588
1589         auto parsedInteraction = Inspector::Protocol::AutomationHelpers::parseEnumValueFromString<Inspector::Protocol::Automation::MouseInteraction>(mouseInteractionString);
1590         if (!parsedInteraction)
1591             ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(InvalidParameter, "The parameter 'interaction' is invalid.");
1592
1593         auto parsedButton = Inspector::Protocol::AutomationHelpers::parseEnumValueFromString<Inspector::Protocol::Automation::MouseButton>(mouseButtonString);
1594         if (!parsedButton)
1595             ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(InvalidParameter, "The parameter 'button' is invalid.");
1596
1597         auto mouseEventsFlushedCallback = [protectedThis = WTFMove(protectedThis), callback = WTFMove(callback), page = page.copyRef(), x, y](std::optional<AutomationCommandError> error) {
1598             if (error)
1599                 callback->sendFailure(error.value().toProtocolString());
1600             else {
1601                 callback->sendSuccess(Inspector::Protocol::Automation::Point::create()
1602                     .setX(x)
1603                     .setY(y - page->topContentInset())
1604                     .release());
1605             }
1606         };
1607
1608         auto& callbackInMap = m_pendingMouseEventsFlushedCallbacksPerPage.add(page->pageID(), nullptr).iterator->value;
1609         if (callbackInMap)
1610             callbackInMap(AUTOMATION_COMMAND_ERROR_WITH_NAME(Timeout));
1611         callbackInMap = WTFMove(mouseEventsFlushedCallback);
1612
1613         platformSimulateMouseInteraction(page, parsedInteraction.value(), protocolMouseButtonToWebMouseEventButton(parsedButton.value()), positionInView, keyModifiers);
1614
1615         // If the event location was previously clipped and does not hit test anything in the window, then it will not be processed.
1616         // For compatibility with pre-W3C driver implementations, don't make this a hard error; just do nothing silently.
1617         // In W3C-only code paths, we can reject any pointer actions whose coordinates are outside the viewport rect.
1618         if (callbackInMap && !page->isProcessingMouseEvents()) {
1619             auto callbackToCancel = m_pendingMouseEventsFlushedCallbacksPerPage.take(page->pageID());
1620             callbackToCancel(std::nullopt);
1621         }
1622     });
1623 #endif // USE(APPKIT) || PLATFORM(GTK) || PLATFORM(WPE)
1624 }
1625
1626 void WebAutomationSession::performKeyboardInteractions(const String& handle, const JSON::Array& interactions, Ref<PerformKeyboardInteractionsCallback>&& callback)
1627 {
1628 #if !PLATFORM(COCOA) && !PLATFORM(GTK) && !PLATFORM(WPE)
1629     ASYNC_FAIL_WITH_PREDEFINED_ERROR(NotImplemented);
1630 #else
1631     WebPageProxy* page = webPageProxyForHandle(handle);
1632     if (!page)
1633         ASYNC_FAIL_WITH_PREDEFINED_ERROR(WindowNotFound);
1634
1635     if (!interactions.length())
1636         ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(InvalidParameter, "The parameter 'interactions' was not found or empty.");
1637
1638     // Validate all of the parameters before performing any interactions with the browsing context under test.
1639     Vector<WTF::Function<void()>> actionsToPerform;
1640     actionsToPerform.reserveCapacity(interactions.length());
1641
1642     for (auto interaction : interactions) {
1643         RefPtr<JSON::Object> interactionObject;
1644         if (!interaction->asObject(interactionObject))
1645             ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(InvalidParameter, "An interaction in the 'interactions' parameter was invalid.");
1646
1647         String interactionTypeString;
1648         if (!interactionObject->getString("type"_s, interactionTypeString))
1649             ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(InvalidParameter, "An interaction in the 'interactions' parameter is missing the 'type' key.");
1650         auto interactionType = Inspector::Protocol::AutomationHelpers::parseEnumValueFromString<Inspector::Protocol::Automation::KeyboardInteractionType>(interactionTypeString);
1651         if (!interactionType)
1652             ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(InvalidParameter, "An interaction in the 'interactions' parameter has an invalid 'type' key.");
1653
1654         String virtualKeyString;
1655         bool foundVirtualKey = interactionObject->getString("key"_s, virtualKeyString);
1656         if (foundVirtualKey) {
1657             std::optional<VirtualKey> virtualKey = Inspector::Protocol::AutomationHelpers::parseEnumValueFromString<Inspector::Protocol::Automation::VirtualKey>(virtualKeyString);
1658             if (!virtualKey)
1659                 ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(InvalidParameter, "An interaction in the 'interactions' parameter has an invalid 'key' value.");
1660
1661             actionsToPerform.uncheckedAppend([this, page, interactionType, virtualKey] {
1662                 platformSimulateKeyboardInteraction(*page, interactionType.value(), virtualKey.value());
1663             });
1664         }
1665
1666         String keySequence;
1667         bool foundKeySequence = interactionObject->getString("text"_s, keySequence);
1668         if (foundKeySequence) {
1669             switch (interactionType.value()) {
1670             case Inspector::Protocol::Automation::KeyboardInteractionType::KeyPress:
1671             case Inspector::Protocol::Automation::KeyboardInteractionType::KeyRelease:
1672                 // 'KeyPress' and 'KeyRelease' are meant for a virtual key and are not supported for a string (sequence of codepoints).
1673                 ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(InvalidParameter, "An interaction in the 'interactions' parameter has an invalid 'key' value.");
1674
1675             case Inspector::Protocol::Automation::KeyboardInteractionType::InsertByKey:
1676                 actionsToPerform.uncheckedAppend([this, page, keySequence] {
1677                     platformSimulateKeySequence(*page, keySequence);
1678                 });
1679                 break;
1680             }
1681         }
1682
1683         if (!foundVirtualKey && !foundKeySequence)
1684             ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(MissingParameter, "An interaction in the 'interactions' parameter is missing both 'key' and 'text'. One must be provided.");
1685     }
1686
1687     ASSERT(actionsToPerform.size());
1688     if (!actionsToPerform.size())
1689         ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(InternalError, "No actions to perform.");
1690
1691     auto keyboardEventsFlushedCallback = [protectedThis = makeRef(*this), callback = WTFMove(callback), page = makeRef(*page)](std::optional<AutomationCommandError> error) {
1692         if (error)
1693             callback->sendFailure(error.value().toProtocolString());
1694         else
1695             callback->sendSuccess();
1696     };
1697
1698     auto& callbackInMap = m_pendingKeyboardEventsFlushedCallbacksPerPage.add(page->pageID(), nullptr).iterator->value;
1699     if (callbackInMap)
1700         callbackInMap(AUTOMATION_COMMAND_ERROR_WITH_NAME(Timeout));
1701     callbackInMap = WTFMove(keyboardEventsFlushedCallback);
1702
1703     for (auto& action : actionsToPerform)
1704         action();
1705 #endif // PLATFORM(COCOA) || PLATFORM(GTK) || PLATFORM(WPE)
1706 }
1707
1708 #if USE(APPKIT) || PLATFORM(GTK) || PLATFORM(WPE)
1709 static SimulatedInputSourceType simulatedInputSourceTypeFromProtocolSourceType(Inspector::Protocol::Automation::InputSourceType protocolType)
1710 {
1711     switch (protocolType) {
1712     case Inspector::Protocol::Automation::InputSourceType::Null:
1713         return SimulatedInputSourceType::Null;
1714     case Inspector::Protocol::Automation::InputSourceType::Keyboard:
1715         return SimulatedInputSourceType::Keyboard;
1716     case Inspector::Protocol::Automation::InputSourceType::Mouse:
1717         return SimulatedInputSourceType::Mouse;
1718     case Inspector::Protocol::Automation::InputSourceType::Touch:
1719         return SimulatedInputSourceType::Touch;
1720     }
1721
1722     RELEASE_ASSERT_NOT_REACHED();
1723 }
1724 #endif // USE(APPKIT) || PLATFORM(GTK) || PLATFORM(WPE)
1725
1726 void WebAutomationSession::performInteractionSequence(const String& handle, const String* optionalFrameHandle, const JSON::Array& inputSources, const JSON::Array& steps, Ref<WebAutomationSession::PerformInteractionSequenceCallback>&& callback)
1727 {
1728     // This command implements WebKit support for §17.5 Perform Actions.
1729
1730 #if !USE(APPKIT) && !PLATFORM(GTK) && !PLATFORM(WPE)
1731     ASYNC_FAIL_WITH_PREDEFINED_ERROR(NotImplemented);
1732 #else
1733     WebPageProxy* page = webPageProxyForHandle(handle);
1734     if (!page)
1735         ASYNC_FAIL_WITH_PREDEFINED_ERROR(WindowNotFound);
1736
1737     auto frameID = webFrameIDForHandle(optionalFrameHandle ? *optionalFrameHandle : emptyString());
1738     if (!frameID)
1739         ASYNC_FAIL_WITH_PREDEFINED_ERROR(FrameNotFound);
1740
1741     HashMap<String, Ref<SimulatedInputSource>> sourceIdToInputSourceMap;
1742     HashMap<SimulatedInputSourceType, String, WTF::IntHash<SimulatedInputSourceType>, WTF::StrongEnumHashTraits<SimulatedInputSourceType>> typeToSourceIdMap;
1743
1744     // Parse and validate Automation protocol arguments. By this point, the driver has
1745     // already performed the steps in §17.3 Processing Actions Requests.
1746     if (!inputSources.length())
1747         ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(InvalidParameter, "The parameter 'inputSources' was not found or empty.");
1748
1749     for (auto inputSource : inputSources) {
1750         RefPtr<JSON::Object> inputSourceObject;
1751         if (!inputSource->asObject(inputSourceObject))
1752             ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(InvalidParameter, "An input source in the 'inputSources' parameter was invalid.");
1753         
1754         String sourceId;
1755         if (!inputSourceObject->getString("sourceId"_s, sourceId))
1756             ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(InvalidParameter, "An input source in the 'inputSources' parameter is missing a 'sourceId'.");
1757
1758         String sourceType;
1759         if (!inputSourceObject->getString("sourceType"_s, sourceType))
1760             ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(InvalidParameter, "An input source in the 'inputSources' parameter is missing a 'sourceType'.");
1761
1762         auto parsedInputSourceType = Inspector::Protocol::AutomationHelpers::parseEnumValueFromString<Inspector::Protocol::Automation::InputSourceType>(sourceType);
1763         if (!parsedInputSourceType)
1764             ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(InvalidParameter, "An input source in the 'inputSources' parameter has an invalid 'sourceType'.");
1765
1766         SimulatedInputSourceType inputSourceType = simulatedInputSourceTypeFromProtocolSourceType(*parsedInputSourceType);
1767         if (inputSourceType == SimulatedInputSourceType::Touch)
1768             ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(NotImplemented, "Touch input sources are not yet supported.");
1769
1770         if (typeToSourceIdMap.contains(inputSourceType))
1771             ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(InvalidParameter, "Two input sources with the same type were specified.");
1772         if (sourceIdToInputSourceMap.contains(sourceId))
1773             ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(InvalidParameter, "Two input sources with the same sourceId were specified.");
1774
1775         typeToSourceIdMap.add(inputSourceType, sourceId);
1776         sourceIdToInputSourceMap.add(sourceId, *inputSourceForType(inputSourceType));
1777     }
1778
1779     Vector<SimulatedInputKeyFrame> keyFrames;
1780
1781     if (!steps.length())
1782         ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(InvalidParameter, "The parameter 'steps' was not found or empty.");
1783
1784     for (auto step : steps) {
1785         RefPtr<JSON::Object> stepObject;
1786         if (!step->asObject(stepObject))
1787             ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(InvalidParameter, "A step in the 'steps' parameter was not an object.");
1788
1789         RefPtr<JSON::Array> stepStates;
1790         if (!stepObject->getArray("states"_s, stepStates))
1791             ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(InvalidParameter, "A step is missing the 'states' property.");
1792
1793         Vector<SimulatedInputKeyFrame::StateEntry> entries;
1794         entries.reserveCapacity(stepStates->length());
1795
1796         for (auto state : *stepStates) {
1797             RefPtr<JSON::Object> stateObject;
1798             if (!state->asObject(stateObject))
1799                 ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(InvalidParameter, "Encountered a non-object step state.");
1800
1801             String sourceId;
1802             if (!stateObject->getString("sourceId"_s, sourceId))
1803                 ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(InvalidParameter, "Step state lacks required 'sourceId' property.");
1804
1805             if (!sourceIdToInputSourceMap.contains(sourceId))
1806                 ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(InvalidParameter, "Unknown 'sourceId' specified.");
1807
1808             SimulatedInputSource& inputSource = *sourceIdToInputSourceMap.get(sourceId);
1809             SimulatedInputSourceState sourceState { };
1810
1811             String pressedCharKeyString;
1812             if (stateObject->getString("pressedCharKey"_s, pressedCharKeyString))
1813                 sourceState.pressedCharKey = pressedCharKeyString.characterAt(0);
1814
1815             RefPtr<JSON::Array> pressedVirtualKeysArray;
1816             if (stateObject->getArray("pressedVirtualKeys"_s, pressedVirtualKeysArray)) {
1817                 VirtualKeySet pressedVirtualKeys { };
1818
1819                 for (auto it = pressedVirtualKeysArray->begin(); it != pressedVirtualKeysArray->end(); ++it) {
1820                     String pressedVirtualKeyString;
1821                     if (!(*it)->asString(pressedVirtualKeyString))
1822                         ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(InvalidParameter, "Encountered a non-string virtual key value.");
1823
1824                     std::optional<VirtualKey> parsedVirtualKey = Inspector::Protocol::AutomationHelpers::parseEnumValueFromString<Inspector::Protocol::Automation::VirtualKey>(pressedVirtualKeyString);
1825                     if (!parsedVirtualKey)
1826                         ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(InvalidParameter, "Encountered an unknown virtual key value.");
1827                     else
1828                         pressedVirtualKeys.add(parsedVirtualKey.value());
1829                 }
1830
1831                 sourceState.pressedVirtualKeys = pressedVirtualKeys;
1832             }
1833
1834             String pressedButtonString;
1835             if (stateObject->getString("pressedButton"_s, pressedButtonString)) {
1836                 auto protocolButton = Inspector::Protocol::AutomationHelpers::parseEnumValueFromString<Inspector::Protocol::Automation::MouseButton>(pressedButtonString);
1837                 sourceState.pressedMouseButton = protocolMouseButtonToWebMouseEventButton(protocolButton.value_or(Inspector::Protocol::Automation::MouseButton::None));
1838             }
1839
1840             String originString;
1841             if (stateObject->getString("origin"_s, originString))
1842                 sourceState.origin = Inspector::Protocol::AutomationHelpers::parseEnumValueFromString<Inspector::Protocol::Automation::MouseMoveOrigin>(originString);
1843
1844             if (sourceState.origin && sourceState.origin.value() == Inspector::Protocol::Automation::MouseMoveOrigin::Element) {
1845                 String nodeHandleString;
1846                 if (!stateObject->getString("nodeHandle"_s, nodeHandleString))
1847                     ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(InvalidParameter, "Node handle not provided for 'Element' origin");
1848                 sourceState.nodeHandle = nodeHandleString;
1849             }
1850
1851             RefPtr<JSON::Object> locationObject;
1852             if (stateObject->getObject("location"_s, locationObject)) {
1853                 int x, y;
1854                 if (locationObject->getInteger("x"_s, x) && locationObject->getInteger("y"_s, y))
1855                     sourceState.location = WebCore::IntPoint(x, y);
1856             }
1857
1858             int parsedDuration;
1859             if (stateObject->getInteger("duration"_s, parsedDuration))
1860                 sourceState.duration = Seconds::fromMilliseconds(parsedDuration);
1861
1862             entries.uncheckedAppend(std::pair<SimulatedInputSource&, SimulatedInputSourceState> { inputSource, sourceState });
1863         }
1864         
1865         keyFrames.append(SimulatedInputKeyFrame(WTFMove(entries)));
1866     }
1867
1868     SimulatedInputDispatcher& inputDispatcher = inputDispatcherForPage(*page);
1869     if (inputDispatcher.isActive()) {
1870         ASSERT_NOT_REACHED();
1871         ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(InternalError, "A previous interaction is still underway.");
1872     }
1873
1874     // Delegate the rest of §17.4 Dispatching Actions to the dispatcher.
1875     inputDispatcher.run(frameID.value(), WTFMove(keyFrames), m_inputSources, [protectedThis = makeRef(*this), callback = WTFMove(callback)](std::optional<AutomationCommandError> error) {
1876         if (error)
1877             callback->sendFailure(error.value().toProtocolString());
1878         else
1879             callback->sendSuccess();
1880     });
1881 #endif // PLATFORM(COCOA) || PLATFORM(GTK) || PLATFORM(WPE)
1882 }
1883
1884 void WebAutomationSession::cancelInteractionSequence(const String& handle, const String* optionalFrameHandle, Ref<CancelInteractionSequenceCallback>&& callback)
1885 {
1886     // This command implements WebKit support for §17.6 Release Actions.
1887
1888 #if !USE(APPKIT) && !PLATFORM(GTK) && !PLATFORM(WPE)
1889     ASYNC_FAIL_WITH_PREDEFINED_ERROR(NotImplemented);
1890 #else
1891     WebPageProxy* page = webPageProxyForHandle(handle);
1892     if (!page)
1893         ASYNC_FAIL_WITH_PREDEFINED_ERROR(WindowNotFound);
1894
1895     auto frameID = webFrameIDForHandle(optionalFrameHandle ? *optionalFrameHandle : emptyString());
1896     if (!frameID)
1897         ASYNC_FAIL_WITH_PREDEFINED_ERROR(FrameNotFound);
1898
1899     Vector<SimulatedInputKeyFrame> keyFrames({ SimulatedInputKeyFrame::keyFrameToResetInputSources(m_inputSources) });
1900     SimulatedInputDispatcher& inputDispatcher = inputDispatcherForPage(*page);
1901     inputDispatcher.cancel();
1902     
1903     inputDispatcher.run(frameID.value(), WTFMove(keyFrames), m_inputSources, [protectedThis = makeRef(*this), callback = WTFMove(callback)](std::optional<AutomationCommandError> error) {
1904         if (error)
1905             callback->sendFailure(error.value().toProtocolString());
1906         else
1907             callback->sendSuccess();
1908     });
1909 #endif // PLATFORM(COCOA) || PLATFORM(GTK) || PLATFORM(WPE)
1910 }
1911
1912 void WebAutomationSession::takeScreenshot(const String& handle, const String* optionalFrameHandle, const String* optionalNodeHandle, const bool* optionalScrollIntoViewIfNeeded, const bool* optionalClipToViewport, Ref<TakeScreenshotCallback>&& callback)
1913 {
1914     WebPageProxy* page = webPageProxyForHandle(handle);
1915     if (!page)
1916         ASYNC_FAIL_WITH_PREDEFINED_ERROR(WindowNotFound);
1917
1918     std::optional<uint64_t> frameID = webFrameIDForHandle(optionalFrameHandle ? *optionalFrameHandle : emptyString());
1919     if (!frameID)
1920         ASYNC_FAIL_WITH_PREDEFINED_ERROR(FrameNotFound);
1921
1922     bool scrollIntoViewIfNeeded = optionalScrollIntoViewIfNeeded ? *optionalScrollIntoViewIfNeeded : false;
1923     String nodeHandle = optionalNodeHandle ? *optionalNodeHandle : emptyString();
1924     bool clipToViewport = optionalClipToViewport ? *optionalClipToViewport : false;
1925
1926     uint64_t callbackID = m_nextScreenshotCallbackID++;
1927     m_screenshotCallbacks.set(callbackID, WTFMove(callback));
1928
1929     page->process().send(Messages::WebAutomationSessionProxy::TakeScreenshot(page->pageID(), frameID.value(), nodeHandle, scrollIntoViewIfNeeded, clipToViewport, callbackID), 0);
1930 }
1931
1932 void WebAutomationSession::didTakeScreenshot(uint64_t callbackID, const ShareableBitmap::Handle& imageDataHandle, const String& errorType)
1933 {
1934     auto callback = m_screenshotCallbacks.take(callbackID);
1935     if (!callback)
1936         return;
1937
1938     if (!errorType.isEmpty()) {
1939         callback->sendFailure(STRING_FOR_PREDEFINED_ERROR_MESSAGE(errorType));
1940         return;
1941     }
1942
1943     std::optional<String> base64EncodedData = platformGetBase64EncodedPNGData(imageDataHandle);
1944     if (!base64EncodedData)
1945         ASYNC_FAIL_WITH_PREDEFINED_ERROR(InternalError);
1946
1947     callback->sendSuccess(base64EncodedData.value());
1948 }
1949
1950 // Platform-dependent Implementation Stubs.
1951
1952 #if !PLATFORM(MAC) && !PLATFORM(GTK) && !PLATFORM(WPE)
1953 void WebAutomationSession::platformSimulateMouseInteraction(WebPageProxy&, MouseInteraction, WebMouseEvent::Button, const WebCore::IntPoint&, WebEvent::Modifiers)
1954 {
1955 }
1956 #endif // !PLATFORM(MAC) && !PLATFORM(GTK) && !PLATFORM(WPE)
1957
1958 #if !PLATFORM(COCOA) && !PLATFORM(GTK) && !PLATFORM(WPE)
1959
1960
1961 void WebAutomationSession::platformSimulateKeyboardInteraction(WebPageProxy&, KeyboardInteraction, WTF::Variant<VirtualKey, CharKey>&&)
1962 {
1963 }
1964 #endif // !PLATFORM(COCOA) && !PLATFORM(GTK) && !PLATFORM(WPE)
1965
1966 #if !PLATFORM(COCOA) && !USE(CAIRO)
1967 std::optional<String> WebAutomationSession::platformGetBase64EncodedPNGData(const ShareableBitmap::Handle&)
1968 {
1969     return std::nullopt;
1970 }
1971 #endif // !PLATFORM(COCOA) && !USE(CAIRO)
1972
1973 } // namespace WebKit