26a74e9c6541b652faeda6f3ad45e69df2bfdf3f
[WebKit-https.git] / Source / WebKit / UIProcess / Automation / SimulatedInputDispatcher.cpp
1 /*
2  * Copyright (C) 2018, 2019 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
14  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
15  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
17  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
18  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
19  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
20  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
21  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
22  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
23  * THE POSSIBILITY OF SUCH DAMAGE.
24  */
25
26 #include "config.h"
27 #include "SimulatedInputDispatcher.h"
28
29 #if ENABLE(WEBDRIVER_ACTIONS_API)
30
31 #include "AutomationProtocolObjects.h"
32 #include "Logging.h"
33 #include "WebAutomationSession.h"
34 #include "WebAutomationSessionMacros.h"
35
36 namespace WebKit {
37
38 SimulatedInputSourceState SimulatedInputSourceState::emptyStateForSourceType(SimulatedInputSourceType type)
39 {
40     SimulatedInputSourceState result { };
41     switch (type) {
42     case SimulatedInputSourceType::Null:
43     case SimulatedInputSourceType::Keyboard:
44         break;
45     case SimulatedInputSourceType::Mouse:
46     case SimulatedInputSourceType::Touch:
47         result.location = WebCore::IntPoint();
48     }
49
50     return result;
51 }
52
53
54 SimulatedInputKeyFrame::SimulatedInputKeyFrame(Vector<StateEntry>&& entries)
55     : states(WTFMove(entries))
56 {
57 }
58
59 Seconds SimulatedInputKeyFrame::maximumDuration() const
60 {
61     // The "compute the tick duration" algorithm (§17.4 Dispatching Actions).
62     Seconds result;
63     for (auto& entry : states)
64         result = std::max(result, entry.second.duration.valueOr(Seconds(0)));
65     
66     return result;
67 }
68
69 SimulatedInputKeyFrame SimulatedInputKeyFrame::keyFrameFromStateOfInputSources(HashSet<Ref<SimulatedInputSource>>& inputSources)
70 {
71     // The client of this class is required to intern SimulatedInputSource instances if the last state
72     // from the previous command should be used as the inital state for the next command. This is the
73     // case for Perform Actions and Release Actions, but not Element Click or Element Send Keys.
74     Vector<SimulatedInputKeyFrame::StateEntry> entries;
75     entries.reserveCapacity(inputSources.size());
76
77     for (auto& inputSource : inputSources)
78         entries.uncheckedAppend(std::pair<SimulatedInputSource&, SimulatedInputSourceState> { inputSource.get(), inputSource->state });
79
80     return SimulatedInputKeyFrame(WTFMove(entries));
81 }
82
83 SimulatedInputKeyFrame SimulatedInputKeyFrame::keyFrameToResetInputSources(HashSet<Ref<SimulatedInputSource>>& inputSources)
84 {
85     Vector<SimulatedInputKeyFrame::StateEntry> entries;
86     entries.reserveCapacity(inputSources.size());
87
88     for (auto& inputSource : inputSources)
89         entries.uncheckedAppend(std::pair<SimulatedInputSource&, SimulatedInputSourceState> { inputSource.get(), SimulatedInputSourceState::emptyStateForSourceType(inputSource->type) });
90
91     return SimulatedInputKeyFrame(WTFMove(entries));
92 }
93     
94 SimulatedInputDispatcher::SimulatedInputDispatcher(WebPageProxy& page, SimulatedInputDispatcher::Client& client)
95     : m_page(page)
96     , m_client(client)
97     , m_keyFrameTransitionDurationTimer(RunLoop::current(), this, &SimulatedInputDispatcher::keyFrameTransitionDurationTimerFired)
98 {
99 }
100
101 SimulatedInputDispatcher::~SimulatedInputDispatcher()
102 {
103     ASSERT(!m_runCompletionHandler);
104     ASSERT(!m_keyFrameTransitionDurationTimer.isActive());
105 }
106
107 bool SimulatedInputDispatcher::isActive() const
108 {
109     return !!m_runCompletionHandler;
110 }
111
112 void SimulatedInputDispatcher::keyFrameTransitionDurationTimerFired()
113 {
114     ASSERT(m_keyFrameTransitionCompletionHandler);
115
116     m_keyFrameTransitionDurationTimer.stop();
117
118     LOG(Automation, "SimulatedInputDispatcher[%p]: timer finished for transition between keyframes: %d --> %d", this, m_keyframeIndex - 1, m_keyframeIndex);
119
120     if (isKeyFrameTransitionComplete()) {
121         auto finish = std::exchange(m_keyFrameTransitionCompletionHandler, nullptr);
122         finish(WTF::nullopt);
123     }
124 }
125
126 bool SimulatedInputDispatcher::isKeyFrameTransitionComplete() const
127 {
128     ASSERT(m_keyframeIndex < m_keyframes.size());
129
130     if (m_inputSourceStateIndex < m_keyframes[m_keyframeIndex].states.size())
131         return false;
132
133     if (m_keyFrameTransitionDurationTimer.isActive())
134         return false;
135
136     return true;
137 }
138
139 void SimulatedInputDispatcher::transitionToNextKeyFrame()
140 {
141     ++m_keyframeIndex;
142     if (m_keyframeIndex == m_keyframes.size()) {
143         finishDispatching(WTF::nullopt);
144         return;
145     }
146
147     transitionBetweenKeyFrames(m_keyframes[m_keyframeIndex - 1], m_keyframes[m_keyframeIndex], [this, protectedThis = makeRef(*this)](Optional<AutomationCommandError> error) {
148         if (error) {
149             finishDispatching(error);
150             return;
151         }
152
153         transitionToNextKeyFrame();
154     });
155 }
156
157 void SimulatedInputDispatcher::transitionToNextInputSourceState()
158 {
159     if (isKeyFrameTransitionComplete()) {
160         auto finish = std::exchange(m_keyFrameTransitionCompletionHandler, nullptr);
161         finish(WTF::nullopt);
162         return;
163     }
164
165     // In this case, transitions are done but we need to wait for the tick timer.
166     if (m_inputSourceStateIndex == m_keyframes[m_keyframeIndex].states.size())
167         return;
168
169     auto& nextKeyFrame = m_keyframes[m_keyframeIndex];
170     auto& postStateEntry = nextKeyFrame.states[m_inputSourceStateIndex];
171     SimulatedInputSource& inputSource = postStateEntry.first;
172
173     transitionInputSourceToState(inputSource, postStateEntry.second, [this, protectedThis = makeRef(*this)](Optional<AutomationCommandError> error) {
174         if (error) {
175             auto finish = std::exchange(m_keyFrameTransitionCompletionHandler, nullptr);
176             finish(error);
177             return;
178         }
179
180         // Perform state transitions in the order specified by the currentKeyFrame.
181         ++m_inputSourceStateIndex;
182
183         transitionToNextInputSourceState();
184     });
185 }
186
187 void SimulatedInputDispatcher::transitionBetweenKeyFrames(const SimulatedInputKeyFrame& a, const SimulatedInputKeyFrame& b, AutomationCompletionHandler&& completionHandler)
188 {
189     m_inputSourceStateIndex = 0;
190
191     // The "dispatch tick actions" algorithm (§17.4 Dispatching Actions).
192     m_keyFrameTransitionCompletionHandler = WTFMove(completionHandler);
193     m_keyFrameTransitionDurationTimer.startOneShot(b.maximumDuration());
194
195     LOG(Automation, "SimulatedInputDispatcher[%p]: started transition between keyframes: %d --> %d", this, m_keyframeIndex - 1, m_keyframeIndex);
196     LOG(Automation, "SimulatedInputDispatcher[%p]: timer started to ensure minimum duration of %.2f seconds for transition %d --> %d", this, b.maximumDuration().value(), m_keyframeIndex - 1, m_keyframeIndex);
197
198     transitionToNextInputSourceState();
199 }
200
201 void SimulatedInputDispatcher::resolveLocation(const WebCore::IntPoint& currentLocation, Optional<WebCore::IntPoint> location, MouseMoveOrigin origin, Optional<String> nodeHandle, Function<void (Optional<WebCore::IntPoint>, Optional<AutomationCommandError>)>&& completionHandler)
202 {
203     if (!location) {
204         completionHandler(currentLocation, WTF::nullopt);
205         return;
206     }
207
208     switch (origin) {
209     case MouseMoveOrigin::Viewport:
210         completionHandler(location.value(), WTF::nullopt);
211         break;
212     case MouseMoveOrigin::Pointer: {
213         WebCore::IntPoint destination(currentLocation);
214         destination.moveBy(location.value());
215         completionHandler(destination, WTF::nullopt);
216         break;
217     }
218     case MouseMoveOrigin::Element: {
219         m_client.viewportInViewCenterPointOfElement(m_page, m_frameID.value(), nodeHandle.value(), [destination = location.value(), completionHandler = WTFMove(completionHandler)](Optional<WebCore::IntPoint> inViewCenterPoint, Optional<AutomationCommandError> error) mutable {
220             if (error) {
221                 completionHandler(WTF::nullopt, error);
222                 return;
223             }
224
225             if (!inViewCenterPoint) {
226                 completionHandler(WTF::nullopt, AUTOMATION_COMMAND_ERROR_WITH_NAME(ElementNotInteractable));
227                 return;
228             }
229
230             destination.moveBy(inViewCenterPoint.value());
231             completionHandler(destination, WTF::nullopt);
232         });
233         break;
234     }
235     }
236 }
237
238 void SimulatedInputDispatcher::transitionInputSourceToState(SimulatedInputSource& inputSource, SimulatedInputSourceState& newState, AutomationCompletionHandler&& completionHandler)
239 {
240     // Make cases and conditionals more readable by aliasing pre/post states as 'a' and 'b'.
241     SimulatedInputSourceState& a = inputSource.state;
242     SimulatedInputSourceState& b = newState;
243
244     LOG(Automation, "SimulatedInputDispatcher[%p]: transition started between input source states: [%d.%d] --> %d.%d", this, m_keyframeIndex - 1, m_inputSourceStateIndex, m_keyframeIndex, m_inputSourceStateIndex);
245
246     AutomationCompletionHandler eventDispatchFinished = [this, &inputSource, &newState, completionHandler = WTFMove(completionHandler)](Optional<AutomationCommandError> error) mutable {
247         if (error) {
248             completionHandler(error);
249             return;
250         }
251
252 #if !LOG_DISABLED
253         LOG(Automation, "SimulatedInputDispatcher[%p]: transition finished between input source states: %d.%d --> [%d.%d]", this, m_keyframeIndex - 1, m_inputSourceStateIndex, m_keyframeIndex, m_inputSourceStateIndex);
254 #else
255         UNUSED_PARAM(this);
256 #endif
257
258         inputSource.state = newState;
259         completionHandler(WTF::nullopt);
260     };
261
262     switch (inputSource.type) {
263     case SimulatedInputSourceType::Null:
264         // The maximum duration is handled at the keyframe level by m_keyFrameTransitionDurationTimer.
265         eventDispatchFinished(WTF::nullopt);
266         break;
267     case SimulatedInputSourceType::Mouse: {
268 #if !ENABLE(WEBDRIVER_MOUSE_INTERACTIONS)
269         RELEASE_ASSERT_NOT_REACHED();
270 #else
271         resolveLocation(a.location.valueOr(WebCore::IntPoint()), b.location, b.origin.valueOr(MouseMoveOrigin::Viewport), b.nodeHandle, [this, &a, &b, eventDispatchFinished = WTFMove(eventDispatchFinished)](Optional<WebCore::IntPoint> location, Optional<AutomationCommandError> error) mutable {
272             if (error) {
273                 eventDispatchFinished(error);
274                 return;
275             }
276
277             if (!location) {
278                 eventDispatchFinished(AUTOMATION_COMMAND_ERROR_WITH_NAME(ElementNotInteractable));
279                 return;
280             }
281
282             b.location = location;
283             // The "dispatch a pointer{Down,Up,Move} action" algorithms (§17.4 Dispatching Actions).
284             if (!a.pressedMouseButton && b.pressedMouseButton) {
285 #if !LOG_DISABLED
286                 String mouseButtonName = Inspector::Protocol::AutomationHelpers::getEnumConstantValue(b.pressedMouseButton.value());
287                 LOG(Automation, "SimulatedInputDispatcher[%p]: simulating MouseDown[button=%s] @ (%d, %d) for transition to %d.%d", this, mouseButtonName.utf8().data(), b.location.value().x(), b.location.value().y(), m_keyframeIndex, m_inputSourceStateIndex);
288 #endif
289                 m_client.simulateMouseInteraction(m_page, MouseInteraction::Down, b.pressedMouseButton.value(), b.location.value(), WTFMove(eventDispatchFinished));
290             } else if (a.pressedMouseButton && !b.pressedMouseButton) {
291 #if !LOG_DISABLED
292                 String mouseButtonName = Inspector::Protocol::AutomationHelpers::getEnumConstantValue(a.pressedMouseButton.value());
293                 LOG(Automation, "SimulatedInputDispatcher[%p]: simulating MouseUp[button=%s] @ (%d, %d) for transition to %d.%d", this, mouseButtonName.utf8().data(), a.location.value().x(), a.location.value().y(), m_keyframeIndex, m_inputSourceStateIndex);
294 #endif
295                 m_client.simulateMouseInteraction(m_page, MouseInteraction::Up, a.pressedMouseButton.value(), b.location.value(), WTFMove(eventDispatchFinished));
296             } else if (a.location != b.location) {
297                 LOG(Automation, "SimulatedInputDispatcher[%p]: simulating MouseMove from (%d, %d) to (%d, %d) for transition to %d.%d", this, a.location.value().x(), a.location.value().y(), b.location.value().x(), b.location.value().y(), m_keyframeIndex, m_inputSourceStateIndex);
298                 // FIXME: This does not interpolate mousemoves per the "perform a pointer move" algorithm (§17.4 Dispatching Actions).
299                 m_client.simulateMouseInteraction(m_page, MouseInteraction::Move, b.pressedMouseButton.valueOr(MouseButton::NoButton), b.location.value(), WTFMove(eventDispatchFinished));
300             } else
301                 eventDispatchFinished(WTF::nullopt);
302         });
303 #endif // ENABLE(WEBDRIVER_MOUSE_INTERACTIONS)
304         break;
305     }
306     case SimulatedInputSourceType::Touch: {
307 #if !ENABLE(WEBDRIVER_TOUCH_INTERACTIONS)
308         RELEASE_ASSERT_NOT_REACHED();
309 #else
310         resolveLocation(a.location.valueOr(WebCore::IntPoint()), b.location, b.origin.valueOr(MouseMoveOrigin::Viewport), b.nodeHandle, [this, &a, &b, eventDispatchFinished = WTFMove(eventDispatchFinished)](Optional<WebCore::IntPoint> location, Optional<AutomationCommandError> error) mutable {
311             if (error) {
312                 eventDispatchFinished(error);
313                 return;
314             }
315
316             if (!location) {
317                 eventDispatchFinished(AUTOMATION_COMMAND_ERROR_WITH_NAME(ElementNotInteractable));
318                 return;
319             }
320
321             b.location = location;
322             // The "dispatch a pointer{Down,Up,Move} action" algorithms (§17.4 Dispatching Actions).
323             if (!a.pressedMouseButton && b.pressedMouseButton) {
324                 LOG(Automation, "SimulatedInputDispatcher[%p]: simulating TouchDown @ (%d, %d) for transition to %d.%d", this, b.location.value().x(), b.location.value().y(), m_keyframeIndex, m_inputSourceStateIndex);
325                 m_client.simulateTouchInteraction(m_page, TouchInteraction::TouchDown, b.location.value(), WTF::nullopt, WTFMove(eventDispatchFinished));
326             } else if (a.pressedMouseButton && !b.pressedMouseButton) {
327                 LOG(Automation, "SimulatedInputDispatcher[%p]: simulating LiftUp @ (%d, %d) for transition to %d.%d", this, a.location.value().x(), a.location.value().y(), m_keyframeIndex, m_inputSourceStateIndex);
328                 m_client.simulateTouchInteraction(m_page, TouchInteraction::LiftUp, b.location.value(), WTF::nullopt, WTFMove(eventDispatchFinished));
329             } else if (a.location != b.location) {
330                 LOG(Automation, "SimulatedInputDispatcher[%p]: simulating MoveTo from (%d, %d) to (%d, %d) for transition to %d.%d", this, a.location.value().x(), a.location.value().y(), b.location.value().x(), b.location.value().y(), m_keyframeIndex, m_inputSourceStateIndex);
331                 m_client.simulateTouchInteraction(m_page, TouchInteraction::MoveTo, b.location.value(), a.duration.valueOr(0_s), WTFMove(eventDispatchFinished));
332             } else
333                 eventDispatchFinished(WTF::nullopt);
334         });
335 #endif // !ENABLE(WEBDRIVER_TOUCH_INTERACTIONS)
336         break;
337     }
338     case SimulatedInputSourceType::Keyboard:
339 #if !ENABLE(WEBDRIVER_KEYBOARD_INTERACTIONS)
340         RELEASE_ASSERT_NOT_REACHED();
341 #else
342         // The "dispatch a key{Down,Up} action" algorithms (§17.4 Dispatching Actions).
343         if (!a.pressedCharKey && b.pressedCharKey) {
344             LOG(Automation, "SimulatedInputDispatcher[%p]: simulating KeyPress[key=%c] for transition to %d.%d", this, b.pressedCharKey.value(), m_keyframeIndex, m_inputSourceStateIndex);
345             m_client.simulateKeyboardInteraction(m_page, KeyboardInteraction::KeyPress, b.pressedCharKey.value(), WTFMove(eventDispatchFinished));
346         } else if (a.pressedCharKey && !b.pressedCharKey) {
347             LOG(Automation, "SimulatedInputDispatcher[%p]: simulating KeyRelease[key=%c] for transition to %d.%d", this, a.pressedCharKey.value(), m_keyframeIndex, m_inputSourceStateIndex);
348             m_client.simulateKeyboardInteraction(m_page, KeyboardInteraction::KeyRelease, a.pressedCharKey.value(), WTFMove(eventDispatchFinished));
349         } else if (a.pressedVirtualKeys != b.pressedVirtualKeys) {
350             bool simulatedAnInteraction = false;
351             for (VirtualKey key : b.pressedVirtualKeys) {
352                 if (!a.pressedVirtualKeys.contains(key)) {
353                     ASSERT_WITH_MESSAGE(!simulatedAnInteraction, "Only one VirtualKey may differ at a time between two input source states.");
354                     if (simulatedAnInteraction)
355                         continue;
356                     simulatedAnInteraction = true;
357 #if !LOG_DISABLED
358                     String virtualKeyName = Inspector::Protocol::AutomationHelpers::getEnumConstantValue(key);
359                     LOG(Automation, "SimulatedInputDispatcher[%p]: simulating KeyPress[key=%s] for transition to %d.%d", this, virtualKeyName.utf8().data(), m_keyframeIndex, m_inputSourceStateIndex);
360 #endif
361                     m_client.simulateKeyboardInteraction(m_page, KeyboardInteraction::KeyPress, key, WTFMove(eventDispatchFinished));
362                 }
363             }
364
365             for (VirtualKey key : a.pressedVirtualKeys) {
366                 if (!b.pressedVirtualKeys.contains(key)) {
367                     ASSERT_WITH_MESSAGE(!simulatedAnInteraction, "Only one VirtualKey may differ at a time between two input source states.");
368                     if (simulatedAnInteraction)
369                         continue;
370                     simulatedAnInteraction = true;
371 #if !LOG_DISABLED
372                     String virtualKeyName = Inspector::Protocol::AutomationHelpers::getEnumConstantValue(key);
373                     LOG(Automation, "SimulatedInputDispatcher[%p]: simulating KeyRelease[key=%s] for transition to %d.%d", this, virtualKeyName.utf8().data(), m_keyframeIndex, m_inputSourceStateIndex);
374 #endif
375                     m_client.simulateKeyboardInteraction(m_page, KeyboardInteraction::KeyRelease, key, WTFMove(eventDispatchFinished));
376                 }
377             }
378         } else
379             eventDispatchFinished(WTF::nullopt);
380 #endif // !ENABLE(WEBDRIVER_KEYBOARD_INTERACTIONS)
381         break;
382     }
383 }
384
385 void SimulatedInputDispatcher::run(uint64_t frameID, Vector<SimulatedInputKeyFrame>&& keyFrames, HashSet<Ref<SimulatedInputSource>>& inputSources, AutomationCompletionHandler&& completionHandler)
386 {
387     ASSERT(!isActive());
388     if (isActive()) {
389         completionHandler(AUTOMATION_COMMAND_ERROR_WITH_NAME(InternalError));
390         return;
391     }
392
393     m_frameID = frameID;
394     m_runCompletionHandler = WTFMove(completionHandler);
395     for (const Ref<SimulatedInputSource>& inputSource : inputSources)
396         m_inputSources.add(inputSource.copyRef());
397
398     // The "dispatch actions" algorithm (§17.4 Dispatching Actions).
399
400     m_keyframes.reserveCapacity(keyFrames.size() + 1);
401     m_keyframes.append(SimulatedInputKeyFrame::keyFrameFromStateOfInputSources(m_inputSources));
402     m_keyframes.appendVector(WTFMove(keyFrames));
403
404     LOG(Automation, "SimulatedInputDispatcher[%p]: starting input simulation using %d keyframes", this, m_keyframeIndex);
405
406     transitionToNextKeyFrame();
407 }
408
409 void SimulatedInputDispatcher::cancel()
410 {
411     // If we were waiting for m_client to finish an interaction and the interaction had an error,
412     // then the rest of the async chain will have been torn down. If we are just waiting on a
413     // dispatch timer, then this will cancel the timer and clear
414
415     if (isActive())
416         finishDispatching(AUTOMATION_COMMAND_ERROR_WITH_NAME(InternalError));
417 }
418
419 void SimulatedInputDispatcher::finishDispatching(Optional<AutomationCommandError> error)
420 {
421     m_keyFrameTransitionDurationTimer.stop();
422
423     LOG(Automation, "SimulatedInputDispatcher[%p]: finished all input simulation at [%u.%u]", this, m_keyframeIndex, m_inputSourceStateIndex);
424
425     auto finish = std::exchange(m_runCompletionHandler, nullptr);
426     m_frameID = WTF::nullopt;
427     m_keyframes.clear();
428     m_inputSources.clear();
429     m_keyframeIndex = 0;
430     m_inputSourceStateIndex = 0;
431
432     finish(error);
433 }
434
435 } // namespace Webkit
436
437 #endif // ENABLE(WEBDRIVER_ACTIONS_API)