dbe8397209da168db119baf460c5f2d8b695ff3f
[WebKit-https.git] / Source / WebKit / UIProcess / Automation / SimulatedInputDispatcher.cpp
1 /*
2  * Copyright (C) 2018 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
14  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
15  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
17  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
18  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
19  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
20  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
21  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
22  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
23  * THE POSSIBILITY OF SUCH DAMAGE.
24  */
25
26 #include "config.h"
27 #include "SimulatedInputDispatcher.h"
28
29 #include "AutomationProtocolObjects.h"
30 #include "Logging.h"
31 #include "WebAutomationSession.h"
32 #include "WebAutomationSessionMacros.h"
33
34 namespace WebKit {
35
36 SimulatedInputSourceState SimulatedInputSourceState::emptyStateForSourceType(SimulatedInputSourceType type)
37 {
38     SimulatedInputSourceState result { };
39     switch (type) {
40     case SimulatedInputSourceType::Null:
41     case SimulatedInputSourceType::Keyboard:
42         break;
43     case SimulatedInputSourceType::Mouse:
44     case SimulatedInputSourceType::Touch:
45         result.location = WebCore::IntPoint();
46     }
47
48     return result;
49 }
50
51
52 SimulatedInputKeyFrame::SimulatedInputKeyFrame(Vector<StateEntry>&& entries)
53     : states(WTFMove(entries))
54 {
55 }
56
57 Seconds SimulatedInputKeyFrame::maximumDuration() const
58 {
59     // The "compute the tick duration" algorithm (§17.4 Dispatching Actions).
60     Seconds result;
61     for (auto& entry : states)
62         result = std::max(result, entry.second.duration.value_or(Seconds(0)));
63     
64     return result;
65 }
66
67 SimulatedInputKeyFrame SimulatedInputKeyFrame::keyFrameFromStateOfInputSources(HashSet<Ref<SimulatedInputSource>>& inputSources)
68 {
69     // The client of this class is required to intern SimulatedInputSource instances if the last state
70     // from the previous command should be used as the inital state for the next command. This is the
71     // case for Perform Actions and Release Actions, but not Element Click or Element Send Keys.
72     Vector<SimulatedInputKeyFrame::StateEntry> entries;
73     entries.reserveCapacity(inputSources.size());
74
75     for (auto& inputSource : inputSources)
76         entries.uncheckedAppend(std::pair<SimulatedInputSource&, SimulatedInputSourceState> { inputSource.get(), inputSource->state });
77
78     return SimulatedInputKeyFrame(WTFMove(entries));
79 }
80
81 SimulatedInputKeyFrame SimulatedInputKeyFrame::keyFrameToResetInputSources(HashSet<Ref<SimulatedInputSource>>& inputSources)
82 {
83     Vector<SimulatedInputKeyFrame::StateEntry> entries;
84     entries.reserveCapacity(inputSources.size());
85
86     for (auto& inputSource : inputSources)
87         entries.uncheckedAppend(std::pair<SimulatedInputSource&, SimulatedInputSourceState> { inputSource.get(), SimulatedInputSourceState::emptyStateForSourceType(inputSource->type) });
88
89     return SimulatedInputKeyFrame(WTFMove(entries));
90 }
91     
92 SimulatedInputDispatcher::SimulatedInputDispatcher(WebPageProxy& page, SimulatedInputDispatcher::Client& client)
93     : m_page(page)
94     , m_client(client)
95     , m_keyFrameTransitionDurationTimer(RunLoop::current(), this, &SimulatedInputDispatcher::keyFrameTransitionDurationTimerFired)
96 {
97 }
98
99 SimulatedInputDispatcher::~SimulatedInputDispatcher()
100 {
101     ASSERT(!m_runCompletionHandler);
102     ASSERT(!m_keyFrameTransitionDurationTimer.isActive());
103 }
104
105 bool SimulatedInputDispatcher::isActive() const
106 {
107     return !!m_runCompletionHandler;
108 }
109
110 void SimulatedInputDispatcher::keyFrameTransitionDurationTimerFired()
111 {
112     ASSERT(m_keyFrameTransitionCompletionHandler);
113
114     m_keyFrameTransitionDurationTimer.stop();
115
116     LOG(Automation, "SimulatedInputDispatcher[%p]: timer finished for transition between keyframes: %d --> %d", this, m_keyframeIndex - 1, m_keyframeIndex);
117
118     if (isKeyFrameTransitionComplete()) {
119         auto finish = std::exchange(m_keyFrameTransitionCompletionHandler, nullptr);
120         finish(WTF::nullopt);
121     }
122 }
123
124 bool SimulatedInputDispatcher::isKeyFrameTransitionComplete() const
125 {
126     ASSERT(m_keyframeIndex < m_keyframes.size());
127
128     if (m_inputSourceStateIndex < m_keyframes[m_keyframeIndex].states.size())
129         return false;
130
131     if (m_keyFrameTransitionDurationTimer.isActive())
132         return false;
133
134     return true;
135 }
136
137 void SimulatedInputDispatcher::transitionToNextKeyFrame()
138 {
139     ++m_keyframeIndex;
140     if (m_keyframeIndex == m_keyframes.size()) {
141         finishDispatching(WTF::nullopt);
142         return;
143     }
144
145     transitionBetweenKeyFrames(m_keyframes[m_keyframeIndex - 1], m_keyframes[m_keyframeIndex], [this, protectedThis = makeRef(*this)](Optional<AutomationCommandError> error) {
146         if (error) {
147             finishDispatching(error);
148             return;
149         }
150
151         transitionToNextKeyFrame();
152     });
153 }
154
155 void SimulatedInputDispatcher::transitionToNextInputSourceState()
156 {
157     if (isKeyFrameTransitionComplete()) {
158         auto finish = std::exchange(m_keyFrameTransitionCompletionHandler, nullptr);
159         finish(WTF::nullopt);
160         return;
161     }
162
163     // In this case, transitions are done but we need to wait for the tick timer.
164     if (m_inputSourceStateIndex == m_keyframes[m_keyframeIndex].states.size())
165         return;
166
167     auto& nextKeyFrame = m_keyframes[m_keyframeIndex];
168     auto& postStateEntry = nextKeyFrame.states[m_inputSourceStateIndex];
169     SimulatedInputSource& inputSource = postStateEntry.first;
170
171     transitionInputSourceToState(inputSource, postStateEntry.second, [this, protectedThis = makeRef(*this)](Optional<AutomationCommandError> error) {
172         if (error) {
173             auto finish = std::exchange(m_keyFrameTransitionCompletionHandler, nullptr);
174             finish(error);
175             return;
176         }
177
178         // Perform state transitions in the order specified by the currentKeyFrame.
179         ++m_inputSourceStateIndex;
180
181         transitionToNextInputSourceState();
182     });
183 }
184
185 void SimulatedInputDispatcher::transitionBetweenKeyFrames(const SimulatedInputKeyFrame& a, const SimulatedInputKeyFrame& b, AutomationCompletionHandler&& completionHandler)
186 {
187     m_inputSourceStateIndex = 0;
188
189     // The "dispatch tick actions" algorithm (§17.4 Dispatching Actions).
190     m_keyFrameTransitionCompletionHandler = WTFMove(completionHandler);
191     m_keyFrameTransitionDurationTimer.startOneShot(b.maximumDuration());
192
193     LOG(Automation, "SimulatedInputDispatcher[%p]: started transition between keyframes: %d --> %d", this, m_keyframeIndex - 1, m_keyframeIndex);
194     LOG(Automation, "SimulatedInputDispatcher[%p]: timer started to ensure minimum duration of %.2f seconds for transition %d --> %d", this, b.maximumDuration().value(), m_keyframeIndex - 1, m_keyframeIndex);
195
196     transitionToNextInputSourceState();
197 }
198
199 void SimulatedInputDispatcher::resolveLocation(const WebCore::IntPoint& currentLocation, Optional<WebCore::IntPoint> location, MouseMoveOrigin origin, Optional<String> nodeHandle, Function<void (Optional<WebCore::IntPoint>, Optional<AutomationCommandError>)>&& completionHandler)
200 {
201     if (!location) {
202         completionHandler(currentLocation, WTF::nullopt);
203         return;
204     }
205
206     switch (origin) {
207     case MouseMoveOrigin::Viewport:
208         completionHandler(location.value(), WTF::nullopt);
209         break;
210     case MouseMoveOrigin::Pointer: {
211         WebCore::IntPoint destination(currentLocation);
212         destination.moveBy(location.value());
213         completionHandler(destination, WTF::nullopt);
214         break;
215     }
216     case MouseMoveOrigin::Element: {
217         m_client.viewportInViewCenterPointOfElement(m_page, m_frameID.value(), nodeHandle.value(), [destination = location.value(), completionHandler = WTFMove(completionHandler)](Optional<WebCore::IntPoint> inViewCenterPoint, Optional<AutomationCommandError> error) mutable {
218             if (error) {
219                 completionHandler(WTF::nullopt, error);
220                 return;
221             }
222
223             ASSERT(inViewCenterPoint);
224             destination.moveBy(inViewCenterPoint.value());
225             completionHandler(destination, WTF::nullopt);
226         });
227         break;
228     }
229     }
230 }
231
232 void SimulatedInputDispatcher::transitionInputSourceToState(SimulatedInputSource& inputSource, SimulatedInputSourceState& newState, AutomationCompletionHandler&& completionHandler)
233 {
234     // Make cases and conditionals more readable by aliasing pre/post states as 'a' and 'b'.
235     SimulatedInputSourceState& a = inputSource.state;
236     SimulatedInputSourceState& b = newState;
237
238     LOG(Automation, "SimulatedInputDispatcher[%p]: transition started between input source states: [%d.%d] --> %d.%d", this, m_keyframeIndex - 1, m_inputSourceStateIndex, m_keyframeIndex, m_inputSourceStateIndex);
239
240     AutomationCompletionHandler eventDispatchFinished = [this, &inputSource, &newState, completionHandler = WTFMove(completionHandler)](Optional<AutomationCommandError> error) mutable {
241         if (error) {
242             completionHandler(error);
243             return;
244         }
245
246 #if !LOG_DISABLED
247         LOG(Automation, "SimulatedInputDispatcher[%p]: transition finished between input source states: %d.%d --> [%d.%d]", this, m_keyframeIndex - 1, m_inputSourceStateIndex, m_keyframeIndex, m_inputSourceStateIndex);
248 #else
249         UNUSED_PARAM(this);
250 #endif
251
252         inputSource.state = newState;
253         completionHandler(WTF::nullopt);
254     };
255
256     switch (inputSource.type) {
257     case SimulatedInputSourceType::Null:
258         // The maximum duration is handled at the keyframe level by m_keyFrameTransitionDurationTimer.
259         eventDispatchFinished(WTF::nullopt);
260         break;
261     case SimulatedInputSourceType::Mouse: {
262         resolveLocation(a.location.value_or(WebCore::IntPoint()), b.location, b.origin.value_or(MouseMoveOrigin::Viewport), b.nodeHandle, [this, &a, &b, eventDispatchFinished = WTFMove(eventDispatchFinished)](Optional<WebCore::IntPoint> location, Optional<AutomationCommandError> error) mutable {
263             if (error) {
264                 eventDispatchFinished(error);
265                 return;
266             }
267             RELEASE_ASSERT(location);
268             b.location = location;
269             // The "dispatch a pointer{Down,Up,Move} action" algorithms (§17.4 Dispatching Actions).
270             if (!a.pressedMouseButton && b.pressedMouseButton) {
271 #if !LOG_DISABLED
272                 String mouseButtonName = Inspector::Protocol::AutomationHelpers::getEnumConstantValue(b.pressedMouseButton.value());
273                 LOG(Automation, "SimulatedInputDispatcher[%p]: simulating MouseDown[button=%s] for transition to %d.%d", this, mouseButtonName.utf8().data(), m_keyframeIndex, m_inputSourceStateIndex);
274 #endif
275                 m_client.simulateMouseInteraction(m_page, MouseInteraction::Down, b.pressedMouseButton.value(), b.location.value(), WTFMove(eventDispatchFinished));
276             } else if (a.pressedMouseButton && !b.pressedMouseButton) {
277 #if !LOG_DISABLED
278                 String mouseButtonName = Inspector::Protocol::AutomationHelpers::getEnumConstantValue(a.pressedMouseButton.value());
279                 LOG(Automation, "SimulatedInputDispatcher[%p]: simulating MouseUp[button=%s] for transition to %d.%d", this, mouseButtonName.utf8().data(), m_keyframeIndex, m_inputSourceStateIndex);
280 #endif
281                 m_client.simulateMouseInteraction(m_page, MouseInteraction::Up, a.pressedMouseButton.value(), b.location.value(), WTFMove(eventDispatchFinished));
282             } else if (a.location != b.location) {
283                 LOG(Automation, "SimulatedInputDispatcher[%p]: simulating MouseMove for transition to %d.%d", this, m_keyframeIndex, m_inputSourceStateIndex);
284                 // FIXME: This does not interpolate mousemoves per the "perform a pointer move" algorithm (§17.4 Dispatching Actions).
285                 m_client.simulateMouseInteraction(m_page, MouseInteraction::Move, b.pressedMouseButton.value_or(MouseButton::NoButton), b.location.value(), WTFMove(eventDispatchFinished));
286             } else
287                 eventDispatchFinished(WTF::nullopt);
288         });
289         break;
290     }
291     case SimulatedInputSourceType::Keyboard:
292         // The "dispatch a key{Down,Up} action" algorithms (§17.4 Dispatching Actions).
293         if (!a.pressedCharKey && b.pressedCharKey) {
294             LOG(Automation, "SimulatedInputDispatcher[%p]: simulating KeyPress[key=%c] for transition to %d.%d", this, b.pressedCharKey.value(), m_keyframeIndex, m_inputSourceStateIndex);
295             m_client.simulateKeyboardInteraction(m_page, KeyboardInteraction::KeyPress, b.pressedCharKey.value(), WTFMove(eventDispatchFinished));
296         } else if (a.pressedCharKey && !b.pressedCharKey) {
297             LOG(Automation, "SimulatedInputDispatcher[%p]: simulating KeyRelease[key=%c] for transition to %d.%d", this, a.pressedCharKey.value(), m_keyframeIndex, m_inputSourceStateIndex);
298             m_client.simulateKeyboardInteraction(m_page, KeyboardInteraction::KeyRelease, a.pressedCharKey.value(), WTFMove(eventDispatchFinished));
299         } else if (a.pressedVirtualKeys != b.pressedVirtualKeys) {
300             bool simulatedAnInteraction = false;
301             for (VirtualKey key : b.pressedVirtualKeys) {
302                 if (!a.pressedVirtualKeys.contains(key)) {
303                     ASSERT_WITH_MESSAGE(!simulatedAnInteraction, "Only one VirtualKey may differ at a time between two input source states.");
304                     if (simulatedAnInteraction)
305                         continue;
306                     simulatedAnInteraction = true;
307 #if !LOG_DISABLED
308                     String virtualKeyName = Inspector::Protocol::AutomationHelpers::getEnumConstantValue(key);
309                     LOG(Automation, "SimulatedInputDispatcher[%p]: simulating KeyPress[key=%s] for transition to %d.%d", this, virtualKeyName.utf8().data(), m_keyframeIndex, m_inputSourceStateIndex);
310 #endif
311                     m_client.simulateKeyboardInteraction(m_page, KeyboardInteraction::KeyPress, key, WTFMove(eventDispatchFinished));
312                 }
313             }
314
315             for (VirtualKey key : a.pressedVirtualKeys) {
316                 if (!b.pressedVirtualKeys.contains(key)) {
317                     ASSERT_WITH_MESSAGE(!simulatedAnInteraction, "Only one VirtualKey may differ at a time between two input source states.");
318                     if (simulatedAnInteraction)
319                         continue;
320                     simulatedAnInteraction = true;
321 #if !LOG_DISABLED
322                     String virtualKeyName = Inspector::Protocol::AutomationHelpers::getEnumConstantValue(key);
323                     LOG(Automation, "SimulatedInputDispatcher[%p]: simulating KeyRelease[key=%s] for transition to %d.%d", this, virtualKeyName.utf8().data(), m_keyframeIndex, m_inputSourceStateIndex);
324 #endif
325                     m_client.simulateKeyboardInteraction(m_page, KeyboardInteraction::KeyRelease, key, WTFMove(eventDispatchFinished));
326                 }
327             }
328         } else
329             eventDispatchFinished(WTF::nullopt);
330         break;
331     case SimulatedInputSourceType::Touch:
332         // Not supported yet.
333         ASSERT_NOT_REACHED();
334         eventDispatchFinished(AUTOMATION_COMMAND_ERROR_WITH_NAME(NotImplemented));
335         break;
336     }
337 }
338
339 void SimulatedInputDispatcher::run(uint64_t frameID, Vector<SimulatedInputKeyFrame>&& keyFrames, HashSet<Ref<SimulatedInputSource>>& inputSources, AutomationCompletionHandler&& completionHandler)
340 {
341     ASSERT(!isActive());
342     if (isActive()) {
343         completionHandler(AUTOMATION_COMMAND_ERROR_WITH_NAME(InternalError));
344         return;
345     }
346
347     m_frameID = frameID;
348     m_runCompletionHandler = WTFMove(completionHandler);
349     for (const Ref<SimulatedInputSource>& inputSource : inputSources)
350         m_inputSources.add(inputSource.copyRef());
351
352     // The "dispatch actions" algorithm (§17.4 Dispatching Actions).
353
354     m_keyframes.reserveCapacity(keyFrames.size() + 1);
355     m_keyframes.append(SimulatedInputKeyFrame::keyFrameFromStateOfInputSources(m_inputSources));
356     m_keyframes.appendVector(WTFMove(keyFrames));
357
358     LOG(Automation, "SimulatedInputDispatcher[%p]: starting input simulation using %d keyframes", this, m_keyframeIndex);
359
360     transitionToNextKeyFrame();
361 }
362
363 void SimulatedInputDispatcher::cancel()
364 {
365     // If we were waiting for m_client to finish an interaction and the interaction had an error,
366     // then the rest of the async chain will have been torn down. If we are just waiting on a
367     // dispatch timer, then this will cancel the timer and clear
368
369     if (isActive())
370         finishDispatching(AUTOMATION_COMMAND_ERROR_WITH_NAME(InternalError));
371 }
372
373 void SimulatedInputDispatcher::finishDispatching(Optional<AutomationCommandError> error)
374 {
375     m_keyFrameTransitionDurationTimer.stop();
376
377     LOG(Automation, "SimulatedInputDispatcher[%p]: finished all input simulation at [%u.%u]", this, m_keyframeIndex, m_inputSourceStateIndex);
378
379     auto finish = std::exchange(m_runCompletionHandler, nullptr);
380     m_frameID = WTF::nullopt;
381     m_keyframes.clear();
382     m_inputSources.clear();
383     m_keyframeIndex = 0;
384     m_inputSourceStateIndex = 0;
385
386     finish(error);
387 }
388
389 } // namespace Webkit