Web Replay: capture and replay keyboard events
authorbburg@apple.com <bburg@apple.com@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Tue, 25 Mar 2014 00:52:46 +0000 (00:52 +0000)
committerbburg@apple.com <bburg@apple.com@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Tue, 25 Mar 2014 00:52:46 +0000 (00:52 +0000)
https://bugs.webkit.org/show_bug.cgi?id=130314

Reviewed by Joseph Pecoraro.

.:

* ManualTests/inspector/replay-keyboard-events.html: Added.

Source/WebCore:

Upstream support for capturing and replaying non-IME keyboard input.
This works similarly to mouse events. It introduces optional fields and
macros to encode/decode them.

Test: ManualTests/inspector/replay-keyboard-events.html

* replay/ReplayInputDispatchMethods.cpp:
(WebCore::HandleKeyPress::dispatch):
* replay/SerializationMethods.cpp:
(JSC::EncodingTraits<NondeterministicInputBase>::encodeValue):
(JSC::EncodingTraits<NondeterministicInputBase>::decodeValue): Switch existing
encode/decode calls to use the shortening macros.
(JSC::EncodingTraits<KeypressCommand>::encodeValue): Added.
(JSC::EncodingTraits<KeypressCommand>::decodeValue): Added.
(JSC::PlatformKeyboardEventAppKit::PlatformKeyboardEventAppKit): Subclass
PlatformKeyboardEvent so that we can set AppKit-specific members not
initialized through the main constructors.
(JSC::EncodingTraits<PlatformKeyboardEvent>::encodeValue): Added.
(JSC::EncodingTraits<PlatformKeyboardEvent>::decodeValue): Added.
* replay/SerializationMethods.h:
* replay/UserInputBridge.cpp:
(WebCore::UserInputBridge::UserInputBridge): initialize m_state inside a guard.
(WebCore::UserInputBridge::handleKeyEvent): fill in the implementation.
* replay/WebInputs.json:

git-svn-id: https://svn.webkit.org/repository/webkit/trunk@166212 268f45cc-cd09-0410-ab3c-d52691b4dbfc

ChangeLog
ManualTests/inspector/replay-keyboard-events.html [new file with mode: 0644]
Source/WebCore/ChangeLog
Source/WebCore/replay/ReplayInputDispatchMethods.cpp
Source/WebCore/replay/SerializationMethods.cpp
Source/WebCore/replay/SerializationMethods.h
Source/WebCore/replay/UserInputBridge.cpp
Source/WebCore/replay/WebInputs.json

index 079d9ef..84b726e 100644 (file)
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,12 @@
+2014-03-24  Brian Burg  <bburg@apple.com>
+
+        Web Replay: capture and replay keyboard events
+        https://bugs.webkit.org/show_bug.cgi?id=130314
+
+        Reviewed by Joseph Pecoraro.
+
+        * ManualTests/inspector/replay-keyboard-events.html: Added.
+
 2014-03-24  Sangyong Park  <sy302.park@gmail.com>
 
         [EFL] Inspector page is not loaded.
diff --git a/ManualTests/inspector/replay-keyboard-events.html b/ManualTests/inspector/replay-keyboard-events.html
new file mode 100644 (file)
index 0000000..4dc3c4c
--- /dev/null
@@ -0,0 +1,104 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN"
+        "http://www.w3.org/TR/html4/strict.dtd">
+<html lang="en">
+<head>
+<script src="./resources/crypto-md5.js"></script>
+<script type="text/javascript" language="javascript" charset="utf-8">
+    document.onkeypress = handleEvent;
+    document.onkeydown = handleEvent;
+    document.onkeyup = handleEvent;
+    window.dumpedEvents = [];
+
+    function handleEvent(event) {
+        var properties = ["type", "eventPhase", "bubbles", "cancelable", "keyIdentifier", "location", "shiftKey", "altKey", "metaKey", "altGraphKey", "keyCode", "charCode"];
+        obj = {};
+        for (var key of properties)
+            obj[key] = event[key];
+
+        dumpedEvents.push(obj);
+
+        var block = createBlock(hex_md5(JSON.stringify(obj)), event);
+        var blocksContainer = document.getElementById("blocks");
+        blocksContainer.appendChild(block);
+
+        var hashLabel = document.getElementById("hash");
+        hash.textContent = hex_md5(JSON.stringify(dumpedEvents));
+    }
+
+    function createBlock(hash, event) {
+
+        function glyphForType(type) {
+            switch (type) {
+            case "keydown": return "D";
+            case "keyup": return "U";
+            case "keypress": return "K";
+            }
+        }
+
+        var color = "#" + hash.substr(0,6);
+        var block = document.createElement("span");
+        var eventTypeLabel = document.createElement("span");
+        eventTypeLabel.className = "event-label";
+        eventTypeLabel.textContent = glyphForType(event.type);
+        block.appendChild(eventTypeLabel);
+
+        var keyLabel = document.createElement("span");
+        keyLabel.className = "key-label";
+        var codepointRegEx = /^U\+\d{4}$/;
+        keyLabel.textContent = event.keyIdentifier.match(codepointRegEx) ? String.fromCharCode(event.keyCode) : event.keyIdentifier;
+        block.appendChild(keyLabel);
+        block.style.backgroundColor = color;
+        return block;
+    }
+
+    function stateHash() {
+        return hex_md5(JSON.stringify(dumpedEvents));
+    }
+</script>
+
+<style type="text/css">
+body {
+    max-width: 800px;
+}
+#blocks {
+    display: -webkit-flex;
+    width: 600px;
+    -webkit-flex-flow: row wrap;
+}
+
+#blocks > span {
+    padding: 2px 2px;
+    border-radius: 4px;
+    margin: 2px;
+    font-size: 12px;
+    font-weight: bold;
+    font-family: sans-serif;
+    color: #fff;
+    text-align: center;
+}
+
+.event-label::before {
+    content: '(';
+}
+
+.event-label::after {
+    content: ') ';
+}
+
+.key-label {
+}
+</style>
+</head>
+<body>
+<p>This page is a manual test for capture and replay of keyboard-related events.</p>
+<p>Below, a block is created for each keyboard event, where the color is derived from a hash of the keyboard event. At the bottom is a cumulative hash of all keyboard event data.</p>
+<hr/>
+<p>
+To test the replay functionality, open the Web Inspector, start capturing, and then type text into the textarea. After some time, stop capturing and then replay. The replayed execution should produce the same sequence of blocks. More importantly, the cumulative hash value should be the same at the end of capturing and at the end of any subsequent replays.</p>
+</p>
+<hr/>
+<form><textarea cols="80" rows="20"></textarea></form>
+<div id="hash"></div>
+<div id="blocks"></div>
+</body>
+</html>
index 93dd76a..6cc01ab 100644 (file)
@@ -1,3 +1,35 @@
+2014-03-24  Brian Burg  <bburg@apple.com>
+
+        Web Replay: capture and replay keyboard events
+        https://bugs.webkit.org/show_bug.cgi?id=130314
+
+        Reviewed by Joseph Pecoraro.
+
+        Upstream support for capturing and replaying non-IME keyboard input.
+        This works similarly to mouse events. It introduces optional fields and
+        macros to encode/decode them.
+
+        Test: ManualTests/inspector/replay-keyboard-events.html
+
+        * replay/ReplayInputDispatchMethods.cpp:
+        (WebCore::HandleKeyPress::dispatch):
+        * replay/SerializationMethods.cpp:
+        (JSC::EncodingTraits<NondeterministicInputBase>::encodeValue):
+        (JSC::EncodingTraits<NondeterministicInputBase>::decodeValue): Switch existing
+        encode/decode calls to use the shortening macros.
+        (JSC::EncodingTraits<KeypressCommand>::encodeValue): Added.
+        (JSC::EncodingTraits<KeypressCommand>::decodeValue): Added.
+        (JSC::PlatformKeyboardEventAppKit::PlatformKeyboardEventAppKit): Subclass
+        PlatformKeyboardEvent so that we can set AppKit-specific members not
+        initialized through the main constructors.
+        (JSC::EncodingTraits<PlatformKeyboardEvent>::encodeValue): Added.
+        (JSC::EncodingTraits<PlatformKeyboardEvent>::decodeValue): Added.
+        * replay/SerializationMethods.h:
+        * replay/UserInputBridge.cpp:
+        (WebCore::UserInputBridge::UserInputBridge): initialize m_state inside a guard.
+        (WebCore::UserInputBridge::handleKeyEvent): fill in the implementation.
+        * replay/WebInputs.json:
+
 2014-03-24  Thiago de Barros Lacerda  <thiago.lacerda@openbossa.org>
 
         Optimizing string construction for type error in JSNavigatorCustom.cpp
index 3855225..1045e65 100644 (file)
@@ -53,6 +53,11 @@ void InitialNavigation::dispatch(ReplayController& controller)
     controller.page().mainFrame().navigationScheduler().scheduleLocationChange(m_securityOrigin.get(), m_url, m_referrer, true, true);
 }
 
+void HandleKeyPress::dispatch(ReplayController& controller)
+{
+    controller.page().userInputBridge().handleKeyEvent(platformEvent(), InputSource::Synthetic);
+}
+
 // User interaction inputs.
 void HandleMouseMove::dispatch(ReplayController& controller)
 {
index e860984..5cb9ef1 100644 (file)
 #if ENABLE(WEB_REPLAY)
 
 #include "AllReplayInputs.h"
+#include "PlatformKeyboardEvent.h"
 #include "PlatformMouseEvent.h"
 #include "ReplayInputTypes.h"
 #include "SecurityOrigin.h"
 #include "URL.h"
+#include <wtf/text/Base64.h>
 
+using WebCore::KeypressCommand;
 using WebCore::IntPoint;
 using WebCore::MouseButton;
 using WebCore::PlatformEvent;
+using WebCore::PlatformKeyboardEvent;
 using WebCore::PlatformMouseEvent;
 using WebCore::SecurityOrigin;
 using WebCore::URL;
@@ -58,13 +62,22 @@ WEB_REPLAY_INPUT_NAMES_FOR_EACH(IMPORT_FROM_WEBCORE_NAMESPACE)
     if (!_encodedValue.get<_type>(ASCIILiteral(#_key), _key)) \
         return false
 
+#define ENCODE_OPTIONAL_SCALAR_TYPE_WITH_KEY(_encodedValue, _type, _key, _value, condition) \
+    if (condition) \
+        _encodedValue.put<_type>(ASCIILiteral(#_key), _value)
+
+#define DECODE_OPTIONAL_SCALAR_TYPE_WITH_KEY(_encodedValue, _type, _key) \
+    _type _key; \
+    bool _key ## WasDecoded = _encodedValue.get<_type>(ASCIILiteral(#_key), _key)
+
 namespace JSC {
 
 EncodedValue EncodingTraits<NondeterministicInputBase>::encodeValue(const NondeterministicInputBase& input)
 {
     EncodedValue encodedValue = EncodedValue::createObject();
     const AtomicString& type = input.type();
-    encodedValue.put<String>(ASCIILiteral("type"), type.string());
+
+    ENCODE_SCALAR_TYPE_WITH_KEY(encodedValue, String, type, type.string());
 
 #define ENCODE_IF_TYPE_TAG_MATCHES(name) \
     if (type == inputTypes().name) { \
@@ -88,9 +101,7 @@ EncodedValue EncodingTraits<NondeterministicInputBase>::encodeValue(const Nondet
 
 bool EncodingTraits<NondeterministicInputBase>::decodeValue(EncodedValue& encodedValue, std::unique_ptr<NondeterministicInputBase>& input)
 {
-    String type;
-    if (!encodedValue.get<String>(ASCIILiteral("type"), type))
-        return false;
+    DECODE_SCALAR_TYPE_WITH_KEY(encodedValue, String, type);
 
 #define DECODE_IF_TYPE_TAG_MATCHES(name) \
     if (type == inputTypes().name) { \
@@ -118,6 +129,88 @@ bool EncodingTraits<NondeterministicInputBase>::decodeValue(EncodedValue& encode
     return false;
 }
 
+#if USE(APPKIT)
+EncodedValue EncodingTraits<KeypressCommand>::encodeValue(const KeypressCommand& command)
+{
+    EncodedValue encodedValue = EncodedValue::createObject();
+
+    ENCODE_SCALAR_TYPE_WITH_KEY(encodedValue, String, commandName, command.commandName);
+    ENCODE_OPTIONAL_SCALAR_TYPE_WITH_KEY(encodedValue, String, text, command.text, !command.text.isEmpty());
+
+    return encodedValue;
+}
+
+bool EncodingTraits<KeypressCommand>::decodeValue(EncodedValue& encodedValue, KeypressCommand& decodedValue)
+{
+    DECODE_SCALAR_TYPE_WITH_KEY(encodedValue, String, commandName);
+    DECODE_OPTIONAL_SCALAR_TYPE_WITH_KEY(encodedValue, String, text);
+
+    decodedValue = textWasDecoded ? KeypressCommand(commandName, text) : KeypressCommand(commandName);
+    return true;
+}
+
+class PlatformKeyboardEventAppKit : public WebCore::PlatformKeyboardEvent {
+public:
+    PlatformKeyboardEventAppKit(const PlatformKeyboardEvent& event, bool handledByInputMethod, Vector<KeypressCommand>& commands)
+        : PlatformKeyboardEvent(event)
+    {
+        m_handledByInputMethod = handledByInputMethod;
+        m_commands = commands;
+    }
+};
+#endif // USE(APPKIT)
+
+EncodedValue EncodingTraits<PlatformKeyboardEvent>::encodeValue(const PlatformKeyboardEvent& input)
+{
+    EncodedValue encodedValue = EncodedValue::createObject();
+
+    ENCODE_SCALAR_TYPE_WITH_KEY(encodedValue, double, timestamp, input.timestamp());
+    ENCODE_SCALAR_TYPE_WITH_KEY(encodedValue, PlatformEvent::Type, type, input.type());
+    ENCODE_SCALAR_TYPE_WITH_KEY(encodedValue, PlatformEvent::Modifiers, modifiers, static_cast<PlatformEvent::Modifiers>(input.modifiers()));
+    ENCODE_SCALAR_TYPE_WITH_KEY(encodedValue, String, text, input.text());
+    ENCODE_SCALAR_TYPE_WITH_KEY(encodedValue, String, unmodifiedText, input.unmodifiedText());
+    ENCODE_SCALAR_TYPE_WITH_KEY(encodedValue, String, keyIdentifier, input.keyIdentifier());
+    ENCODE_SCALAR_TYPE_WITH_KEY(encodedValue, int, windowsVirtualKeyCode, input.windowsVirtualKeyCode());
+    ENCODE_SCALAR_TYPE_WITH_KEY(encodedValue, int, nativeVirtualKeyCode, input.nativeVirtualKeyCode());
+    ENCODE_SCALAR_TYPE_WITH_KEY(encodedValue, int, macCharCode, input.macCharCode());
+    ENCODE_SCALAR_TYPE_WITH_KEY(encodedValue, bool, autoRepeat, input.isAutoRepeat());
+    ENCODE_SCALAR_TYPE_WITH_KEY(encodedValue, bool, keypad, input.isKeypad());
+    ENCODE_SCALAR_TYPE_WITH_KEY(encodedValue, bool, systemKey, input.isSystemKey());
+#if USE(APPKIT)
+    ENCODE_SCALAR_TYPE_WITH_KEY(encodedValue, bool, handledByInputMethod, input.handledByInputMethod());
+    ENCODE_SCALAR_TYPE_WITH_KEY(encodedValue, Vector<KeypressCommand>, commands, input.commands());
+#endif
+    return encodedValue;
+}
+
+bool EncodingTraits<PlatformKeyboardEvent>::decodeValue(EncodedValue& encodedValue, std::unique_ptr<PlatformKeyboardEvent>& input)
+{
+    DECODE_SCALAR_TYPE_WITH_KEY(encodedValue, double, timestamp);
+    DECODE_SCALAR_TYPE_WITH_KEY(encodedValue, PlatformEvent::Type, type);
+    DECODE_SCALAR_TYPE_WITH_KEY(encodedValue, PlatformEvent::Modifiers, modifiers);
+    DECODE_SCALAR_TYPE_WITH_KEY(encodedValue, String, text);
+    DECODE_SCALAR_TYPE_WITH_KEY(encodedValue, String, unmodifiedText);
+    DECODE_SCALAR_TYPE_WITH_KEY(encodedValue, String, keyIdentifier);
+    DECODE_SCALAR_TYPE_WITH_KEY(encodedValue, int, windowsVirtualKeyCode);
+    DECODE_SCALAR_TYPE_WITH_KEY(encodedValue, int, nativeVirtualKeyCode);
+    DECODE_SCALAR_TYPE_WITH_KEY(encodedValue, int, macCharCode);
+    DECODE_SCALAR_TYPE_WITH_KEY(encodedValue, bool, autoRepeat);
+    DECODE_SCALAR_TYPE_WITH_KEY(encodedValue, bool, keypad);
+    DECODE_SCALAR_TYPE_WITH_KEY(encodedValue, bool, systemKey);
+#if USE(APPKIT)
+    DECODE_SCALAR_TYPE_WITH_KEY(encodedValue, bool, handledByInputMethod);
+    DECODE_SCALAR_TYPE_WITH_KEY(encodedValue, Vector<KeypressCommand>, commands);
+#endif
+
+    PlatformKeyboardEvent platformEvent = PlatformKeyboardEvent(type, text, unmodifiedText, keyIdentifier, windowsVirtualKeyCode, nativeVirtualKeyCode, macCharCode, autoRepeat, keypad, systemKey, modifiers, timestamp);
+#if USE(APPKIT)
+    input = std::make_unique<PlatformKeyboardEventAppKit>(platformEvent, handledByInputMethod, commands);
+#else
+    input = std::make_unique<PlatformKeyboardEvent>(platformEvent);
+#endif
+    return true;
+}
+
 EncodedValue EncodingTraits<PlatformMouseEvent>::encodeValue(const PlatformMouseEvent& input)
 {
     EncodedValue encodedValue = EncodedValue::createObject();
index 839c45c..7afa232 100644 (file)
 
 #include <replay/EncodedValue.h>
 #include <replay/NondeterministicInput.h>
+#include <wtf/Vector.h>
 
 namespace WebCore {
 class Document;
 class Frame;
 class Page;
+class PlatformKeyboardEvent;
 class PlatformMouseEvent;
 class SecurityOrigin;
 class URL;
+
+#if USE(APPKIT)
+struct KeypressCommand;
+#endif
 } // namespace WebCore
 
 // Template specializations must be defined in the same namespace as the template declaration.
 namespace JSC {
 
+#if USE(APPKIT)
+template<> struct EncodingTraits<WebCore::KeypressCommand> {
+    typedef WebCore::KeypressCommand DecodedType;
+
+    static EncodedValue encodeValue(const WebCore::KeypressCommand& value);
+    static bool decodeValue(EncodedValue&, WebCore::KeypressCommand& value);
+};
+#endif // USE(APPKIT)
+
 template<> struct EncodingTraits<NondeterministicInputBase> {
     typedef NondeterministicInputBase DecodedType;
 
@@ -52,6 +67,13 @@ template<> struct EncodingTraits<NondeterministicInputBase> {
     static bool decodeValue(EncodedValue&, std::unique_ptr<NondeterministicInputBase>& value);
 };
 
+template<> struct EncodingTraits<WebCore::PlatformKeyboardEvent> {
+    typedef WebCore::PlatformKeyboardEvent DecodedType;
+
+    static EncodedValue encodeValue(const WebCore::PlatformKeyboardEvent& value);
+    static bool decodeValue(EncodedValue&, std::unique_ptr<WebCore::PlatformKeyboardEvent>& value);
+};
+
 template<> struct EncodingTraits<WebCore::PlatformMouseEvent> {
     typedef WebCore::PlatformMouseEvent DecodedType;
 
index 30a0e6c..383c08c 100644 (file)
@@ -55,6 +55,9 @@ namespace WebCore {
 
 UserInputBridge::UserInputBridge(Page& page)
     : m_page(page)
+#if ENABLE(WEB_REPLAY)
+    , m_state(UserInputBridge::State::Open)
+#endif
 {
 }
 
@@ -136,8 +139,19 @@ bool UserInputBridge::handleMouseMoveOnScrollbarEvent(const PlatformMouseEvent&
     return m_page.mainFrame().eventHandler().passMouseMovedEventToScrollbars(mouseEvent);
 }
 
-bool UserInputBridge::handleKeyEvent(const PlatformKeyboardEvent& keyEvent, InputSource)
+bool UserInputBridge::handleKeyEvent(const PlatformKeyboardEvent& keyEvent, InputSource inputSource)
 {
+#if ENABLE(WEB_REPLAY)
+    EARLY_RETURN_IF_SHOULD_IGNORE_INPUT;
+
+    if (activeCursor().isCapturing()) {
+        std::unique_ptr<PlatformKeyboardEvent> ownedEvent = std::make_unique<PlatformKeyboardEvent>(keyEvent);
+        activeCursor().appendInput<HandleKeyPress>(std::move(ownedEvent));
+    }
+#else
+    UNUSED_PARAM(inputSource);
+#endif
+
     return m_page.focusController().focusedOrMainFrame().eventHandler().keyEvent(keyEvent);
 }
 
index 6c32283..e24fd9b 100644 (file)
                 "header": "page/Page.h"
             },
             {
+                "name": "PlatformKeyboardEvent", "mode": "OWNED",
+                "header": "platform/PlatformKeyboardEvent.h"
+            },
+            {
                 "name": "PlatformMouseEvent", "mode": "OWNED",
                 "header": "platform/PlatformMouseEvent.h"
             },
             ]
         },
         {
+            "name": "HandleKeyPress",
+            "description": "The embedder signalled a key press event.",
+            "queue": "EVENT_LOOP",
+            "members": [
+                { "name": "platformEvent", "type": "PlatformKeyboardEvent" }
+            ]
+        },
+        {
             "name": "InitialNavigation",
             "description": "Initiate the initial main frame navigation.",
             "queue": "EVENT_LOOP",