Extend event stream to include interpolated events and add a force press test that...
authorcommit-queue@webkit.org <commit-queue@webkit.org@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Tue, 11 Oct 2016 18:38:49 +0000 (18:38 +0000)
committercommit-queue@webkit.org <commit-queue@webkit.org@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Tue, 11 Oct 2016 18:38:49 +0000 (18:38 +0000)
https://bugs.webkit.org/show_bug.cgi?id=163161

Patch by Megan Gardner <megan_gardner@apple.com> on 2016-10-11
Reviewed by Simon Fraser.

Added functionality to the event stream to allow for interpolated events.
Can now do long press, as well as a better way to do drag and other time-based
events that require a large stream of descrete HID events.
Added a basic force touch test to demostrate this interpolation.
Also updated the script to allow for iPhone 7 specific tests, as force touch
needs to be on a device that had force touch.

* Scripts/webkitpy/port/ios.py:
(IOSSimulatorPort):
* TestRunnerShared/UIScriptContext/Bindings/UIScriptController.idl:
* WebKitTestRunner/ios/HIDEventGenerator.mm:
(linearInterpolation):
(simpleCurveInterpolation):
(calculateNextCurveLocation):
(phaseFromString):
(interpolationFromString):
(-[HIDEventGenerator eventMaskFromEventInfo:]):
(-[HIDEventGenerator _createIOHIDEventWithInfo:]):
(-[HIDEventGenerator moveToPoints:touchCount:duration:]):
(-[HIDEventGenerator interpolatedEvents:]):
(-[HIDEventGenerator processEventsArray:withStartTime:]):
(-[HIDEventGenerator eventDispatchThreadEntry:]):
(simpleDragCurve): Deleted.
(calculateNextLocation): Deleted.

git-svn-id: https://svn.webkit.org/repository/webkit/trunk@207153 268f45cc-cd09-0410-ab3c-d52691b4dbfc

LayoutTests/fast/events/touch/ios/iphone7/force-press-event-expected.txt [new file with mode: 0644]
LayoutTests/fast/events/touch/ios/iphone7/force-press-event.html [new file with mode: 0644]
Tools/ChangeLog
Tools/Scripts/webkitpy/port/ios.py
Tools/TestRunnerShared/UIScriptContext/Bindings/UIScriptController.idl
Tools/WebKitTestRunner/ios/HIDEventGenerator.mm

diff --git a/LayoutTests/fast/events/touch/ios/iphone7/force-press-event-expected.txt b/LayoutTests/fast/events/touch/ios/iphone7/force-press-event-expected.txt
new file mode 100644 (file)
index 0000000..a73de45
--- /dev/null
@@ -0,0 +1 @@
+PASS: Generated increasing force
diff --git a/LayoutTests/fast/events/touch/ios/iphone7/force-press-event.html b/LayoutTests/fast/events/touch/ios/iphone7/force-press-event.html
new file mode 100644 (file)
index 0000000..670a25f
--- /dev/null
@@ -0,0 +1,104 @@
+<!DOCTYPE html> <!-- webkit-test-runner [ useFlexibleViewport=true ] -->
+
+<html>
+<head>
+    <script>
+        if (window.testRunner) {
+            testRunner.dumpAsText();
+            testRunner.waitUntilDone();
+        }
+
+        function getUIScript()
+        {
+            return `
+            (function() {
+                var eventStream = {
+                    events : [
+                        {
+                            interpolate : "linear",
+                            timestep: 0.025,
+                            startEvent : {
+                                inputType : "hand",
+                                timeOffset : 0,
+                                touches : [
+                                    {
+                                        inputType : "finger",
+                                        phase : "began",
+                                        id : 1,
+                                        x : 20,
+                                        y : 40,
+                                        pressure : 0
+                                    }
+                                ]
+                            },
+                            endEvent : {
+                                inputType : "hand",
+                                timeOffset : 3.0,
+                                touches : [
+                                    {
+                                        inputType : "finger",
+                                        phase : "stationary",
+                                        id : 1,
+                                        x : 20,
+                                        y : 40,
+                                        pressure : 500
+                                    }
+                                ]
+                            }
+                        }
+                    ]
+                };
+
+                uiController.sendEventStream(JSON.stringify(eventStream), function() {
+                    uiController.uiScriptComplete();
+                });
+            })();`
+        }
+
+        function runTest()
+        {
+            if (!testRunner.runUIScript)
+                return;
+
+            var output = '';
+            var target = document.getElementById('target');
+            var forces = [];
+            
+            target.addEventListener('touchforcechange', function(event) {
+                forces.push(event.touches[0].force);
+            });
+            
+            target.addEventListener('touchstart', function(event) {
+                event.preventDefault();
+            });
+            
+            if (testRunner.runUIScript) {
+                testRunner.runUIScript(getUIScript(), function(result) {
+                    var middleIndex = Math.floor(forces.length/2);
+                
+                    if ((forces[0] < forces[middleIndex]) && (forces[middleIndex] < forces[forces.length - 1]))
+                        output += 'PASS: Generated increasing force';
+                    
+                    document.getElementById('target').innerHTML = output;
+                    testRunner.notifyDone();
+               });
+            }
+        }
+
+        window.addEventListener('load', runTest, false);
+    </script>
+    <style>
+        #target {
+            height: 100px;
+            width: 200px;
+            background-color: silver;
+        }
+    </style>
+    <meta name="viewport" content="initial-scale=1">
+</head>
+<body>
+<div id="target">
+    This test requires UIScriptController to run.
+</div>
+</body>
+</html>
index ba312b4..dfe6d48 100644 (file)
@@ -1,3 +1,35 @@
+2016-10-11  Megan Gardner  <megan_gardner@apple.com>
+
+        Extend event stream to include interpolated events and add a force press test that uses that interpolation
+        https://bugs.webkit.org/show_bug.cgi?id=163161
+
+        Reviewed by Simon Fraser.
+
+        Added functionality to the event stream to allow for interpolated events.
+        Can now do long press, as well as a better way to do drag and other time-based
+        events that require a large stream of descrete HID events.
+        Added a basic force touch test to demostrate this interpolation.
+        Also updated the script to allow for iPhone 7 specific tests, as force touch
+        needs to be on a device that had force touch.
+
+        * Scripts/webkitpy/port/ios.py:
+        (IOSSimulatorPort):
+        * TestRunnerShared/UIScriptContext/Bindings/UIScriptController.idl:
+        * WebKitTestRunner/ios/HIDEventGenerator.mm:
+        (linearInterpolation):
+        (simpleCurveInterpolation):
+        (calculateNextCurveLocation):
+        (phaseFromString):
+        (interpolationFromString):
+        (-[HIDEventGenerator eventMaskFromEventInfo:]):
+        (-[HIDEventGenerator _createIOHIDEventWithInfo:]):
+        (-[HIDEventGenerator moveToPoints:touchCount:duration:]):
+        (-[HIDEventGenerator interpolatedEvents:]):
+        (-[HIDEventGenerator processEventsArray:withStartTime:]):
+        (-[HIDEventGenerator eventDispatchThreadEntry:]):
+        (simpleDragCurve): Deleted.
+        (calculateNextLocation): Deleted.
+
 2016-10-11  Alex Christensen  <achristensen@webkit.org>
 
         URLParser should percent-encode non-ASCII and non-printable characters in fragment
index fc556e7..b79bb70 100644 (file)
@@ -74,7 +74,7 @@ class IOSSimulatorPort(DarwinPort):
     DEFAULT_ARCHITECTURE = 'x86_64'
 
     DEFAULT_DEVICE_CLASS = 'iphone'
-    CUSTOM_DEVICE_CLASSES = ['ipad']
+    CUSTOM_DEVICE_CLASSES = ['ipad', 'iphone7']
     SDK = 'iphonesimulator'
 
     SIMULATOR_BUNDLE_ID = 'com.apple.iphonesimulator'
@@ -86,6 +86,7 @@ class IOSSimulatorPort(DarwinPort):
     DEVICE_CLASS_MAP = {
         'x86_64': {
             'iphone': 'iPhone 5s',
+            'iphone7': 'iPhone 7',
             'ipad': 'iPad Air'
         },
         'x86': {
index 5e6c26e..891293a 100644 (file)
@@ -67,6 +67,38 @@ interface UIScriptController {
     //              ]
     //          },
     //          {
+    //              "interpolate" : "linear",
+    //              "timestep" : 0.025,
+    //              "startEvent" : {
+    //                  "inputType" : "hand",
+    //                  "timeOffset" : 0.025,
+    //                  "touches" : [
+    //                      {
+    //                          "inputType" : "finger",
+    //                          "phase" : "began",
+    //                          "id" : 1,
+    //                          "x" : 100,
+    //                          "y" : 120,
+    //                          "pressure" : 0
+    //                      }
+    //                  ]
+    //              },
+    //              "endEvent" : {
+    //                  "inputType" : "hand",
+    //                  "timeOffset" : 3.0,
+    //                  "touches" : [
+    //                      {
+    //                          "inputType" : "finger",
+    //                          "phase" : "stationary",
+    //                          "id" : 1,
+    //                          "x" : 20,
+    //                          "y" : 40,
+    //                          "pressure" : 500
+    //                      }
+    //                  ]
+    //              }
+    //          },
+    //          {
     //              "inputType" : "hand",
     //              "timeOffset" : 0.002, // seconds relative to the first event
     //              "touches" : [
index c277fb2..30de4ed 100644 (file)
@@ -42,6 +42,10 @@ NSString* const HIDEventInputType = @"inputType";
 NSString* const HIDEventTimeOffsetKey = @"timeOffset";
 NSString* const HIDEventTouchesKey = @"touches";
 NSString* const HIDEventPhaseKey = @"phase";
+NSString* const HIDEventInterpolateKey = @"interpolate";
+NSString* const HIDEventTimestepKey = @"timestep";
+NSString* const HIDEventStartEventKey = @"startEvent";
+NSString* const HIDEventEndEventKey = @"endEvent";
 NSString* const HIDEventTouchIDKey = @"id";
 NSString* const HIDEventPressureKey = @"pressure";
 NSString* const HIDEventXKey = @"x";
@@ -54,7 +58,11 @@ NSString* const HIDEventInputTypeHand = @"hand";
 NSString* const HIDEventInputTypeFinger = @"finger";
 NSString* const HIDEventInputTypeStylus = @"stylus";
 
+NSString* const HIDEventInterpolationTypeLinear = @"linear";
+NSString* const HIDEventInterpolationTypeSimpleCurve = @"simpleCurve";
+
 NSString* const HIDEventPhaseBegan = @"began";
+NSString* const HIDEventPhaseStationary = @"stationary";
 NSString* const HIDEventPhaseMoved = @"moved";
 NSString* const HIDEventPhaseEnded = @"ended";
 NSString* const HIDEventPhaseCanceled = @"canceled";
@@ -71,6 +79,11 @@ static const long nanosecondsPerSecond = 1e9;
 static int fingerIdentifiers[maxTouchCount] = { 2, 3, 4, 5, 1 };
 
 typedef enum {
+    InterpolationTypeLinear,
+    InterpolationTypeSimpleCurve,
+} InterpolationType;
+
+typedef enum {
     HandEventNull,
     HandEventTouched,
     HandEventMoved,
@@ -98,16 +111,28 @@ static CFTimeInterval secondsSinceAbsoluteTime(CFAbsoluteTime startTime)
     return (CFAbsoluteTimeGetCurrent() - startTime);
 }
 
-static double simpleDragCurve(double a, double b, double t)
+static double linearInterpolation(double a, double b, double t)
+{
+    return (a + (b - a) * t );
+}
+
+static double simpleCurveInterpolation(double a, double b, double t)
 {
     return (a + (b - a) * sin(sin(t * M_PI / 2) * t * M_PI / 2));
 }
 
-static CGPoint calculateNextLocation(CGPoint a, CGPoint b, CFTimeInterval t)
+
+static CGPoint calculateNextCurveLocation(CGPoint a, CGPoint b, CFTimeInterval t)
 {
-    return CGPointMake(simpleDragCurve(a.x, b.x, t), simpleDragCurve(a.y, b.y, t));
+    return CGPointMake(simpleCurveInterpolation(a.x, b.x, t), simpleCurveInterpolation(a.y, b.y, t));
 }
 
+typedef double(*pressureInterpolationFunction)(double, double, CFTimeInterval);
+static pressureInterpolationFunction interpolations[] = {
+    linearInterpolation,
+    simpleCurveInterpolation,
+};
+
 static void delayBetweenMove(int eventIndex, double elapsed)
 {
     // Delay next event until expected elapsed time.
@@ -187,6 +212,9 @@ static UITouchPhase phaseFromString(NSString *string)
 {
     if ([string isEqualToString:HIDEventPhaseBegan])
         return UITouchPhaseBegan;
+    
+    if ([string isEqualToString:HIDEventPhaseStationary])
+        return UITouchPhaseStationary;
 
     if ([string isEqualToString:HIDEventPhaseMoved])
         return UITouchPhaseMoved;
@@ -200,20 +228,35 @@ static UITouchPhase phaseFromString(NSString *string)
     return UITouchPhaseStationary;
 }
 
+static InterpolationType interpolationFromString(NSString *string)
+{
+    if ([string isEqualToString:HIDEventInterpolationTypeLinear])
+        return InterpolationTypeLinear;
+    
+    if ([string isEqualToString:HIDEventInterpolationTypeSimpleCurve])
+        return InterpolationTypeSimpleCurve;
+    
+    return InterpolationTypeLinear;
+}
+
 - (IOHIDDigitizerEventMask)eventMaskFromEventInfo:(NSDictionary *)info
 {
+    IOHIDDigitizerEventMask eventMask = 0;
     NSArray *childEvents = info[HIDEventTouchesKey];
     for (NSDictionary *touchInfo in childEvents) {
         UITouchPhase phase = phaseFromString(touchInfo[HIDEventPhaseKey]);
         // If there are any new or ended events, mask includes touch.
         if (phase == UITouchPhaseBegan || phase == UITouchPhaseEnded || phase == UITouchPhaseCancelled)
-            return kIOHIDDigitizerEventTouch;
+            eventMask |= kIOHIDDigitizerEventTouch;
+        // If there are any pressure readings, set mask must include attribute
+        if ([touchInfo[HIDEventPressureKey] floatValue])
+            eventMask |= kIOHIDDigitizerEventAttribute;
     }
     
-    return 0;
+    return eventMask;
 }
 
-// Returns 1 for all events where the fingers are on the glass (everything but enced and canceled).
+// Returns 1 for all events where the fingers are on the glass (everything but ended and canceled).
 - (CFIndex)touchFromEventInfo:(NSDictionary *)info
 {
     NSArray *childEvents = info[HIDEventTouchesKey];
@@ -260,7 +303,7 @@ static UITouchPhase phaseFromString(NSString *string)
         IOHIDDigitizerEventMask childEventMask = 0;
 
         UITouchPhase phase = phaseFromString(touchInfo[HIDEventPhaseKey]);
-        if (phase != UITouchPhaseCancelled && phase != UITouchPhaseBegan && phase != UITouchPhaseEnded)
+        if (phase != UITouchPhaseCancelled && phase != UITouchPhaseBegan && phase != UITouchPhaseEnded && phase != UITouchPhaseStationary)
             childEventMask |= kIOHIDDigitizerEventPosition;
 
         if (phase == UITouchPhaseBegan || phase == UITouchPhaseEnded || phase == UITouchPhaseCancelled)
@@ -268,6 +311,9 @@ static UITouchPhase phaseFromString(NSString *string)
 
         if (phase == UITouchPhaseCancelled)
             childEventMask |= kIOHIDDigitizerEventCancel;
+        
+        if ([touchInfo[HIDEventPressureKey] floatValue])
+            childEventMask |= kIOHIDDigitizerEventAttribute;
 
         IOHIDEventRef subEvent = IOHIDEventCreateDigitizerFingerEvent(kCFAllocatorDefault, machTime,
             [touchInfo[HIDEventTouchIDKey] intValue],               // index
@@ -551,7 +597,7 @@ static UITouchPhase phaseFromString(NSString *string)
             if (!eventIndex)
                 startLocations[i] = _activePoints[i].point;
 
-            nextLocations[i] = calculateNextLocation(startLocations[i], newLocations[i], interval);
+            nextLocations[i] = calculateNextCurveLocation(startLocations[i], newLocations[i], interval);
         }
         [self _updateTouchPoints:nextLocations count:touchCount];
 
@@ -920,32 +966,109 @@ static inline uint32_t hidUsageCodeForCharacter(NSString *key)
     [self _sendHIDEvent:eventRef.get()];
 }
 
+- (NSArray *)interpolatedEvents:(NSDictionary *)interpolationsDictionary
+{
+    NSDictionary *startEvent = interpolationsDictionary[HIDEventStartEventKey];
+    NSDictionary *endEvent = interpolationsDictionary[HIDEventEndEventKey];
+    NSTimeInterval timeStep = [interpolationsDictionary[HIDEventTimestepKey] doubleValue];
+    InterpolationType interpolationType = interpolationFromString(interpolationsDictionary[HIDEventInterpolateKey]);
+    
+    NSMutableArray *interpolatedEvents = [NSMutableArray arrayWithObject:startEvent];
+    
+    NSTimeInterval startTime = [startEvent[HIDEventTimeOffsetKey] doubleValue];
+    NSTimeInterval endTime = [endEvent[HIDEventTimeOffsetKey] doubleValue];
+    NSTimeInterval time = startTime + timeStep;
+    
+    NSArray *startTouches = startEvent[HIDEventTouchesKey];
+    NSArray *endTouches = endEvent[HIDEventTouchesKey];
+    
+    while (time < endTime) {
+        NSMutableDictionary *newEvent = [endEvent mutableCopy];
+        double timeRatio = (time - startTime) / (endTime - startTime);
+        newEvent[HIDEventTimeOffsetKey] = [NSNumber numberWithDouble:(time)];
+        
+        NSEnumerator *startEnumerator = [startTouches objectEnumerator];
+        NSDictionary *startTouch;
+        NSMutableArray *newTouches = [NSMutableArray arrayWithCapacity:[endTouches count]];
+        while (startTouch = [startEnumerator nextObject])  {
+            NSEnumerator *endEnumerator = [endTouches objectEnumerator];
+            NSDictionary *endTouch = [endEnumerator nextObject];
+            NSInteger startTouchID = [startTouch[HIDEventTouchIDKey] integerValue];
+            
+            while (endTouch && ([endTouch[HIDEventTouchIDKey] integerValue] != startTouchID))
+                endTouch = [endEnumerator nextObject];
+            
+            if (endTouch) {
+                NSMutableDictionary *newTouch = [endTouch mutableCopy];
+                
+                if (newTouch[HIDEventXKey] != startTouch[HIDEventXKey])
+                    newTouch[HIDEventXKey] = @(interpolations[interpolationType]([startTouch[HIDEventXKey] doubleValue], [endTouch[HIDEventXKey] doubleValue], timeRatio));
+                
+                if (newTouch[HIDEventYKey] != startTouch[HIDEventYKey])
+                    newTouch[HIDEventYKey] = @(interpolations[interpolationType]([startTouch[HIDEventYKey] doubleValue], [endTouch[HIDEventYKey] doubleValue], timeRatio));
+                
+                if (newTouch[HIDEventPressureKey] != startTouch[HIDEventPressureKey])
+                    newTouch[HIDEventPressureKey] = @(interpolations[interpolationType]([startTouch[HIDEventPressureKey] doubleValue], [endTouch[HIDEventPressureKey] doubleValue], timeRatio));
+                
+                [newTouches addObject:newTouch];
+                [newTouch release];
+            } else
+                NSLog(@"Missing End Touch with ID: %ld", (long)startTouchID);
+        }
+        
+        newEvent[HIDEventTouchesKey] = newTouches;
+        
+        [interpolatedEvents addObject:newEvent];
+        [newEvent release];
+        time += timeStep;
+    }
+
+    return interpolatedEvents;
+}
+
+- (NSArray *)expandEvents:(NSArray *)events withStartTime:(CFAbsoluteTime)startTime
+{
+    NSMutableArray *expandedEvents = [NSMutableArray array];
+    for (NSDictionary *event in events) {
+        NSString *interpolate = event[HIDEventInterpolateKey];
+        // we have key events that we need to generate
+        if (interpolate) {
+            NSArray *newEvents = [self interpolatedEvents:event];
+            [expandedEvents addObjectsFromArray:[self expandEvents:newEvents withStartTime:startTime]];
+        } else
+            [expandedEvents addObject:event];
+    }
+    return expandedEvents;
+}
+
 - (void)eventDispatchThreadEntry:(NSDictionary *)threadData
 {
     NSDictionary *eventStream = threadData[@"eventInfo"];
     void (^completionBlock)() = threadData[@"completionBlock"];
-
+    
     NSArray *events = eventStream[TopLevelEventInfoKey];
     if (!events.count) {
         NSLog(@"No events found in event stream");
         return;
     }
-
+    
     CFAbsoluteTime startTime = CFAbsoluteTimeGetCurrent();
     
-    for (NSDictionary *eventInfo in events) {
+    NSArray *expandedEvents = [self expandEvents:events withStartTime:startTime];
+    
+    for (NSDictionary *eventInfo in expandedEvents) {
         NSTimeInterval eventRelativeTime = [eventInfo[HIDEventTimeOffsetKey] doubleValue];
         CFAbsoluteTime targetTime = startTime + eventRelativeTime;
         
         CFTimeInterval waitTime = targetTime - CFAbsoluteTimeGetCurrent();
         if (waitTime > 0)
             [NSThread sleepForTimeInterval:waitTime];
-
+        
         dispatch_async(dispatch_get_main_queue(), ^ {
             [self dispatchEventWithInfo:eventInfo];
         });
     }
-
+    
     dispatch_async(dispatch_get_main_queue(), ^ {
         [self _sendMarkerHIDEventWithCompletionBlock:completionBlock];
     });