[MediaStream][Mac] Mark captured video frames as ready for display immediately
authoreric.carlson@apple.com <eric.carlson@apple.com@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Wed, 9 Nov 2016 18:27:37 +0000 (18:27 +0000)
committereric.carlson@apple.com <eric.carlson@apple.com@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Wed, 9 Nov 2016 18:27:37 +0000 (18:27 +0000)
https://bugs.webkit.org/show_bug.cgi?id=164482
<rdar://problem/29139073>

Reviewed by Jer Noble.

* platform/cf/CoreMediaSoftLink.cpp: Add new constant.
* platform/cf/CoreMediaSoftLink.h:

* platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.h:
* platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.mm:
(WebCore::MediaPlayerPrivateMediaStreamAVFObjC::requestNotificationWhenReadyForMediaData):
  New, ask register for a callback when the sample buffer display layer is ready
  for more media data.
(WebCore::MediaPlayerPrivateMediaStreamAVFObjC::enqueueVideoSampleBuffer): Don't change
  the sample timestamps, assume the caller has configured the sample correctly.
(WebCore::MediaPlayerPrivateMediaStreamAVFObjC::prepareVideoSampleBufferFromTrack): Don't
  drop frames when the display layer isn't ready.
(WebCore::MediaPlayerPrivateMediaStreamAVFObjC::destroyLayer): Call stopRequestingMediaData.
(WebCore::MediaPlayerPrivateMediaStreamAVFObjC::sampleBufferUpdated):
(WebCore::MediaPlayerPrivateMediaStreamAVFObjC::enqueueVideoSampleBufferFromTrack): Deleted.

* platform/mediastream/mac/AVVideoCaptureSource.mm:
(WebCore::AVVideoCaptureSource::setupCaptureSession): Tell the video output to always discard
  late video frames, we don't need them.
(WebCore::AVVideoCaptureSource::processNewFrame): Add a kCMSampleAttachmentKey_DisplayImmediately
  attachment.

git-svn-id: https://svn.webkit.org/repository/webkit/trunk@208444 268f45cc-cd09-0410-ab3c-d52691b4dbfc

Source/WebCore/ChangeLog
Source/WebCore/platform/cf/CoreMediaSoftLink.cpp
Source/WebCore/platform/cf/CoreMediaSoftLink.h
Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.h
Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.mm
Source/WebCore/platform/mediastream/mac/AVVideoCaptureSource.mm

index 192a028..dde2155 100644 (file)
@@ -1,3 +1,33 @@
+2016-11-09  Eric Carlson  <eric.carlson@apple.com>
+
+        [MediaStream][Mac] Mark captured video frames as ready for display immediately
+        https://bugs.webkit.org/show_bug.cgi?id=164482
+        <rdar://problem/29139073>
+
+        Reviewed by Jer Noble.
+
+        * platform/cf/CoreMediaSoftLink.cpp: Add new constant.
+        * platform/cf/CoreMediaSoftLink.h:
+
+        * platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.h:
+        * platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.mm:
+        (WebCore::MediaPlayerPrivateMediaStreamAVFObjC::requestNotificationWhenReadyForMediaData):
+          New, ask register for a callback when the sample buffer display layer is ready
+          for more media data.
+        (WebCore::MediaPlayerPrivateMediaStreamAVFObjC::enqueueVideoSampleBuffer): Don't change
+          the sample timestamps, assume the caller has configured the sample correctly.
+        (WebCore::MediaPlayerPrivateMediaStreamAVFObjC::prepareVideoSampleBufferFromTrack): Don't
+          drop frames when the display layer isn't ready.
+        (WebCore::MediaPlayerPrivateMediaStreamAVFObjC::destroyLayer): Call stopRequestingMediaData.
+        (WebCore::MediaPlayerPrivateMediaStreamAVFObjC::sampleBufferUpdated):
+        (WebCore::MediaPlayerPrivateMediaStreamAVFObjC::enqueueVideoSampleBufferFromTrack): Deleted.
+
+        * platform/mediastream/mac/AVVideoCaptureSource.mm:
+        (WebCore::AVVideoCaptureSource::setupCaptureSession): Tell the video output to always discard
+          late video frames, we don't need them.
+        (WebCore::AVVideoCaptureSource::processNewFrame): Add a kCMSampleAttachmentKey_DisplayImmediately
+          attachment.
+
 2016-11-09  Joanmarie Diggs  <jdiggs@igalia.com>
 
         AX: [ATK] Wrong selected element at a given index in a list box (redux)
index e122f01..5be0cdc 100644 (file)
@@ -104,6 +104,8 @@ SOFT_LINK_CONSTANT_FOR_SOURCE(WebCore, CoreMedia, kCMSampleBufferAttachmentKey_D
 SOFT_LINK_CONSTANT_FOR_SOURCE(WebCore, CoreMedia, kCMSampleBufferAttachmentKey_DrainAfterDecoding, CFStringRef)
 SOFT_LINK_CONSTANT_FOR_SOURCE(WebCore, CoreMedia, kCMSampleBufferAttachmentKey_EmptyMedia, CFStringRef)
 SOFT_LINK_CONSTANT_FOR_SOURCE(WebCore, CoreMedia, kCMSampleBufferAttachmentKey_ResetDecoderBeforeDecoding, CFStringRef)
+SOFT_LINK_CONSTANT_FOR_SOURCE(WebCore, CoreMedia, kCMSampleAttachmentKey_DisplayImmediately, CFStringRef)
+
 SOFT_LINK_CONSTANT_FOR_SOURCE(WebCore, CoreMedia, kCMTimebaseNotification_EffectiveRateChanged, CFStringRef)
 SOFT_LINK_CONSTANT_FOR_SOURCE(WebCore, CoreMedia, kCMTimebaseNotification_TimeJumped, CFStringRef)
 SOFT_LINK_FUNCTION_FOR_SOURCE(WebCore, CoreMedia, CMAudioFormatDescriptionGetStreamBasicDescription, const AudioStreamBasicDescription *, (CMAudioFormatDescriptionRef desc), (desc))
index 58d6b2d..f7a21ed 100644 (file)
@@ -166,6 +166,8 @@ SOFT_LINK_CONSTANT_FOR_HEADER(WebCore, CoreMedia, kCMSampleAttachmentKey_DoNotDi
 #define kCMSampleAttachmentKey_DoNotDisplay get_CoreMedia_kCMSampleAttachmentKey_DoNotDisplay()
 SOFT_LINK_CONSTANT_FOR_HEADER(WebCore, CoreMedia, kCMSampleAttachmentKey_NotSync, CFStringRef)
 #define kCMSampleAttachmentKey_NotSync get_CoreMedia_kCMSampleAttachmentKey_NotSync()
+SOFT_LINK_CONSTANT_FOR_HEADER(WebCore, CoreMedia, kCMSampleAttachmentKey_DisplayImmediately, CFStringRef)
+#define kCMSampleAttachmentKey_DisplayImmediately get_CoreMedia_kCMSampleAttachmentKey_DisplayImmediately()
 SOFT_LINK_CONSTANT_FOR_HEADER(WebCore, CoreMedia, kCMSampleBufferAttachmentKey_DisplayEmptyMediaImmediately, CFStringRef)
 #define kCMSampleBufferAttachmentKey_DisplayEmptyMediaImmediately get_CoreMedia_kCMSampleBufferAttachmentKey_DisplayEmptyMediaImmediately()
 SOFT_LINK_CONSTANT_FOR_HEADER(WebCore, CoreMedia, kCMSampleBufferAttachmentKey_DrainAfterDecoding, CFStringRef)
index 167b45c..977f584 100644 (file)
@@ -123,9 +123,12 @@ private:
     void setSize(const IntSize&) override { /* No-op */ }
 
     void enqueueAudioSampleBufferFromTrack(MediaStreamTrackPrivate&, MediaSample&);
-    void enqueueVideoSampleBufferFromTrack(MediaStreamTrackPrivate&, MediaSample&);
+
+    void prepareVideoSampleBufferFromTrack(MediaStreamTrackPrivate&, MediaSample&);
+    void enqueueVideoSampleBuffer(MediaSample&);
     bool shouldEnqueueVideoSampleBuffer() const;
     void flushAndRemoveVideoSampleBuffers();
+    void requestNotificationWhenReadyForMediaData();
 
     void paint(GraphicsContext&, const FloatRect&) override;
     void paintCurrentFrameInContext(GraphicsContext&, const FloatRect&) override;
@@ -194,6 +197,7 @@ private:
 
     HashMap<String, RefPtr<AudioTrackPrivateMediaStream>> m_audioTrackMap;
     HashMap<String, RefPtr<VideoTrackPrivateMediaStream>> m_videoTrackMap;
+    Deque<Ref<MediaSample>> m_sampleQueue;
 
     MediaPlayer::NetworkState m_networkState { MediaPlayer::Empty };
     MediaPlayer::ReadyState m_readyState { MediaPlayer::HaveNothing };
index 3bf4832..d54d2d0 100644 (file)
@@ -128,12 +128,27 @@ void MediaPlayerPrivateMediaStreamAVFObjC::enqueueAudioSampleBufferFromTrack(Med
     // FIXME: https://bugs.webkit.org/show_bug.cgi?id=159836
 }
 
-void MediaPlayerPrivateMediaStreamAVFObjC::enqueueVideoSampleBufferFromTrack(MediaStreamTrackPrivate& track, MediaSample& sample)
+void MediaPlayerPrivateMediaStreamAVFObjC::requestNotificationWhenReadyForMediaData()
 {
-    if (&track != m_mediaStreamPrivate->activeVideoTrack() || !shouldEnqueueVideoSampleBuffer())
-        return;
+    [m_sampleBufferDisplayLayer requestMediaDataWhenReadyOnQueue:dispatch_get_main_queue() usingBlock:^ {
+        [m_sampleBufferDisplayLayer stopRequestingMediaData];
+
+        while (!m_sampleQueue.isEmpty()) {
+            if (![m_sampleBufferDisplayLayer isReadyForMoreMediaData]) {
+                requestNotificationWhenReadyForMediaData();
+                return;
+            }
+
+            auto sample = m_sampleQueue.takeFirst();
+            enqueueVideoSampleBuffer(sample.get());
+        }
+    }];
+}
+
+void MediaPlayerPrivateMediaStreamAVFObjC::enqueueVideoSampleBuffer(MediaSample& sample)
+{
+    ASSERT([m_sampleBufferDisplayLayer isReadyForMoreMediaData]);
 
-    sample.setTimestamps(toMediaTime(CMTimebaseGetTime([m_synchronizer timebase])), MediaTime::invalidTime());
     [m_sampleBufferDisplayLayer enqueueSampleBuffer:sample.platformSample().sample.cmSampleBuffer];
     m_isFrameDisplayed = true;
 
@@ -144,11 +159,22 @@ void MediaPlayerPrivateMediaStreamAVFObjC::enqueueVideoSampleBufferFromTrack(Med
     }
 }
 
-bool MediaPlayerPrivateMediaStreamAVFObjC::shouldEnqueueVideoSampleBuffer() const
+void MediaPlayerPrivateMediaStreamAVFObjC::prepareVideoSampleBufferFromTrack(MediaStreamTrackPrivate& track, MediaSample& sample)
 {
-    if (![m_sampleBufferDisplayLayer isReadyForMoreMediaData])
-        return false;
+    if (&track != m_mediaStreamPrivate->activeVideoTrack() || !shouldEnqueueVideoSampleBuffer())
+        return;
+
+    if (![m_sampleBufferDisplayLayer isReadyForMoreMediaData]) {
+        m_sampleQueue.append(sample);
+        requestNotificationWhenReadyForMediaData();
+        return;
+    }
+
+    enqueueVideoSampleBuffer(sample);
+}
 
+bool MediaPlayerPrivateMediaStreamAVFObjC::shouldEnqueueVideoSampleBuffer() const
+{
     if (m_displayMode == LivePreview)
         return true;
 
@@ -189,6 +215,7 @@ void MediaPlayerPrivateMediaStreamAVFObjC::destroyLayer()
     if (!m_sampleBufferDisplayLayer)
         return;
     
+    [m_sampleBufferDisplayLayer stopRequestingMediaData];
     [m_sampleBufferDisplayLayer flush];
     CMTime currentTime = CMTimebaseGetTime([m_synchronizer timebase]);
     [m_synchronizer removeRenderer:m_sampleBufferDisplayLayer.get() atTime:currentTime withCompletionHandler:^(BOOL){
@@ -525,7 +552,7 @@ void MediaPlayerPrivateMediaStreamAVFObjC::sampleBufferUpdated(MediaStreamTrackP
         // FIXME: https://bugs.webkit.org/show_bug.cgi?id=159836
         break;
     case RealtimeMediaSource::Video:
-        enqueueVideoSampleBufferFromTrack(track, mediaSample);
+        prepareVideoSampleBufferFromTrack(track, mediaSample);
         m_hasReceivedMedia = true;
         scheduleDeferredTask([this] {
             updateReadyState();
index 818d2ba..eb86be8 100644 (file)
@@ -205,6 +205,7 @@ void AVVideoCaptureSource::setupCaptureSession()
                                                          [NSNumber numberWithInt:kCVPixelFormatType_32BGRA], kCVPixelBufferPixelFormatTypeKey
                                                          , nil]);
     [videoOutput setVideoSettings:settingsDictionary.get()];
+    [videoOutput setAlwaysDiscardsLateVideoFrames:YES];
     setVideoSampleBufferDelegate(videoOutput.get());
 
     if (![session() canAddOutput:videoOutput.get()]) {
@@ -258,11 +259,20 @@ void AVVideoCaptureSource::processNewFrame(RetainPtr<CMSampleBufferRef> sampleBu
 
     updateFramerate(sampleBuffer.get());
 
-    bool settingsChanged = false;
+    CMSampleBufferRef newSampleBuffer = 0;
+    CMSampleBufferCreateCopy(kCFAllocatorDefault, sampleBuffer.get(), &newSampleBuffer);
+    ASSERT(newSampleBuffer);
+
+    CFArrayRef attachmentsArray = CMSampleBufferGetSampleAttachmentsArray(newSampleBuffer, true);
+    for (CFIndex i = 0; i < CFArrayGetCount(attachmentsArray); ++i) {
+        CFMutableDictionaryRef attachments = (CFMutableDictionaryRef)CFArrayGetValueAtIndex(attachmentsArray, i);
+        CFDictionarySetValue(attachments, kCMSampleAttachmentKey_DisplayImmediately, kCFBooleanTrue);
+    }
 
-    m_buffer = sampleBuffer;
+    m_buffer = newSampleBuffer;
     m_lastImage = nullptr;
 
+    bool settingsChanged = false;
     CMVideoDimensions dimensions = CMVideoFormatDescriptionGetDimensions(formatDescription);
     if (dimensions.width != m_width || dimensions.height != m_height) {
         m_width = dimensions.width;
@@ -273,7 +283,7 @@ void AVVideoCaptureSource::processNewFrame(RetainPtr<CMSampleBufferRef> sampleBu
     if (settingsChanged)
         settingsDidChange();
 
-    mediaDataUpdated(MediaSampleAVFObjC::create(sampleBuffer.get()));
+    mediaDataUpdated(MediaSampleAVFObjC::create(m_buffer.get()));
 }
 
 void AVVideoCaptureSource::captureOutputDidOutputSampleBufferFromConnection(AVCaptureOutputType*, CMSampleBufferRef sampleBuffer, AVCaptureConnectionType*)