[MediaStream] Observer AVCaptureDevice "suspended" property
authoreric.carlson@apple.com <eric.carlson@apple.com@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Tue, 13 Nov 2018 22:21:01 +0000 (22:21 +0000)
committereric.carlson@apple.com <eric.carlson@apple.com@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Tue, 13 Nov 2018 22:21:01 +0000 (22:21 +0000)
https://bugs.webkit.org/show_bug.cgi?id=191587
<rdar://problem/46030598>

Reviewed by Youenn Fablet.

No new tests, AVCapture can only be tested manually.

* platform/mediastream/mac/AVCaptureDeviceManager.h:
* platform/mediastream/mac/AVCaptureDeviceManager.mm:
(WebCore::AVCaptureDeviceManager::captureDevicesInternal): Don't notify of devices "changes"
the first time the device list is scanned.
(WebCore::deviceIsAvailable): Don't check for "isInUseByAnotherApplication", it doesn't
change device availability.
(WebCore::AVCaptureDeviceManager::beginObservingDevices): New, observe "suspended" on all
devices and add them to the cached list.
(WebCore::AVCaptureDeviceManager::stopObservingDevices): New, opposite of above.
(WebCore::AVCaptureDeviceManager::refreshCaptureDevices): Watch for changes in the list of
devices.
(WebCore::AVCaptureDeviceManager::~AVCaptureDeviceManager): Stop observing all cached devices.
(WebCore::AVCaptureDeviceManager::registerForDeviceNotifications):
(-[WebCoreAVCaptureDeviceManagerObserver disconnect]):
(-[WebCoreAVCaptureDeviceManagerObserver deviceConnectedDidChange:]):
(-[WebCoreAVCaptureDeviceManagerObserver observeValueForKeyPath:ofObject:change:context:]):
(WebCore::AVCaptureDeviceManager::refreshAVCaptureDevicesOfType): Deleted.
(WebCore::AVCaptureDeviceManager::deviceConnected): Deleted.
(WebCore::AVCaptureDeviceManager::deviceDisconnected): Deleted.
(-[WebCoreAVCaptureDeviceManagerObserver deviceDisconnected:]): Deleted.
(-[WebCoreAVCaptureDeviceManagerObserver deviceConnected:]): Deleted.

* platform/mediastream/mac/AVVideoCaptureSource.h:
* platform/mediastream/mac/AVVideoCaptureSource.mm:
(WebCore::AVVideoCaptureSource::~AVVideoCaptureSource): Stop observing "running" (not "rate")
and "suspended".
(WebCore::AVVideoCaptureSource::setupSession): Observe "running" (not "rate"), and "suspended".
(WebCore::AVVideoCaptureSource::captureDeviceSuspendedDidChange):
(-[WebCoreAVVideoCaptureSourceObserver observeValueForKeyPath:ofObject:change:context:]):

git-svn-id: https://svn.webkit.org/repository/webkit/trunk@238145 268f45cc-cd09-0410-ab3c-d52691b4dbfc

Source/WebCore/ChangeLog
Source/WebCore/platform/mediastream/mac/AVCaptureDeviceManager.h
Source/WebCore/platform/mediastream/mac/AVCaptureDeviceManager.mm
Source/WebCore/platform/mediastream/mac/AVVideoCaptureSource.h
Source/WebCore/platform/mediastream/mac/AVVideoCaptureSource.mm

index c710269..7e8afec 100644 (file)
@@ -1,3 +1,43 @@
+2018-11-13  Eric Carlson  <eric.carlson@apple.com>
+
+        [MediaStream] Observer AVCaptureDevice "suspended" property
+        https://bugs.webkit.org/show_bug.cgi?id=191587
+        <rdar://problem/46030598>
+
+        Reviewed by Youenn Fablet.
+
+        No new tests, AVCapture can only be tested manually.
+
+        * platform/mediastream/mac/AVCaptureDeviceManager.h:
+        * platform/mediastream/mac/AVCaptureDeviceManager.mm:
+        (WebCore::AVCaptureDeviceManager::captureDevicesInternal): Don't notify of devices "changes"
+        the first time the device list is scanned.
+        (WebCore::deviceIsAvailable): Don't check for "isInUseByAnotherApplication", it doesn't
+        change device availability.
+        (WebCore::AVCaptureDeviceManager::beginObservingDevices): New, observe "suspended" on all 
+        devices and add them to the cached list.
+        (WebCore::AVCaptureDeviceManager::stopObservingDevices): New, opposite of above.
+        (WebCore::AVCaptureDeviceManager::refreshCaptureDevices): Watch for changes in the list of
+        devices.
+        (WebCore::AVCaptureDeviceManager::~AVCaptureDeviceManager): Stop observing all cached devices.
+        (WebCore::AVCaptureDeviceManager::registerForDeviceNotifications):
+        (-[WebCoreAVCaptureDeviceManagerObserver disconnect]):
+        (-[WebCoreAVCaptureDeviceManagerObserver deviceConnectedDidChange:]):
+        (-[WebCoreAVCaptureDeviceManagerObserver observeValueForKeyPath:ofObject:change:context:]):
+        (WebCore::AVCaptureDeviceManager::refreshAVCaptureDevicesOfType): Deleted.
+        (WebCore::AVCaptureDeviceManager::deviceConnected): Deleted.
+        (WebCore::AVCaptureDeviceManager::deviceDisconnected): Deleted.
+        (-[WebCoreAVCaptureDeviceManagerObserver deviceDisconnected:]): Deleted.
+        (-[WebCoreAVCaptureDeviceManagerObserver deviceConnected:]): Deleted.
+
+        * platform/mediastream/mac/AVVideoCaptureSource.h:
+        * platform/mediastream/mac/AVVideoCaptureSource.mm:
+        (WebCore::AVVideoCaptureSource::~AVVideoCaptureSource): Stop observing "running" (not "rate")
+        and "suspended".
+        (WebCore::AVVideoCaptureSource::setupSession): Observe "running" (not "rate"), and "suspended".
+        (WebCore::AVVideoCaptureSource::captureDeviceSuspendedDidChange):
+        (-[WebCoreAVVideoCaptureSourceObserver observeValueForKeyPath:ofObject:change:context:]):
+
 2018-11-13  Devin Rousso  <drousso@apple.com>
 
         Web Inspector: REGRESSION(r238122): fetching the CertificateInfo triggers an ASSERT in workers
index b21ca9b..072a259 100644 (file)
@@ -37,6 +37,8 @@
 
 OBJC_CLASS AVCaptureDevice;
 OBJC_CLASS AVCaptureSession;
+OBJC_CLASS NSArray;
+OBJC_CLASS NSMutableArray;
 OBJC_CLASS NSString;
 OBJC_CLASS WebCoreAVCaptureDeviceManagerObserver;
 
@@ -49,22 +51,24 @@ public:
 
     static AVCaptureDeviceManager& singleton();
 
-    void deviceConnected();
     void deviceDisconnected(AVCaptureDevice*);
 
+    void refreshCaptureDevices();
+
 protected:
     static bool isAvailable();
 
     AVCaptureDeviceManager();
     ~AVCaptureDeviceManager() final;
 
-    void refreshCaptureDevices();
     void registerForDeviceNotifications();
-    void refreshAVCaptureDevicesOfType(CaptureDevice::DeviceType);
     Vector<CaptureDevice>& captureDevicesInternal();
+    void updateCachedAVCaptureDevices();
 
     RetainPtr<WebCoreAVCaptureDeviceManagerObserver> m_objcObserver;
     Vector<CaptureDevice> m_devices;
+    RetainPtr<NSMutableArray> m_avCaptureDevices;
+    bool m_notifyWhenDeviceListChanges { false };
 };
 
 } // namespace WebCore
index 39839d9..10626aa 100644 (file)
@@ -72,8 +72,8 @@ using namespace WebCore;
 
 -(id)initWithCallback:(AVCaptureDeviceManager*)callback;
 -(void)disconnect;
--(void)deviceDisconnected:(NSNotification *)notification;
--(void)deviceConnected:(NSNotification *)notification;
+-(void)deviceConnectedDidChange:(NSNotification *)notification;
+-(void)observeValueForKeyPath:keyPath ofObject:(id)object change:(NSDictionary*)change context:(void*)context;
 @end
 
 namespace WebCore {
@@ -85,10 +85,10 @@ Vector<CaptureDevice>& AVCaptureDeviceManager::captureDevicesInternal()
         return m_devices;
 
     static bool firstTime = true;
-    if (firstTime && !m_devices.size()) {
+    if (firstTime) {
         firstTime = false;
         refreshCaptureDevices();
-        registerForDeviceNotifications();
+        m_notifyWhenDeviceListChanges = true;
     }
 
     return m_devices;
@@ -105,38 +105,76 @@ inline static bool deviceIsAvailable(AVCaptureDeviceTypedef *device)
         return false;
 
 #if !PLATFORM(IOS_FAMILY)
-    if ([device isSuspended] || [device isInUseByAnotherApplication])
+    if ([device isSuspended])
         return false;
 #endif
 
     return true;
 }
 
-void AVCaptureDeviceManager::refreshAVCaptureDevicesOfType(CaptureDevice::DeviceType type)
+void AVCaptureDeviceManager::updateCachedAVCaptureDevices()
 {
-    ASSERT(type == CaptureDevice::DeviceType::Camera || type == CaptureDevice::DeviceType::Microphone);
+    auto* currentDevices = [getAVCaptureDeviceClass() devices];
+    auto changedDevices = adoptNS([[NSMutableArray alloc] init]);
+    for (AVCaptureDeviceTypedef *cachedDevice in m_avCaptureDevices.get()) {
+        if (![currentDevices containsObject:cachedDevice])
+            [changedDevices addObject:cachedDevice];
+    }
 
-    NSString *platformType = (type == CaptureDevice::DeviceType::Camera) ? AVMediaTypeVideo : AVMediaTypeAudio;
+    if ([changedDevices count]) {
+        for (AVCaptureDeviceTypedef *device in changedDevices.get())
+            [device removeObserver:m_objcObserver.get() forKeyPath:@"suspended"];
+        [m_avCaptureDevices removeObjectsInArray:changedDevices.get()];
+    }
 
-    for (AVCaptureDeviceTypedef *platformDevice in [getAVCaptureDeviceClass() devices]) {
+    for (AVCaptureDeviceTypedef *device in currentDevices) {
 
-        bool hasMatchingType = [platformDevice hasMediaType:platformType] || [platformDevice hasMediaType:AVMediaTypeMuxed];
-        if (!hasMatchingType)
+        if (![device hasMediaType:AVMediaTypeVideo] && ![device hasMediaType:AVMediaTypeMuxed])
             continue;
 
-        CaptureDevice existingCaptureDevice = captureDeviceFromPersistentID(platformDevice.uniqueID);
-        if (existingCaptureDevice && existingCaptureDevice.type() == type)
+        if ([m_avCaptureDevices.get() containsObject:device])
             continue;
 
-        CaptureDevice captureDevice(platformDevice.uniqueID, type, platformDevice.localizedName);
-        captureDevice.setEnabled(deviceIsAvailable(platformDevice));
-        m_devices.append(captureDevice);
+        [device addObserver:m_objcObserver.get() forKeyPath:@"suspended" options:NSKeyValueObservingOptionNew context:(void *)nil];
+        [m_avCaptureDevices.get() addObject:device];
     }
+
 }
 
 void AVCaptureDeviceManager::refreshCaptureDevices()
 {
-    refreshAVCaptureDevicesOfType(CaptureDevice::DeviceType::Camera);
+    if (!m_avCaptureDevices) {
+        m_avCaptureDevices = adoptNS([[NSMutableArray alloc] init]);
+        registerForDeviceNotifications();
+    }
+
+    updateCachedAVCaptureDevices();
+
+    bool deviceHasChanged = false;
+    auto* currentDevices = [getAVCaptureDeviceClass() devices];
+    Vector<CaptureDevice> deviceList;
+    for (AVCaptureDeviceTypedef *platformDevice in currentDevices) {
+
+        if (![platformDevice hasMediaType:AVMediaTypeVideo] && ![platformDevice hasMediaType:AVMediaTypeMuxed])
+            continue;
+
+        CaptureDevice captureDevice(platformDevice.uniqueID, CaptureDevice::DeviceType::Camera, platformDevice.localizedName);
+        captureDevice.setEnabled(deviceIsAvailable(platformDevice));
+
+        CaptureDevice existingCaptureDevice = captureDeviceFromPersistentID(platformDevice.uniqueID);
+        if (!existingCaptureDevice || (existingCaptureDevice && existingCaptureDevice.type() == CaptureDevice::DeviceType::Camera && captureDevice.enabled() != existingCaptureDevice.enabled()))
+            deviceHasChanged = true;
+
+        deviceList.append(WTFMove(captureDevice));
+    }
+
+    if (deviceHasChanged || m_devices.size() != deviceList.size()) {
+        deviceHasChanged = true;
+        m_devices = WTFMove(deviceList);
+    }
+
+    if (m_notifyWhenDeviceListChanges && deviceHasChanged)
+        deviceChanged();
 }
 
 bool AVCaptureDeviceManager::isAvailable()
@@ -159,37 +197,14 @@ AVCaptureDeviceManager::~AVCaptureDeviceManager()
 {
     [[NSNotificationCenter defaultCenter] removeObserver:m_objcObserver.get()];
     [m_objcObserver disconnect];
+    for (AVCaptureDeviceTypedef *device in m_avCaptureDevices.get())
+        [device removeObserver:m_objcObserver.get() forKeyPath:@"suspended"];
 }
 
 void AVCaptureDeviceManager::registerForDeviceNotifications()
 {
-    [[NSNotificationCenter defaultCenter] addObserver:m_objcObserver.get() selector:@selector(deviceConnected:) name:AVCaptureDeviceWasConnectedNotification object:nil];
-    [[NSNotificationCenter defaultCenter] addObserver:m_objcObserver.get() selector:@selector(deviceDisconnected:) name:AVCaptureDeviceWasDisconnectedNotification object:nil];
-}
-
-void AVCaptureDeviceManager::deviceConnected()
-{
-    refreshCaptureDevices();
-    deviceChanged();
-}
-
-void AVCaptureDeviceManager::deviceDisconnected(AVCaptureDeviceTypedef* device)
-{
-    auto& devices = captureDevicesInternal();
-
-    size_t count = devices.size();
-    if (!count)
-        return;
-
-    String deviceID = device.uniqueID;
-    for (size_t i = 0; i < count; ++i) {
-        if (devices[i].persistentId() == deviceID) {
-            LOG(Media, "AVCaptureDeviceManager::deviceDisconnected(%p), device %d disabled", this, i);
-            devices[i].setEnabled(false);
-        }
-    }
-
-    deviceChanged();
+    [[NSNotificationCenter defaultCenter] addObserver:m_objcObserver.get() selector:@selector(deviceConnectedDidChange:) name:AVCaptureDeviceWasConnectedNotification object:nil];
+    [[NSNotificationCenter defaultCenter] addObserver:m_objcObserver.get() selector:@selector(deviceConnectedDidChange:) name:AVCaptureDeviceWasDisconnectedNotification object:nil];
 }
 
 } // namespace WebCore
@@ -208,32 +223,32 @@ void AVCaptureDeviceManager::deviceDisconnected(AVCaptureDeviceTypedef* device)
 - (void)disconnect
 {
     [NSObject cancelPreviousPerformRequestsWithTarget:self];
-    m_callback = 0;
+    m_callback = nil;
 }
 
-- (void)deviceDisconnected:(NSNotification *)notification
+- (void)deviceConnectedDidChange:(NSNotification *)unusedNotification
 {
+    UNUSED_PARAM(unusedNotification);
     if (!m_callback)
         return;
 
     dispatch_async(dispatch_get_main_queue(), ^{
-        if (m_callback) {
-            AVCaptureDeviceTypedef *device = [notification object];
-            m_callback->deviceDisconnected(device);
-        }
+        if (m_callback)
+            m_callback->refreshCaptureDevices();
     });
 }
 
-- (void)deviceConnected:(NSNotification *)unusedNotification
+- (void)observeValueForKeyPath:keyPath ofObject:(id)object change:(NSDictionary*)change context:(void*)context
 {
-    UNUSED_PARAM(unusedNotification);
+    UNUSED_PARAM(object);
+    UNUSED_PARAM(context);
+    UNUSED_PARAM(change);
+
     if (!m_callback)
         return;
 
-    dispatch_async(dispatch_get_main_queue(), ^{
-        if (m_callback)
-            m_callback->deviceConnected();
-    });
+    if ([keyPath isEqualToString:@"suspended"])
+        m_callback->refreshCaptureDevices();
 }
 
 @end
index 653c8de..a83d661 100644 (file)
@@ -66,6 +66,7 @@ public:
     void captureSessionIsRunningDidChange(bool);
     void captureSessionRuntimeError(RetainPtr<NSError>);
     void captureOutputDidOutputSampleBufferFromConnection(AVCaptureOutput*, CMSampleBufferRef, AVCaptureConnection*);
+    void captureDeviceSuspendedDidChange();
 
 private:
     AVVideoCaptureSource(AVCaptureDevice*, String&& id, String&& hashSalt);
index 198148c..4017961 100644 (file)
@@ -189,7 +189,8 @@ AVVideoCaptureSource::~AVVideoCaptureSource()
     if (!m_session)
         return;
 
-    [m_session removeObserver:m_objcObserver.get() forKeyPath:@"rate"];
+    [m_session removeObserver:m_objcObserver.get() forKeyPath:@"running"];
+    [m_device removeObserver:m_objcObserver.get() forKeyPath:@"suspended"];
     if ([m_session isRunning])
         [m_session stopRunning];
 }
@@ -400,7 +401,8 @@ bool AVVideoCaptureSource::setupSession()
         return true;
 
     m_session = adoptNS([allocAVCaptureSessionInstance() init]);
-    [m_session addObserver:m_objcObserver.get() forKeyPath:@"rate" options:NSKeyValueObservingOptionNew context:(void *)nil];
+    [m_session addObserver:m_objcObserver.get() forKeyPath:@"running" options:NSKeyValueObservingOptionNew context:(void *)nil];
+    [m_device addObserver:m_objcObserver.get() forKeyPath:@"suspended" options:NSKeyValueObservingOptionNew context:(void *)nil];
 
     [m_session beginConfiguration];
     bool success = setupCaptureSession();
@@ -543,6 +545,19 @@ void AVVideoCaptureSource::captureSessionIsRunningDidChange(bool state)
     });
 }
 
+void AVVideoCaptureSource::captureDeviceSuspendedDidChange()
+{
+#if !PLATFORM(IOS_FAMILY)
+    scheduleDeferredTask([this] {
+        auto isSuspended = [m_device isSuspended];
+        if (isSuspended == muted())
+            return;
+
+        notifyMutedChange(isSuspended);
+    });
+#endif
+}
+
 bool AVVideoCaptureSource::interrupted() const
 {
     if (m_interruption != InterruptionReason::None)
@@ -664,8 +679,8 @@ void AVVideoCaptureSource::captureSessionEndInterruption(RetainPtr<NSNotificatio
 
     id newValue = [change valueForKey:NSKeyValueChangeNewKey];
 
-#if !LOG_DISABLED
     bool willChange = [[change valueForKey:NSKeyValueChangeNotificationIsPriorKey] boolValue];
+#if !LOG_DISABLED
 
     if (willChange)
         LOG(Media, "WebCoreAVVideoCaptureSourceObserver::observeValueForKeyPath(%p) - will change, keyPath = %s", self, [keyPath UTF8String]);
@@ -675,8 +690,10 @@ void AVVideoCaptureSource::captureSessionEndInterruption(RetainPtr<NSNotificatio
     }
 #endif
 
-    if ([keyPath isEqualToString:@"running"])
+    if (!willChange && [keyPath isEqualToString:@"running"])
         m_callback->captureSessionIsRunningDidChange([newValue boolValue]);
+    if (!willChange && [keyPath isEqualToString:@"suspended"])
+        m_callback->captureDeviceSuspendedDidChange();
 }
 
 #if PLATFORM(IOS_FAMILY)