'ended' Event doesn't fire on MediaStreamTrack when a USB camera is unplugged
authoreric.carlson@apple.com <eric.carlson@apple.com@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Sat, 22 Dec 2018 01:37:00 +0000 (01:37 +0000)
committereric.carlson@apple.com <eric.carlson@apple.com@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Sat, 22 Dec 2018 01:37:00 +0000 (01:37 +0000)
https://bugs.webkit.org/show_bug.cgi?id=187896
<rdar://problem/42681445>

Reviewed by Jer Noble.

Source/WebCore:

No new tests, tested manually.

* platform/mediastream/mac/AVVideoCaptureSource.h:
* platform/mediastream/mac/AVVideoCaptureSource.mm:
(WebCore::AVVideoCaptureSource::deviceDisconnected):
(-[WebCoreAVVideoCaptureSourceObserver addNotificationObservers]):
(-[WebCoreAVVideoCaptureSourceObserver removeNotificationObservers]):
(-[WebCoreAVVideoCaptureSourceObserver deviceConnectedDidChange:]):
* platform/mediastream/mac/CoreAudioCaptureDeviceManager.cpp:
(WebCore::deviceHasInputStreams):
(WebCore::isValidCaptureDevice):
(WebCore::CoreAudioCaptureDeviceManager::coreAudioCaptureDevices):
(WebCore::CoreAudioCaptureDeviceManager::refreshAudioCaptureDevices):
(WebCore::CoreAudioCaptureDeviceManager::devicesChanged): Deleted.
* platform/mediastream/mac/CoreAudioCaptureDeviceManager.h:
* platform/mediastream/mac/CoreAudioCaptureSource.cpp:
(WebCore::CoreAudioSharedUnit::setCaptureDevice):
(WebCore::CoreAudioSharedUnit::devicesChanged):
(WebCore::CoreAudioSharedUnit::startProducingData):
(WebCore::CoreAudioSharedUnit::startInternal):
(WebCore::CoreAudioSharedUnit::verifyIsCapturing):
(WebCore::CoreAudioSharedUnit::captureFailed):
(WebCore::CoreAudioCaptureSourceFactory::devicesChanged):
(WebCore::CoreAudioCaptureSource::CoreAudioCaptureSource):
(WebCore::CoreAudioSharedUnit::setCaptureDeviceID): Deleted.
* platform/mediastream/mac/CoreAudioCaptureSource.h:

Source/WebCore/PAL:

* pal/spi/cf/CoreAudioSPI.h:

git-svn-id: https://svn.webkit.org/repository/webkit/trunk@239531 268f45cc-cd09-0410-ab3c-d52691b4dbfc

Source/WebCore/ChangeLog
Source/WebCore/PAL/ChangeLog
Source/WebCore/PAL/pal/spi/cf/CoreAudioSPI.h
Source/WebCore/platform/mediastream/mac/AVVideoCaptureSource.h
Source/WebCore/platform/mediastream/mac/AVVideoCaptureSource.mm
Source/WebCore/platform/mediastream/mac/CoreAudioCaptureDeviceManager.cpp
Source/WebCore/platform/mediastream/mac/CoreAudioCaptureDeviceManager.h
Source/WebCore/platform/mediastream/mac/CoreAudioCaptureSource.cpp
Source/WebCore/platform/mediastream/mac/CoreAudioCaptureSource.h

index a98326a..5bb8de0 100644 (file)
@@ -1,3 +1,38 @@
+2018-12-21  Eric Carlson  <eric.carlson@apple.com>
+
+        'ended' Event doesn't fire on MediaStreamTrack when a USB camera is unplugged
+        https://bugs.webkit.org/show_bug.cgi?id=187896
+        <rdar://problem/42681445>
+
+        Reviewed by Jer Noble.
+
+        No new tests, tested manually.
+
+        * platform/mediastream/mac/AVVideoCaptureSource.h:
+        * platform/mediastream/mac/AVVideoCaptureSource.mm:
+        (WebCore::AVVideoCaptureSource::deviceDisconnected):
+        (-[WebCoreAVVideoCaptureSourceObserver addNotificationObservers]):
+        (-[WebCoreAVVideoCaptureSourceObserver removeNotificationObservers]):
+        (-[WebCoreAVVideoCaptureSourceObserver deviceConnectedDidChange:]):
+        * platform/mediastream/mac/CoreAudioCaptureDeviceManager.cpp:
+        (WebCore::deviceHasInputStreams):
+        (WebCore::isValidCaptureDevice):
+        (WebCore::CoreAudioCaptureDeviceManager::coreAudioCaptureDevices):
+        (WebCore::CoreAudioCaptureDeviceManager::refreshAudioCaptureDevices):
+        (WebCore::CoreAudioCaptureDeviceManager::devicesChanged): Deleted.
+        * platform/mediastream/mac/CoreAudioCaptureDeviceManager.h:
+        * platform/mediastream/mac/CoreAudioCaptureSource.cpp:
+        (WebCore::CoreAudioSharedUnit::setCaptureDevice):
+        (WebCore::CoreAudioSharedUnit::devicesChanged):
+        (WebCore::CoreAudioSharedUnit::startProducingData):
+        (WebCore::CoreAudioSharedUnit::startInternal):
+        (WebCore::CoreAudioSharedUnit::verifyIsCapturing):
+        (WebCore::CoreAudioSharedUnit::captureFailed):
+        (WebCore::CoreAudioCaptureSourceFactory::devicesChanged):
+        (WebCore::CoreAudioCaptureSource::CoreAudioCaptureSource):
+        (WebCore::CoreAudioSharedUnit::setCaptureDeviceID): Deleted.
+        * platform/mediastream/mac/CoreAudioCaptureSource.h:
+
 2018-12-20  Ryosuke Niwa  <rniwa@webkit.org>
 
         REGRESSION(r239353): iOS WK1 Assertion failure in notifyChildNodeRemoved while running
index 2ba4673..320423d 100644 (file)
@@ -1,3 +1,13 @@
+2018-12-21  Eric Carlson  <eric.carlson@apple.com>
+
+        'ended' Event doesn't fire on MediaStreamTrack when a USB camera is unplugged
+        https://bugs.webkit.org/show_bug.cgi?id=187896
+        <rdar://problem/42681445>
+
+        Reviewed by Jer Noble.
+
+        * pal/spi/cf/CoreAudioSPI.h:
+
 2018-12-19  Chris Dumez  <cdumez@apple.com>
 
         wtf/Optional.h: move-constructor and move-assignment operator should disengage the value being moved from
index 37799d2..d5e9da3 100644 (file)
 
 #if PLATFORM(MAC)
 #include <CoreAudio/AudioHardware.h>
+
+CF_ENUM(AudioObjectPropertySelector)
+{
+    kAudioDevicePropertyTapEnabled = 'tapd',
+};
+
 #else
 
 WTF_EXTERN_C_BEGIN
@@ -55,7 +61,8 @@ CF_ENUM(AudioObjectPropertyScope)
 
 CF_ENUM(AudioObjectPropertySelector)
 {
-    kAudioHardwarePropertyDefaultInputDevice = 'dIn '
+    kAudioHardwarePropertyDefaultInputDevice = 'dIn ',
+    kAudioDevicePropertyTapEnabled = 'tapd',
 };
 
 CF_ENUM(int)
index 2a53054..6038172 100644 (file)
@@ -60,6 +60,7 @@ public:
     enum class InterruptionReason { None, VideoNotAllowedInBackground, AudioInUse, VideoInUse, VideoNotAllowedInSideBySide };
     void captureSessionBeginInterruption(RetainPtr<NSNotification>);
     void captureSessionEndInterruption(RetainPtr<NSNotification>);
+    void deviceDisconnected(RetainPtr<NSNotification>);
 
     AVCaptureSession* session() const { return m_session.get(); }
 
index a54cd14..8a00f04 100644 (file)
@@ -78,6 +78,9 @@ SOFT_LINK_CLASS(AVFoundation, AVCaptureSession)
 
 SOFT_LINK_CONSTANT(AVFoundation, AVMediaTypeVideo, NSString *)
 
+SOFT_LINK_CONSTANT(AVFoundation, AVCaptureDeviceWasDisconnectedNotification, NSString *)
+#define AVCaptureDeviceWasDisconnectedNotification getAVCaptureDeviceWasDisconnectedNotification()
+
 #if PLATFORM(IOS_FAMILY)
 SOFT_LINK_POINTER_OPTIONAL(AVFoundation, AVCaptureSessionRuntimeErrorNotification, NSString *)
 SOFT_LINK_POINTER_OPTIONAL(AVFoundation, AVCaptureSessionWasInterruptedNotification, NSString *)
@@ -109,6 +112,7 @@ using namespace PAL;
 -(void)sessionRuntimeError:(NSNotification*)notification;
 -(void)beginSessionInterrupted:(NSNotification*)notification;
 -(void)endSessionInterrupted:(NSNotification*)notification;
+-(void)deviceConnectedDidChange:(NSNotification*)notification;
 #endif
 @end
 
@@ -626,6 +630,13 @@ void AVVideoCaptureSource::captureSessionEndInterruption(RetainPtr<NSNotificatio
 }
 #endif
 
+void AVVideoCaptureSource::deviceDisconnected(RetainPtr<NSNotification> notification)
+{
+    if (this->device() == [notification object])
+        captureFailed();
+}
+
+
 } // namespace WebCore
 
 @implementation WebCoreAVVideoCaptureSourceObserver
@@ -650,12 +661,14 @@ void AVVideoCaptureSource::captureSessionEndInterruption(RetainPtr<NSNotificatio
 
 - (void)addNotificationObservers
 {
-#if PLATFORM(IOS_FAMILY)
     ASSERT(m_callback);
 
     NSNotificationCenter* center = [NSNotificationCenter defaultCenter];
-    AVCaptureSessionType* session = m_callback->session();
 
+    [center addObserver:self selector:@selector(deviceConnectedDidChange:) name:AVCaptureDeviceWasDisconnectedNotification object:nil];
+
+#if PLATFORM(IOS_FAMILY)
+    AVCaptureSessionType* session = m_callback->session();
     [center addObserver:self selector:@selector(sessionRuntimeError:) name:AVCaptureSessionRuntimeErrorNotification object:session];
     [center addObserver:self selector:@selector(beginSessionInterrupted:) name:AVCaptureSessionWasInterruptedNotification object:session];
     [center addObserver:self selector:@selector(endSessionInterrupted:) name:AVCaptureSessionInterruptionEndedNotification object:session];
@@ -664,9 +677,7 @@ void AVVideoCaptureSource::captureSessionEndInterruption(RetainPtr<NSNotificatio
 
 - (void)removeNotificationObservers
 {
-#if PLATFORM(IOS_FAMILY)
     [[NSNotificationCenter defaultCenter] removeObserver:self];
-#endif
 }
 
 - (void)captureOutput:(AVCaptureOutputType*)captureOutput didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer fromConnection:(AVCaptureConnectionType*)connection
@@ -704,6 +715,14 @@ void AVVideoCaptureSource::captureSessionEndInterruption(RetainPtr<NSNotificatio
         m_callback->captureDeviceSuspendedDidChange();
 }
 
+- (void)deviceConnectedDidChange:(NSNotification*)notification
+{
+    LOG(Media, "WebCoreAVVideoCaptureSourceObserver::deviceConnectedDidChange(%p)", self);
+
+    if (m_callback)
+        m_callback->deviceDisconnected(notification);
+}
+
 #if PLATFORM(IOS_FAMILY)
 - (void)sessionRuntimeError:(NSNotification*)notification
 {
index 10956ba..1c78ae5 100644 (file)
 #if ENABLE(MEDIA_STREAM) && PLATFORM(MAC)
 
 #include "CoreAudioCaptureDevice.h"
+#include "CoreAudioCaptureSource.h"
 #include "Logging.h"
 #include "RealtimeMediaSourceCenter.h"
 #include <AudioUnit/AudioUnit.h>
 #include <CoreMedia/CMSync.h>
+#include <pal/spi/cf/CoreAudioSPI.h>
 #include <wtf/Assertions.h>
 #include <wtf/NeverDestroyed.h>
 
@@ -67,7 +69,6 @@ static bool deviceHasInputStreams(AudioObjectID deviceID)
     UInt32 dataSize = 0;
     AudioObjectPropertyAddress address = { kAudioDevicePropertyStreamConfiguration, kAudioDevicePropertyScopeInput, kAudioObjectPropertyElementMaster };
     auto err = AudioObjectGetPropertyDataSize(deviceID, &address, 0, nullptr, &dataSize);
-
     if (err || !dataSize)
         return false;
 
@@ -80,6 +81,23 @@ static bool deviceHasInputStreams(AudioObjectID deviceID)
 
 static bool isValidCaptureDevice(const CoreAudioCaptureDevice& device)
 {
+    // Ignore output devices that have input only for echo cancellation.
+    AudioObjectPropertyAddress address = { kAudioDevicePropertyTapEnabled, kAudioDevicePropertyScopeOutput, kAudioObjectPropertyElementMaster };
+    if (AudioObjectHasProperty(device.deviceID(), &address))
+        return false;
+
+    // Ignore non-aggregable devices.
+    UInt32 dataSize = 0;
+    address = { kAudioObjectPropertyCreator, kAudioObjectPropertyScopeGlobal, kAudioObjectPropertyElementMaster };
+    CFStringRef name = nullptr;
+    dataSize = sizeof(name);
+    AudioObjectGetPropertyData(device.deviceID(), &address, 0, nullptr, &dataSize, &name);
+    bool isNonAggregable = !name || !String(name).startsWith("com.apple.audio.CoreAudio");
+    if (name)
+        CFRelease(name);
+    if (isNonAggregable)
+        return false;
+
     // Ignore unnamed devices and aggregate devices created by VPIO.
     return !device.label().isEmpty() && !device.label().startsWith("VPAUAggregateAudioDevice");
 }
@@ -91,8 +109,24 @@ Vector<CoreAudioCaptureDevice>& CoreAudioCaptureDeviceManager::coreAudioCaptureD
         initialized = true;
         refreshAudioCaptureDevices(DoNotNotify);
 
+        auto weakThis = makeWeakPtr(*this);
+        m_listenerBlock = Block_copy(^(UInt32 count, const AudioObjectPropertyAddress properties[]) {
+            if (!weakThis)
+                return;
+
+            for (UInt32 i = 0; i < count; ++i) {
+                const AudioObjectPropertyAddress& property = properties[i];
+
+                if (property.mSelector != kAudioHardwarePropertyDevices)
+                    continue;
+
+                weakThis->refreshAudioCaptureDevices(Notify);
+                return;
+            }
+        });
+
         AudioObjectPropertyAddress address = { kAudioHardwarePropertyDevices, kAudioObjectPropertyScopeGlobal, kAudioObjectPropertyElementMaster };
-        auto err = AudioObjectAddPropertyListener(kAudioObjectSystemObject, &address, devicesChanged, this);
+        auto err = AudioObjectAddPropertyListenerBlock(kAudioObjectSystemObject, &address, dispatch_get_main_queue(), m_listenerBlock);
         if (err)
             LOG_ERROR("CoreAudioCaptureDeviceManager::devices(%p) AudioObjectAddPropertyListener returned error %d (%.4s)", this, (int)err, (char*)&err);
     }
@@ -164,14 +198,10 @@ void CoreAudioCaptureDeviceManager::refreshAudioCaptureDevices(NotifyIfDevicesHa
         m_devices.append(captureDevice);
     }
 
-    if (notify == Notify)
+    if (notify == Notify) {
         deviceChanged();
-}
-
-OSStatus CoreAudioCaptureDeviceManager::devicesChanged(AudioObjectID, UInt32, const AudioObjectPropertyAddress*, void* userData)
-{
-    static_cast<CoreAudioCaptureDeviceManager*>(userData)->refreshAudioCaptureDevices(Notify);
-    return 0;
+        CoreAudioCaptureSourceFactory::singleton().devicesChanged(m_devices);
+    }
 }
 
 } // namespace WebCore
index 5c972f1..19462e4 100644 (file)
@@ -52,7 +52,6 @@ private:
     CoreAudioCaptureDeviceManager() = default;
     ~CoreAudioCaptureDeviceManager() = default;
     
-    static OSStatus devicesChanged(AudioObjectID, UInt32, const AudioObjectPropertyAddress*, void*);
     Vector<CoreAudioCaptureDevice>& coreAudioCaptureDevices();
 
     enum NotifyIfDevicesHaveChanged { Notify, DoNotNotify };
@@ -60,6 +59,8 @@ private:
 
     Vector<CaptureDevice> m_devices;
     Vector<CoreAudioCaptureDevice> m_coreAudioCaptureDevices;
+
+    AudioObjectPropertyListenerBlock m_listenerBlock;
 };
 
 } // namespace WebCore
index 57d6b01..dd13a52 100644 (file)
@@ -46,6 +46,7 @@
 #include <pal/avfoundation/MediaTimeAVFoundation.h>
 #include <pal/spi/cf/CoreAudioSPI.h>
 #include <sys/time.h>
+#include <wtf/Algorithms.h>
 #include <wtf/MainThread.h>
 #include <wtf/NeverDestroyed.h>
 #include <pal/cf/CoreMediaSoftLink.h>
@@ -102,7 +103,9 @@ public:
 
     bool hasAudioUnit() const { return m_ioUnit; }
 
-    void setCaptureDeviceID(uint32_t);
+    void setCaptureDevice(String&&, uint32_t);
+
+    void devicesChanged(const Vector<CaptureDevice>&);
 
 private:
     OSStatus configureSpeakerProc();
@@ -116,10 +119,12 @@ private:
     static OSStatus speakerCallback(void*, AudioUnitRenderActionFlags*, const AudioTimeStamp*, UInt32, UInt32, AudioBufferList*);
     OSStatus provideSpeakerData(AudioUnitRenderActionFlags&, const AudioTimeStamp&, UInt32, UInt32, AudioBufferList*);
 
-    void startInternal();
+    OSStatus startInternal();
     void stopInternal();
 
     void verifyIsCapturing();
+    void devicesChanged();
+    void captureFailed();
 
     Vector<std::reference_wrapper<CoreAudioCaptureSource>> m_clients;
 
@@ -162,6 +167,8 @@ private:
     uint64_t m_speakerProcsCalled { 0 };
 #endif
 
+    String m_persistentID;
+
     uint64_t m_microphoneProcsCalled { 0 };
     uint64_t m_microphoneProcsCalledLastTime { 0 };
     Timer m_verifyCapturingTimer;
@@ -197,8 +204,10 @@ void CoreAudioSharedUnit::removeClient(CoreAudioCaptureSource& client)
     });
 }
 
-void CoreAudioSharedUnit::setCaptureDeviceID(uint32_t captureDeviceID)
+void CoreAudioSharedUnit::setCaptureDevice(String&& persistentID, uint32_t captureDeviceID)
 {
+    m_persistentID = WTFMove(persistentID);
+
 #if PLATFORM(MAC)
     if (m_captureDeviceID == captureDeviceID)
         return;
@@ -210,6 +219,17 @@ void CoreAudioSharedUnit::setCaptureDeviceID(uint32_t captureDeviceID)
 #endif
 }
 
+void CoreAudioSharedUnit::devicesChanged(const Vector<CaptureDevice>& devices)
+{
+    if (!m_ioUnit)
+        return;
+
+    if (WTF::anyOf(devices, [this] (auto& device) { return m_persistentID == device.persistentId(); }))
+        return;
+
+    captureFailed();
+}
+
 void CoreAudioSharedUnit::addEchoCancellationSource(AudioSampleDataSource& source)
 {
     if (!source.setOutputFormat(m_speakerProcFormat)) {
@@ -559,7 +579,8 @@ void CoreAudioSharedUnit::startProducingData()
         ASSERT(!m_ioUnit);
     }
 
-    startInternal();
+    if (startInternal())
+        captureFailed();
 }
 
 OSStatus CoreAudioSharedUnit::resume()
@@ -578,7 +599,7 @@ OSStatus CoreAudioSharedUnit::resume()
     return 0;
 }
 
-void CoreAudioSharedUnit::startInternal()
+OSStatus CoreAudioSharedUnit::startInternal()
 {
     OSStatus err;
     if (!m_ioUnit) {
@@ -586,7 +607,7 @@ void CoreAudioSharedUnit::startInternal()
         if (err) {
             cleanupAudioUnit();
             ASSERT(!m_ioUnit);
-            return;
+            return err;
         }
         ASSERT(m_ioUnit);
     }
@@ -598,7 +619,7 @@ void CoreAudioSharedUnit::startInternal()
     err = AudioOutputUnitStart(m_ioUnit);
     if (err) {
         RELEASE_LOG_ERROR(Media, "CoreAudioSharedUnit::start(%p) AudioOutputUnitStart failed with error %d (%.4s)", this, (int)err, (char*)&err);
-        return;
+        return err;
     }
 
     m_ioUnitStarted = true;
@@ -606,6 +627,8 @@ void CoreAudioSharedUnit::startInternal()
     m_verifyCapturingTimer.startRepeating(10_s);
     m_microphoneProcsCalled = 0;
     m_microphoneProcsCalledLastTime = 0;
+
+    return 0;
 }
 
 void CoreAudioSharedUnit::verifyIsCapturing()
@@ -617,8 +640,14 @@ void CoreAudioSharedUnit::verifyIsCapturing()
         return;
     }
 
+    captureFailed();
+}
+
+
+void CoreAudioSharedUnit::captureFailed()
+{
 #if !RELEASE_LOG_DISABLED
-    RELEASE_LOG_ERROR(Media, "CoreAudioSharedUnit::verifyIsCapturing - capture failed\n");
+    RELEASE_LOG_ERROR(Media, "CoreAudioSharedUnit::captureFailed - capture failed\n");
 #endif
     for (CoreAudioCaptureSource& client : m_clients)
         client.captureFailed();
@@ -783,12 +812,17 @@ CaptureDeviceManager& CoreAudioCaptureSourceFactory::audioCaptureDeviceManager()
 #endif
 }
 
-CoreAudioCaptureSource::CoreAudioCaptureSource(String&& deviceID, String&& label, String&& hashSalt, uint32_t persistentID)
+void CoreAudioCaptureSourceFactory::devicesChanged(const Vector<CaptureDevice>& devices)
+{
+    CoreAudioSharedUnit::singleton().devicesChanged(devices);
+}
+
+CoreAudioCaptureSource::CoreAudioCaptureSource(String&& deviceID, String&& label, String&& hashSalt, uint32_t captureDeviceID)
     : RealtimeMediaSource(RealtimeMediaSource::Type::Audio, WTFMove(label), WTFMove(deviceID), WTFMove(hashSalt))
-    , m_captureDeviceID(persistentID)
+    , m_captureDeviceID(captureDeviceID)
 {
     auto& unit = CoreAudioSharedUnit::singleton();
-    unit.setCaptureDeviceID(m_captureDeviceID);
+    unit.setCaptureDevice(String { persistentID() }, m_captureDeviceID);
 
     initializeEchoCancellation(unit.enableEchoCancellation());
     initializeSampleRate(unit.sampleRate());
index d9fa0ae..f520373 100644 (file)
@@ -117,6 +117,8 @@ public:
     void endInterruption();
     void scheduleReconfiguration();
 
+    void devicesChanged(const Vector<CaptureDevice>&);
+
 #if PLATFORM(IOS_FAMILY)
     void setCoreAudioActiveSource(CoreAudioCaptureSource& source) { setActiveSource(source); }
     void unsetCoreAudioActiveSource(CoreAudioCaptureSource& source) { unsetActiveSource(source); }