Stop using dispatch_set_target_queue()
[WebKit-https.git] / Source / WebCore / platform / mediastream / mac / AVMediaCaptureSource.mm
1 /*
2  * Copyright (C) 2013-2015 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
17  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24  */
25
26 #import "config.h"
27 #import "AVMediaCaptureSource.h"
28
29 #if ENABLE(MEDIA_STREAM) && USE(AVFOUNDATION)
30
31 #import "AVCaptureDeviceManager.h"
32 #import "AudioSourceProvider.h"
33 #import "Logging.h"
34 #import "MediaConstraints.h"
35 #import "RealtimeMediaSourceSettings.h"
36 #import <AVFoundation/AVCaptureDevice.h>
37 #import <AVFoundation/AVCaptureInput.h>
38 #import <AVFoundation/AVCaptureOutput.h>
39 #import <AVFoundation/AVCaptureSession.h>
40 #import <AVFoundation/AVError.h>
41 #import <objc/runtime.h>
42 #import <wtf/MainThread.h>
43
44 #import <pal/cf/CoreMediaSoftLink.h>
45
46 typedef AVCaptureConnection AVCaptureConnectionType;
47 typedef AVCaptureDevice AVCaptureDeviceTypedef;
48 typedef AVCaptureDeviceInput AVCaptureDeviceInputType;
49 typedef AVCaptureOutput AVCaptureOutputType;
50 typedef AVCaptureSession AVCaptureSessionType;
51 typedef AVCaptureAudioDataOutput AVCaptureAudioDataOutputType;
52 typedef AVCaptureVideoDataOutput AVCaptureVideoDataOutputType;
53
54 SOFT_LINK_FRAMEWORK_OPTIONAL(AVFoundation)
55
56 SOFT_LINK_CLASS(AVFoundation, AVCaptureAudioDataOutput)
57 SOFT_LINK_CLASS(AVFoundation, AVCaptureConnection)
58 SOFT_LINK_CLASS(AVFoundation, AVCaptureDevice)
59 SOFT_LINK_CLASS(AVFoundation, AVCaptureDeviceInput)
60 SOFT_LINK_CLASS(AVFoundation, AVCaptureOutput)
61 SOFT_LINK_CLASS(AVFoundation, AVCaptureSession)
62 SOFT_LINK_CLASS(AVFoundation, AVCaptureVideoDataOutput)
63
64 #define AVCaptureAudioDataOutput getAVCaptureAudioDataOutputClass()
65 #define AVCaptureConnection getAVCaptureConnectionClass()
66 #define AVCaptureDevice getAVCaptureDeviceClass()
67 #define AVCaptureDeviceInput getAVCaptureDeviceInputClass()
68 #define AVCaptureOutput getAVCaptureOutputClass()
69 #define AVCaptureSession getAVCaptureSessionClass()
70 #define AVCaptureVideoDataOutput getAVCaptureVideoDataOutputClass()
71
72 SOFT_LINK_POINTER(AVFoundation, AVMediaTypeAudio, NSString *)
73 SOFT_LINK_POINTER(AVFoundation, AVMediaTypeMuxed, NSString *)
74 SOFT_LINK_POINTER(AVFoundation, AVMediaTypeVideo, NSString *)
75
76 #define AVMediaTypeAudio getAVMediaTypeAudio()
77 #define AVMediaTypeMuxed getAVMediaTypeMuxed()
78 #define AVMediaTypeVideo getAVMediaTypeVideo()
79
80 #if PLATFORM(IOS)
81 SOFT_LINK_POINTER_OPTIONAL(AVFoundation, AVCaptureSessionRuntimeErrorNotification, NSString *)
82 SOFT_LINK_POINTER_OPTIONAL(AVFoundation, AVCaptureSessionWasInterruptedNotification, NSString *)
83 SOFT_LINK_POINTER_OPTIONAL(AVFoundation, AVCaptureSessionInterruptionEndedNotification, NSString *)
84 SOFT_LINK_POINTER_OPTIONAL(AVFoundation, AVCaptureSessionInterruptionReasonKey, NSString *)
85 SOFT_LINK_POINTER_OPTIONAL(AVFoundation, AVCaptureSessionErrorKey, NSString *)
86
87 #define AVCaptureSessionRuntimeErrorNotification getAVCaptureSessionRuntimeErrorNotification()
88 #define AVCaptureSessionWasInterruptedNotification getAVCaptureSessionWasInterruptedNotification()
89 #define AVCaptureSessionInterruptionEndedNotification getAVCaptureSessionInterruptionEndedNotification()
90 #define AVCaptureSessionInterruptionReasonKey getAVCaptureSessionInterruptionReasonKey()
91 #define AVCaptureSessionErrorKey getAVCaptureSessionErrorKey()
92 #endif
93
94 using namespace WebCore;
95
96 @interface WebCoreAVMediaCaptureSourceObserver : NSObject<AVCaptureAudioDataOutputSampleBufferDelegate, AVCaptureVideoDataOutputSampleBufferDelegate>
97 {
98     AVMediaCaptureSource* m_callback;
99 }
100
101 -(id)initWithCallback:(AVMediaCaptureSource*)callback;
102 -(void)disconnect;
103 -(void)addNotificationObservers;
104 -(void)removeNotificationObservers;
105 -(void)captureOutput:(AVCaptureOutputType*)captureOutput didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer fromConnection:(AVCaptureConnectionType*)connection;
106 -(void)observeValueForKeyPath:keyPath ofObject:(id)object change:(NSDictionary*)change context:(void*)context;
107 #if PLATFORM(IOS)
108 -(void)sessionRuntimeError:(NSNotification*)notification;
109 -(void)beginSessionInterrupted:(NSNotification*)notification;
110 -(void)endSessionInterrupted:(NSNotification*)notification;
111 #endif
112 @end
113
114 namespace WebCore {
115
116 static NSArray<NSString*>* sessionKVOProperties();
117
118 static dispatch_queue_t globaAudioCaptureSerialQueue()
119 {
120     static dispatch_queue_t globalQueue;
121     static dispatch_once_t onceToken;
122     dispatch_once(&onceToken, ^{
123         globalQueue = dispatch_queue_create("WebCoreAVMediaCaptureSource audio capture queue", DISPATCH_QUEUE_SERIAL);
124     });
125     return globalQueue;
126 }
127
128 static dispatch_queue_t globaVideoCaptureSerialQueue()
129 {
130     static dispatch_queue_t globalQueue;
131     static dispatch_once_t onceToken;
132     dispatch_once(&onceToken, ^{
133         globalQueue = dispatch_queue_create_with_target("WebCoreAVMediaCaptureSource video capture queue", DISPATCH_QUEUE_SERIAL, dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_HIGH, 0));
134     });
135     return globalQueue;
136 }
137
138 AVMediaCaptureSource::AVMediaCaptureSource(AVCaptureDeviceTypedef* device, const AtomicString& id, RealtimeMediaSource::Type type)
139     : RealtimeMediaSource(id, type, device.localizedName)
140     , m_objcObserver(adoptNS([[WebCoreAVMediaCaptureSourceObserver alloc] initWithCallback:this]))
141     , m_device(device)
142 {
143 #if PLATFORM(IOS)
144     static_assert(static_cast<int>(InterruptionReason::VideoNotAllowedInBackground) == static_cast<int>(AVCaptureSessionInterruptionReasonVideoDeviceNotAvailableInBackground), "InterruptionReason::VideoNotAllowedInBackground is not AVCaptureSessionInterruptionReasonVideoDeviceNotAvailableInBackground as expected");
145     static_assert(static_cast<int>(InterruptionReason::VideoNotAllowedInSideBySide) == AVCaptureSessionInterruptionReasonVideoDeviceNotAvailableWithMultipleForegroundApps, "InterruptionReason::VideoNotAllowedInSideBySide is not AVCaptureSessionInterruptionReasonVideoDeviceNotAvailableWithMultipleForegroundApps as expected");
146     static_assert(static_cast<int>(InterruptionReason::VideoInUse) == AVCaptureSessionInterruptionReasonVideoDeviceInUseByAnotherClient, "InterruptionReason::VideoInUse is not AVCaptureSessionInterruptionReasonVideoDeviceInUseByAnotherClient as expected");
147     static_assert(static_cast<int>(InterruptionReason::AudioInUse) == AVCaptureSessionInterruptionReasonAudioDeviceInUseByAnotherClient, "InterruptionReason::AudioInUse is not AVCaptureSessionInterruptionReasonAudioDeviceInUseByAnotherClient as expected");
148 #endif
149     
150     setPersistentID(String(device.uniqueID));
151 }
152
153 AVMediaCaptureSource::~AVMediaCaptureSource()
154 {
155     [m_objcObserver disconnect];
156
157     if (!m_session)
158         return;
159
160     for (NSString *keyName in sessionKVOProperties())
161         [m_session removeObserver:m_objcObserver.get() forKeyPath:keyName];
162
163     if ([m_session isRunning])
164         [m_session stopRunning];
165
166 }
167
168 void AVMediaCaptureSource::startProducingData()
169 {
170     if (!m_session) {
171         if (!setupSession())
172             return;
173     }
174
175     if ([m_session isRunning])
176         return;
177
178     [m_objcObserver addNotificationObservers];
179     [m_session startRunning];
180 }
181
182 void AVMediaCaptureSource::stopProducingData()
183 {
184     if (!m_session)
185         return;
186
187     [m_objcObserver removeNotificationObservers];
188
189     if ([m_session isRunning])
190         [m_session stopRunning];
191
192     m_interruption = InterruptionReason::None;
193 #if PLATFORM(IOS)
194     m_session = nullptr;
195 #endif
196 }
197
198 void AVMediaCaptureSource::beginConfiguration()
199 {
200     if (m_session)
201         [m_session beginConfiguration];
202 }
203
204 void AVMediaCaptureSource::commitConfiguration()
205 {
206     if (m_session)
207         [m_session commitConfiguration];
208 }
209
210 void AVMediaCaptureSource::initializeSettings()
211 {
212     if (m_currentSettings.deviceId().isEmpty())
213         m_currentSettings.setSupportedConstraints(supportedConstraints());
214
215     m_currentSettings.setDeviceId(id());
216     m_currentSettings.setLabel(name());
217     updateSettings(m_currentSettings);
218 }
219
220 const RealtimeMediaSourceSettings& AVMediaCaptureSource::settings() const
221 {
222     const_cast<AVMediaCaptureSource&>(*this).initializeSettings();
223     return m_currentSettings;
224 }
225
226 RealtimeMediaSourceSupportedConstraints& AVMediaCaptureSource::supportedConstraints()
227 {
228     if (m_supportedConstraints.supportsDeviceId())
229         return m_supportedConstraints;
230
231     m_supportedConstraints.setSupportsDeviceId(true);
232     initializeSupportedConstraints(m_supportedConstraints);
233
234     return m_supportedConstraints;
235 }
236
237 void AVMediaCaptureSource::initializeCapabilities()
238 {
239     m_capabilities = std::make_unique<RealtimeMediaSourceCapabilities>(supportedConstraints());
240     m_capabilities->setDeviceId(id());
241
242     initializeCapabilities(*m_capabilities.get());
243 }
244
245 const RealtimeMediaSourceCapabilities& AVMediaCaptureSource::capabilities() const
246 {
247     if (!m_capabilities)
248         const_cast<AVMediaCaptureSource&>(*this).initializeCapabilities();
249     return *m_capabilities;
250 }
251
252 bool AVMediaCaptureSource::setupSession()
253 {
254     if (m_session)
255         return true;
256
257     m_session = adoptNS([allocAVCaptureSessionInstance() init]);
258     for (NSString* keyName in sessionKVOProperties())
259         [m_session addObserver:m_objcObserver.get() forKeyPath:keyName options:NSKeyValueObservingOptionNew context:(void *)nil];
260
261     [m_session beginConfiguration];
262     bool success = setupCaptureSession();
263     [m_session commitConfiguration];
264
265     if (!success)
266         captureFailed();
267
268     return success;
269 }
270
271 void AVMediaCaptureSource::captureSessionIsRunningDidChange(bool state)
272 {
273     scheduleDeferredTask([this, state] {
274         if ((state == m_isRunning) && (state == !muted()))
275             return;
276
277         m_isRunning = state;
278         notifyMutedChange(!m_isRunning);
279     });
280 }
281
282 #if PLATFORM(IOS)
283 void AVMediaCaptureSource::captureSessionRuntimeError(RetainPtr<NSError> error)
284 {
285     if (!m_isRunning || error.get().code != AVErrorMediaServicesWereReset)
286         return;
287
288     // Try to restart the session, but reset m_isRunning immediately so if it fails we won't try again.
289     [m_session startRunning];
290     m_isRunning = [m_session isRunning];
291 }
292
293 void AVMediaCaptureSource::captureSessionBeginInterruption(RetainPtr<NSNotification> notification)
294 {
295     m_interruption = static_cast<AVMediaCaptureSource::InterruptionReason>([notification.get().userInfo[AVCaptureSessionInterruptionReasonKey] integerValue]);
296 }
297
298 void AVMediaCaptureSource::captureSessionEndInterruption(RetainPtr<NSNotification>)
299 {
300     InterruptionReason reason = m_interruption;
301
302     m_interruption = InterruptionReason::None;
303     if (reason != InterruptionReason::VideoNotAllowedInSideBySide || m_isRunning || !m_session)
304         return;
305
306     [m_session startRunning];
307     m_isRunning = [m_session isRunning];
308 }
309 #endif
310
311 void AVMediaCaptureSource::setVideoSampleBufferDelegate(AVCaptureVideoDataOutputType* videoOutput)
312 {
313     [videoOutput setSampleBufferDelegate:m_objcObserver.get() queue:globaVideoCaptureSerialQueue()];
314 }
315
316 void AVMediaCaptureSource::setAudioSampleBufferDelegate(AVCaptureAudioDataOutputType* audioOutput)
317 {
318     [audioOutput setSampleBufferDelegate:m_objcObserver.get() queue:globaAudioCaptureSerialQueue()];
319 }
320
321 bool AVMediaCaptureSource::interrupted() const
322 {
323     if (m_interruption != InterruptionReason::None)
324         return true;
325
326     return RealtimeMediaSource::interrupted();
327 }
328
329 NSArray<NSString*>* sessionKVOProperties()
330 {
331     static NSArray* keys = [@[@"running"] retain];
332     return keys;
333 }
334
335 } // namespace WebCore
336
337 @implementation WebCoreAVMediaCaptureSourceObserver
338
339 - (id)initWithCallback:(AVMediaCaptureSource*)callback
340 {
341     self = [super init];
342     if (!self)
343         return nil;
344
345     m_callback = callback;
346
347     return self;
348 }
349
350 - (void)disconnect
351 {
352     [NSObject cancelPreviousPerformRequestsWithTarget:self];
353     [self removeNotificationObservers];
354     m_callback = nullptr;
355 }
356
357 - (void)addNotificationObservers
358 {
359 #if PLATFORM(IOS)
360     ASSERT(m_callback);
361
362     NSNotificationCenter* center = [NSNotificationCenter defaultCenter];
363     AVCaptureSessionType* session = m_callback->session();
364
365     [center addObserver:self selector:@selector(sessionRuntimeError:) name:AVCaptureSessionRuntimeErrorNotification object:session];
366     [center addObserver:self selector:@selector(beginSessionInterrupted:) name:AVCaptureSessionWasInterruptedNotification object:session];
367     [center addObserver:self selector:@selector(endSessionInterrupted:) name:AVCaptureSessionInterruptionEndedNotification object:session];
368 #endif
369 }
370
371 - (void)removeNotificationObservers
372 {
373 #if PLATFORM(IOS)
374     [[NSNotificationCenter defaultCenter] removeObserver:self];
375 #endif
376 }
377
378 - (void)captureOutput:(AVCaptureOutputType*)captureOutput didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer fromConnection:(AVCaptureConnectionType*)connection
379 {
380     if (!m_callback)
381         return;
382
383     m_callback->captureOutputDidOutputSampleBufferFromConnection(captureOutput, sampleBuffer, connection);
384 }
385
386 - (void)observeValueForKeyPath:keyPath ofObject:(id)object change:(NSDictionary*)change context:(void*)context
387 {
388     UNUSED_PARAM(object);
389     UNUSED_PARAM(context);
390
391     if (!m_callback)
392         return;
393
394     id newValue = [change valueForKey:NSKeyValueChangeNewKey];
395
396 #if !LOG_DISABLED
397     bool willChange = [[change valueForKey:NSKeyValueChangeNotificationIsPriorKey] boolValue];
398
399     if (willChange)
400         LOG(Media, "WebCoreAVMediaCaptureSourceObserver::observeValueForKeyPath(%p) - will change, keyPath = %s", self, [keyPath UTF8String]);
401     else {
402         RetainPtr<NSString> valueString = adoptNS([[NSString alloc] initWithFormat:@"%@", newValue]);
403         LOG(Media, "WebCoreAVMediaCaptureSourceObserver::observeValueForKeyPath(%p) - did change, keyPath = %s, value = %s", self, [keyPath UTF8String], [valueString.get() UTF8String]);
404     }
405 #endif
406
407     if ([keyPath isEqualToString:@"running"])
408         m_callback->captureSessionIsRunningDidChange([newValue boolValue]);
409 }
410
411 #if PLATFORM(IOS)
412 - (void)sessionRuntimeError:(NSNotification*)notification
413 {
414     NSError *error = notification.userInfo[AVCaptureSessionErrorKey];
415     LOG(Media, "WebCoreAVMediaCaptureSourceObserver::sessionRuntimeError(%p) - error = %s", self, [[error localizedDescription] UTF8String]);
416
417     if (m_callback)
418         m_callback->captureSessionRuntimeError(error);
419 }
420
421 -(void)beginSessionInterrupted:(NSNotification*)notification
422 {
423     LOG(Media, "WebCoreAVMediaCaptureSourceObserver::beginSessionInterrupted(%p) - reason = %d", self, [notification.userInfo[AVCaptureSessionInterruptionReasonKey] integerValue]);
424
425     if (m_callback)
426         m_callback->captureSessionBeginInterruption(notification);
427 }
428
429 - (void)endSessionInterrupted:(NSNotification*)notification
430 {
431     LOG(Media, "WebCoreAVMediaCaptureSourceObserver::endSessionInterrupted(%p)", self);
432
433     if (m_callback)
434         m_callback->captureSessionEndInterruption(notification);
435 }
436 #endif
437
438 @end
439
440 #endif // ENABLE(MEDIA_STREAM)