e61953f4c315706cbaee22f7703f6d4dcc0a8067
[WebKit-https.git] / Source / WebCore / platform / mediastream / mac / AVMediaCaptureSource.mm
1 /*
2  * Copyright (C) 2013-2015 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
17  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24  */
25
26 #import "config.h"
27 #import "AVMediaCaptureSource.h"
28
29 #if ENABLE(MEDIA_STREAM) && USE(AVFOUNDATION)
30
31 #import "AVCaptureDeviceManager.h"
32 #import "AudioSourceProvider.h"
33 #import "Logging.h"
34 #import "MediaConstraints.h"
35 #import "RealtimeMediaSourceSettings.h"
36 #import <AVFoundation/AVCaptureDevice.h>
37 #import <AVFoundation/AVCaptureInput.h>
38 #import <AVFoundation/AVCaptureOutput.h>
39 #import <AVFoundation/AVCaptureSession.h>
40 #import <AVFoundation/AVError.h>
41 #import <objc/runtime.h>
42 #import <wtf/MainThread.h>
43
44 #import <pal/cf/CoreMediaSoftLink.h>
45
46 typedef AVCaptureConnection AVCaptureConnectionType;
47 typedef AVCaptureDevice AVCaptureDeviceTypedef;
48 typedef AVCaptureDeviceInput AVCaptureDeviceInputType;
49 typedef AVCaptureOutput AVCaptureOutputType;
50 typedef AVCaptureSession AVCaptureSessionType;
51 typedef AVCaptureAudioDataOutput AVCaptureAudioDataOutputType;
52 typedef AVCaptureVideoDataOutput AVCaptureVideoDataOutputType;
53
54 SOFT_LINK_FRAMEWORK_OPTIONAL(AVFoundation)
55
56 SOFT_LINK_CLASS(AVFoundation, AVCaptureAudioDataOutput)
57 SOFT_LINK_CLASS(AVFoundation, AVCaptureConnection)
58 SOFT_LINK_CLASS(AVFoundation, AVCaptureDevice)
59 SOFT_LINK_CLASS(AVFoundation, AVCaptureDeviceInput)
60 SOFT_LINK_CLASS(AVFoundation, AVCaptureOutput)
61 SOFT_LINK_CLASS(AVFoundation, AVCaptureSession)
62 SOFT_LINK_CLASS(AVFoundation, AVCaptureVideoDataOutput)
63
64 #define AVCaptureAudioDataOutput getAVCaptureAudioDataOutputClass()
65 #define AVCaptureConnection getAVCaptureConnectionClass()
66 #define AVCaptureDevice getAVCaptureDeviceClass()
67 #define AVCaptureDeviceInput getAVCaptureDeviceInputClass()
68 #define AVCaptureOutput getAVCaptureOutputClass()
69 #define AVCaptureSession getAVCaptureSessionClass()
70 #define AVCaptureVideoDataOutput getAVCaptureVideoDataOutputClass()
71
72 SOFT_LINK_POINTER(AVFoundation, AVMediaTypeAudio, NSString *)
73 SOFT_LINK_POINTER(AVFoundation, AVMediaTypeMuxed, NSString *)
74 SOFT_LINK_POINTER(AVFoundation, AVMediaTypeVideo, NSString *)
75
76 #define AVMediaTypeAudio getAVMediaTypeAudio()
77 #define AVMediaTypeMuxed getAVMediaTypeMuxed()
78 #define AVMediaTypeVideo getAVMediaTypeVideo()
79
80 #if PLATFORM(IOS)
81 SOFT_LINK_POINTER_OPTIONAL(AVFoundation, AVCaptureSessionRuntimeErrorNotification, NSString *)
82 SOFT_LINK_POINTER_OPTIONAL(AVFoundation, AVCaptureSessionWasInterruptedNotification, NSString *)
83 SOFT_LINK_POINTER_OPTIONAL(AVFoundation, AVCaptureSessionInterruptionEndedNotification, NSString *)
84 SOFT_LINK_POINTER_OPTIONAL(AVFoundation, AVCaptureSessionInterruptionReasonKey, NSString *)
85 SOFT_LINK_POINTER_OPTIONAL(AVFoundation, AVCaptureSessionErrorKey, NSString *)
86
87 #define AVCaptureSessionRuntimeErrorNotification getAVCaptureSessionRuntimeErrorNotification()
88 #define AVCaptureSessionWasInterruptedNotification getAVCaptureSessionWasInterruptedNotification()
89 #define AVCaptureSessionInterruptionEndedNotification getAVCaptureSessionInterruptionEndedNotification()
90 #define AVCaptureSessionInterruptionReasonKey getAVCaptureSessionInterruptionReasonKey()
91 #define AVCaptureSessionErrorKey getAVCaptureSessionErrorKey()
92 #endif
93
94 using namespace WebCore;
95
96 @interface WebCoreAVMediaCaptureSourceObserver : NSObject<AVCaptureAudioDataOutputSampleBufferDelegate, AVCaptureVideoDataOutputSampleBufferDelegate>
97 {
98     AVMediaCaptureSource* m_callback;
99 }
100
101 -(id)initWithCallback:(AVMediaCaptureSource*)callback;
102 -(void)disconnect;
103 -(void)addNotificationObservers;
104 -(void)removeNotificationObservers;
105 -(void)captureOutput:(AVCaptureOutputType*)captureOutput didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer fromConnection:(AVCaptureConnectionType*)connection;
106 -(void)observeValueForKeyPath:keyPath ofObject:(id)object change:(NSDictionary*)change context:(void*)context;
107 #if PLATFORM(IOS)
108 -(void)sessionRuntimeError:(NSNotification*)notification;
109 -(void)beginSessionInterrupted:(NSNotification*)notification;
110 -(void)endSessionInterrupted:(NSNotification*)notification;
111 #endif
112 @end
113
114 namespace WebCore {
115
116 static NSArray<NSString*>* sessionKVOProperties();
117
118 static dispatch_queue_t globaAudioCaptureSerialQueue()
119 {
120     static dispatch_queue_t globalQueue;
121     static dispatch_once_t onceToken;
122     dispatch_once(&onceToken, ^{
123         globalQueue = dispatch_queue_create("WebCoreAVMediaCaptureSource audio capture queue", DISPATCH_QUEUE_SERIAL);
124     });
125     return globalQueue;
126 }
127
128 static dispatch_queue_t globaVideoCaptureSerialQueue()
129 {
130     static dispatch_queue_t globalQueue;
131     static dispatch_once_t onceToken;
132     dispatch_once(&onceToken, ^{
133         globalQueue = dispatch_queue_create("WebCoreAVMediaCaptureSource video capture queue", DISPATCH_QUEUE_SERIAL);
134         dispatch_set_target_queue(globalQueue, dispatch_get_global_queue( DISPATCH_QUEUE_PRIORITY_HIGH, 0));
135     });
136     return globalQueue;
137 }
138
139 AVMediaCaptureSource::AVMediaCaptureSource(AVCaptureDeviceTypedef* device, const AtomicString& id, RealtimeMediaSource::Type type)
140     : RealtimeMediaSource(id, type, device.localizedName)
141     , m_objcObserver(adoptNS([[WebCoreAVMediaCaptureSourceObserver alloc] initWithCallback:this]))
142     , m_device(device)
143 {
144 #if PLATFORM(IOS)
145     static_assert(static_cast<int>(InterruptionReason::VideoNotAllowedInBackground) == static_cast<int>(AVCaptureSessionInterruptionReasonVideoDeviceNotAvailableInBackground), "InterruptionReason::VideoNotAllowedInBackground is not AVCaptureSessionInterruptionReasonVideoDeviceNotAvailableInBackground as expected");
146     static_assert(static_cast<int>(InterruptionReason::VideoNotAllowedInSideBySide) == AVCaptureSessionInterruptionReasonVideoDeviceNotAvailableWithMultipleForegroundApps, "InterruptionReason::VideoNotAllowedInSideBySide is not AVCaptureSessionInterruptionReasonVideoDeviceNotAvailableWithMultipleForegroundApps as expected");
147     static_assert(static_cast<int>(InterruptionReason::VideoInUse) == AVCaptureSessionInterruptionReasonVideoDeviceInUseByAnotherClient, "InterruptionReason::VideoInUse is not AVCaptureSessionInterruptionReasonVideoDeviceInUseByAnotherClient as expected");
148     static_assert(static_cast<int>(InterruptionReason::AudioInUse) == AVCaptureSessionInterruptionReasonAudioDeviceInUseByAnotherClient, "InterruptionReason::AudioInUse is not AVCaptureSessionInterruptionReasonAudioDeviceInUseByAnotherClient as expected");
149 #endif
150     
151     setPersistentID(String(device.uniqueID));
152 }
153
154 AVMediaCaptureSource::~AVMediaCaptureSource()
155 {
156     [m_objcObserver disconnect];
157
158     if (!m_session)
159         return;
160
161     for (NSString *keyName in sessionKVOProperties())
162         [m_session removeObserver:m_objcObserver.get() forKeyPath:keyName];
163
164     if ([m_session isRunning])
165         [m_session stopRunning];
166
167 }
168
169 void AVMediaCaptureSource::startProducingData()
170 {
171     if (!m_session) {
172         if (!setupSession())
173             return;
174     }
175
176     if ([m_session isRunning])
177         return;
178
179     [m_objcObserver addNotificationObservers];
180     [m_session startRunning];
181 }
182
183 void AVMediaCaptureSource::stopProducingData()
184 {
185     if (!m_session)
186         return;
187
188     [m_objcObserver removeNotificationObservers];
189
190     if ([m_session isRunning])
191         [m_session stopRunning];
192
193     m_interruption = InterruptionReason::None;
194 #if PLATFORM(IOS)
195     m_session = nullptr;
196 #endif
197 }
198
199 void AVMediaCaptureSource::beginConfiguration()
200 {
201     if (m_session)
202         [m_session beginConfiguration];
203 }
204
205 void AVMediaCaptureSource::commitConfiguration()
206 {
207     if (m_session)
208         [m_session commitConfiguration];
209 }
210
211 void AVMediaCaptureSource::initializeSettings()
212 {
213     if (m_currentSettings.deviceId().isEmpty())
214         m_currentSettings.setSupportedConstraints(supportedConstraints());
215
216     m_currentSettings.setDeviceId(id());
217     m_currentSettings.setLabel(name());
218     updateSettings(m_currentSettings);
219 }
220
221 const RealtimeMediaSourceSettings& AVMediaCaptureSource::settings() const
222 {
223     const_cast<AVMediaCaptureSource&>(*this).initializeSettings();
224     return m_currentSettings;
225 }
226
227 RealtimeMediaSourceSupportedConstraints& AVMediaCaptureSource::supportedConstraints()
228 {
229     if (m_supportedConstraints.supportsDeviceId())
230         return m_supportedConstraints;
231
232     m_supportedConstraints.setSupportsDeviceId(true);
233     initializeSupportedConstraints(m_supportedConstraints);
234
235     return m_supportedConstraints;
236 }
237
238 void AVMediaCaptureSource::initializeCapabilities()
239 {
240     m_capabilities = std::make_unique<RealtimeMediaSourceCapabilities>(supportedConstraints());
241     m_capabilities->setDeviceId(id());
242
243     initializeCapabilities(*m_capabilities.get());
244 }
245
246 const RealtimeMediaSourceCapabilities& AVMediaCaptureSource::capabilities() const
247 {
248     if (!m_capabilities)
249         const_cast<AVMediaCaptureSource&>(*this).initializeCapabilities();
250     return *m_capabilities;
251 }
252
253 bool AVMediaCaptureSource::setupSession()
254 {
255     if (m_session)
256         return true;
257
258     m_session = adoptNS([allocAVCaptureSessionInstance() init]);
259     for (NSString* keyName in sessionKVOProperties())
260         [m_session addObserver:m_objcObserver.get() forKeyPath:keyName options:NSKeyValueObservingOptionNew context:(void *)nil];
261
262     [m_session beginConfiguration];
263     bool success = setupCaptureSession();
264     [m_session commitConfiguration];
265
266     if (!success)
267         captureFailed();
268
269     return success;
270 }
271
272 void AVMediaCaptureSource::captureSessionIsRunningDidChange(bool state)
273 {
274     scheduleDeferredTask([this, state] {
275         if ((state == m_isRunning) && (state == !muted()))
276             return;
277
278         m_isRunning = state;
279         notifyMutedChange(!m_isRunning);
280     });
281 }
282
283 #if PLATFORM(IOS)
284 void AVMediaCaptureSource::captureSessionRuntimeError(RetainPtr<NSError> error)
285 {
286     if (!m_isRunning || error.get().code != AVErrorMediaServicesWereReset)
287         return;
288
289     // Try to restart the session, but reset m_isRunning immediately so if it fails we won't try again.
290     [m_session startRunning];
291     m_isRunning = [m_session isRunning];
292 }
293
294 void AVMediaCaptureSource::captureSessionBeginInterruption(RetainPtr<NSNotification> notification)
295 {
296     m_interruption = static_cast<AVMediaCaptureSource::InterruptionReason>([notification.get().userInfo[AVCaptureSessionInterruptionReasonKey] integerValue]);
297 }
298
299 void AVMediaCaptureSource::captureSessionEndInterruption(RetainPtr<NSNotification>)
300 {
301     InterruptionReason reason = m_interruption;
302
303     m_interruption = InterruptionReason::None;
304     if (reason != InterruptionReason::VideoNotAllowedInSideBySide || m_isRunning || !m_session)
305         return;
306
307     [m_session startRunning];
308     m_isRunning = [m_session isRunning];
309 }
310 #endif
311
312 void AVMediaCaptureSource::setVideoSampleBufferDelegate(AVCaptureVideoDataOutputType* videoOutput)
313 {
314     [videoOutput setSampleBufferDelegate:m_objcObserver.get() queue:globaVideoCaptureSerialQueue()];
315 }
316
317 void AVMediaCaptureSource::setAudioSampleBufferDelegate(AVCaptureAudioDataOutputType* audioOutput)
318 {
319     [audioOutput setSampleBufferDelegate:m_objcObserver.get() queue:globaAudioCaptureSerialQueue()];
320 }
321
322 bool AVMediaCaptureSource::interrupted() const
323 {
324     if (m_interruption != InterruptionReason::None)
325         return true;
326
327     return RealtimeMediaSource::interrupted();
328 }
329
330 NSArray<NSString*>* sessionKVOProperties()
331 {
332     static NSArray* keys = [@[@"running"] retain];
333     return keys;
334 }
335
336 } // namespace WebCore
337
338 @implementation WebCoreAVMediaCaptureSourceObserver
339
340 - (id)initWithCallback:(AVMediaCaptureSource*)callback
341 {
342     self = [super init];
343     if (!self)
344         return nil;
345
346     m_callback = callback;
347
348     return self;
349 }
350
351 - (void)disconnect
352 {
353     [NSObject cancelPreviousPerformRequestsWithTarget:self];
354     [self removeNotificationObservers];
355     m_callback = nullptr;
356 }
357
358 - (void)addNotificationObservers
359 {
360 #if PLATFORM(IOS)
361     ASSERT(m_callback);
362
363     NSNotificationCenter* center = [NSNotificationCenter defaultCenter];
364     AVCaptureSessionType* session = m_callback->session();
365
366     [center addObserver:self selector:@selector(sessionRuntimeError:) name:AVCaptureSessionRuntimeErrorNotification object:session];
367     [center addObserver:self selector:@selector(beginSessionInterrupted:) name:AVCaptureSessionWasInterruptedNotification object:session];
368     [center addObserver:self selector:@selector(endSessionInterrupted:) name:AVCaptureSessionInterruptionEndedNotification object:session];
369 #endif
370 }
371
372 - (void)removeNotificationObservers
373 {
374 #if PLATFORM(IOS)
375     [[NSNotificationCenter defaultCenter] removeObserver:self];
376 #endif
377 }
378
379 - (void)captureOutput:(AVCaptureOutputType*)captureOutput didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer fromConnection:(AVCaptureConnectionType*)connection
380 {
381     if (!m_callback)
382         return;
383
384     m_callback->captureOutputDidOutputSampleBufferFromConnection(captureOutput, sampleBuffer, connection);
385 }
386
387 - (void)observeValueForKeyPath:keyPath ofObject:(id)object change:(NSDictionary*)change context:(void*)context
388 {
389     UNUSED_PARAM(object);
390     UNUSED_PARAM(context);
391
392     if (!m_callback)
393         return;
394
395     id newValue = [change valueForKey:NSKeyValueChangeNewKey];
396
397 #if !LOG_DISABLED
398     bool willChange = [[change valueForKey:NSKeyValueChangeNotificationIsPriorKey] boolValue];
399
400     if (willChange)
401         LOG(Media, "WebCoreAVMediaCaptureSourceObserver::observeValueForKeyPath(%p) - will change, keyPath = %s", self, [keyPath UTF8String]);
402     else {
403         RetainPtr<NSString> valueString = adoptNS([[NSString alloc] initWithFormat:@"%@", newValue]);
404         LOG(Media, "WebCoreAVMediaCaptureSourceObserver::observeValueForKeyPath(%p) - did change, keyPath = %s, value = %s", self, [keyPath UTF8String], [valueString.get() UTF8String]);
405     }
406 #endif
407
408     if ([keyPath isEqualToString:@"running"])
409         m_callback->captureSessionIsRunningDidChange([newValue boolValue]);
410 }
411
412 #if PLATFORM(IOS)
413 - (void)sessionRuntimeError:(NSNotification*)notification
414 {
415     NSError *error = notification.userInfo[AVCaptureSessionErrorKey];
416     LOG(Media, "WebCoreAVMediaCaptureSourceObserver::sessionRuntimeError(%p) - error = %s", self, [[error localizedDescription] UTF8String]);
417
418     if (m_callback)
419         m_callback->captureSessionRuntimeError(error);
420 }
421
422 -(void)beginSessionInterrupted:(NSNotification*)notification
423 {
424     LOG(Media, "WebCoreAVMediaCaptureSourceObserver::beginSessionInterrupted(%p) - reason = %d", self, [notification.userInfo[AVCaptureSessionInterruptionReasonKey] integerValue]);
425
426     if (m_callback)
427         m_callback->captureSessionBeginInterruption(notification);
428 }
429
430 - (void)endSessionInterrupted:(NSNotification*)notification
431 {
432     LOG(Media, "WebCoreAVMediaCaptureSourceObserver::endSessionInterrupted(%p)", self);
433
434     if (m_callback)
435         m_callback->captureSessionEndInterruption(notification);
436 }
437 #endif
438
439 @end
440
441 #endif // ENABLE(MEDIA_STREAM)