[MediaStream] Simplify logic when changing RealtimeMediaSource settings
[WebKit-https.git] / Source / WebCore / platform / mediastream / mac / AVVideoCaptureSource.mm
1 /*
2  * Copyright (C) 2013-2017 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
17  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24  */
25
26 #import "config.h"
27 #import "AVVideoCaptureSource.h"
28
29 #if ENABLE(MEDIA_STREAM) && USE(AVFOUNDATION)
30
31 #import "ImageBuffer.h"
32 #import "IntRect.h"
33 #import "Logging.h"
34 #import "MediaConstraints.h"
35 #import "MediaSampleAVFObjC.h"
36 #import "PlatformLayer.h"
37 #import "RealtimeMediaSourceCenterMac.h"
38 #import "RealtimeMediaSourceSettings.h"
39 #import <AVFoundation/AVCaptureDevice.h>
40 #import <AVFoundation/AVCaptureInput.h>
41 #import <AVFoundation/AVCaptureOutput.h>
42 #import <AVFoundation/AVCaptureSession.h>
43 #import <AVFoundation/AVError.h>
44 #import <objc/runtime.h>
45
46 #import <pal/cf/CoreMediaSoftLink.h>
47 #import "CoreVideoSoftLink.h"
48
49 typedef AVCaptureConnection AVCaptureConnectionType;
50 typedef AVCaptureDevice AVCaptureDeviceTypedef;
51 typedef AVCaptureDeviceFormat AVCaptureDeviceFormatType;
52 typedef AVCaptureDeviceInput AVCaptureDeviceInputType;
53 typedef AVCaptureOutput AVCaptureOutputType;
54 typedef AVCaptureVideoDataOutput AVCaptureVideoDataOutputType;
55 typedef AVFrameRateRange AVFrameRateRangeType;
56 typedef AVCaptureSession AVCaptureSessionType;
57
58 SOFT_LINK_FRAMEWORK_OPTIONAL(AVFoundation)
59
60 SOFT_LINK_CLASS(AVFoundation, AVCaptureConnection)
61 SOFT_LINK_CLASS(AVFoundation, AVCaptureDevice)
62 SOFT_LINK_CLASS(AVFoundation, AVCaptureDeviceFormat)
63 SOFT_LINK_CLASS(AVFoundation, AVCaptureDeviceInput)
64 SOFT_LINK_CLASS(AVFoundation, AVCaptureOutput)
65 SOFT_LINK_CLASS(AVFoundation, AVCaptureVideoDataOutput)
66 SOFT_LINK_CLASS(AVFoundation, AVFrameRateRange)
67 SOFT_LINK_CLASS(AVFoundation, AVCaptureSession)
68
69 #define AVCaptureConnection getAVCaptureConnectionClass()
70 #define AVCaptureDevice getAVCaptureDeviceClass()
71 #define AVCaptureDeviceFormat getAVCaptureDeviceFormatClass()
72 #define AVCaptureDeviceInput getAVCaptureDeviceInputClass()
73 #define AVCaptureOutput getAVCaptureOutputClass()
74 #define AVCaptureVideoDataOutput getAVCaptureVideoDataOutputClass()
75 #define AVFrameRateRange getAVFrameRateRangeClass()
76
77 SOFT_LINK_CONSTANT(AVFoundation, AVMediaTypeVideo, NSString *)
78 SOFT_LINK_CONSTANT_MAY_FAIL(AVFoundation, AVCaptureSessionPreset1280x720, NSString *)
79 SOFT_LINK_CONSTANT_MAY_FAIL(AVFoundation, AVCaptureSessionPreset960x540, NSString *)
80 SOFT_LINK_CONSTANT_MAY_FAIL(AVFoundation, AVCaptureSessionPreset640x480, NSString *)
81 SOFT_LINK_CONSTANT_MAY_FAIL(AVFoundation, AVCaptureSessionPreset352x288, NSString *)
82 SOFT_LINK_CONSTANT_MAY_FAIL(AVFoundation, AVCaptureSessionPreset320x240, NSString *)
83
84 #define AVCaptureSessionPreset1280x720 getAVCaptureSessionPreset1280x720()
85 #define AVCaptureSessionPreset960x540 getAVCaptureSessionPreset960x540()
86 #define AVCaptureSessionPreset640x480 getAVCaptureSessionPreset640x480()
87 #define AVCaptureSessionPreset352x288 getAVCaptureSessionPreset352x288()
88 #define AVCaptureSessionPreset320x240 getAVCaptureSessionPreset320x240()
89
90 #if PLATFORM(IOS)
91 SOFT_LINK_CONSTANT_MAY_FAIL(AVFoundation, AVCaptureSessionPreset3840x2160, NSString *)
92 SOFT_LINK_CONSTANT_MAY_FAIL(AVFoundation, AVCaptureSessionPreset1920x1080, NSString *)
93 SOFT_LINK_POINTER_OPTIONAL(AVFoundation, AVCaptureSessionRuntimeErrorNotification, NSString *)
94 SOFT_LINK_POINTER_OPTIONAL(AVFoundation, AVCaptureSessionWasInterruptedNotification, NSString *)
95 SOFT_LINK_POINTER_OPTIONAL(AVFoundation, AVCaptureSessionInterruptionEndedNotification, NSString *)
96 SOFT_LINK_POINTER_OPTIONAL(AVFoundation, AVCaptureSessionInterruptionReasonKey, NSString *)
97 SOFT_LINK_POINTER_OPTIONAL(AVFoundation, AVCaptureSessionErrorKey, NSString *)
98
99 #define AVCaptureSessionPreset3840x2160 getAVCaptureSessionPreset3840x2160()
100 #define AVCaptureSessionPreset1920x1080 getAVCaptureSessionPreset1920x1080()
101 #define AVCaptureSessionRuntimeErrorNotification getAVCaptureSessionRuntimeErrorNotification()
102 #define AVCaptureSessionWasInterruptedNotification getAVCaptureSessionWasInterruptedNotification()
103 #define AVCaptureSessionInterruptionEndedNotification getAVCaptureSessionInterruptionEndedNotification()
104 #define AVCaptureSessionInterruptionReasonKey getAVCaptureSessionInterruptionReasonKey()
105 #define AVCaptureSessionErrorKey getAVCaptureSessionErrorKey()
106 #endif
107
108 using namespace WebCore;
109
110 @interface WebCoreAVVideoCaptureSourceObserver : NSObject<AVCaptureVideoDataOutputSampleBufferDelegate> {
111     AVVideoCaptureSource* m_callback;
112 }
113
114 -(id)initWithCallback:(AVVideoCaptureSource*)callback;
115 -(void)disconnect;
116 -(void)addNotificationObservers;
117 -(void)removeNotificationObservers;
118 -(void)captureOutput:(AVCaptureOutputType*)captureOutput didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer fromConnection:(AVCaptureConnectionType*)connection;
119 -(void)observeValueForKeyPath:keyPath ofObject:(id)object change:(NSDictionary*)change context:(void*)context;
120 #if PLATFORM(IOS)
121 -(void)sessionRuntimeError:(NSNotification*)notification;
122 -(void)beginSessionInterrupted:(NSNotification*)notification;
123 -(void)endSessionInterrupted:(NSNotification*)notification;
124 #endif
125 @end
126
127 namespace WebCore {
128
129 #if PLATFORM(MAC)
130 const OSType videoCaptureFormat = kCVPixelFormatType_420YpCbCr8Planar;
131 #else
132 const OSType videoCaptureFormat = kCVPixelFormatType_420YpCbCr8BiPlanarFullRange;
133 #endif
134
135 static dispatch_queue_t globaVideoCaptureSerialQueue()
136 {
137     static dispatch_queue_t globalQueue;
138     static dispatch_once_t onceToken;
139     dispatch_once(&onceToken, ^{
140         globalQueue = dispatch_queue_create_with_target("WebCoreAVVideoCaptureSource video capture queue", DISPATCH_QUEUE_SERIAL, dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_HIGH, 0));
141     });
142     return globalQueue;
143 }
144
145 CaptureSourceOrError AVVideoCaptureSource::create(const AtomicString& id, const MediaConstraints* constraints)
146 {
147     AVCaptureDeviceTypedef *device = [getAVCaptureDeviceClass() deviceWithUniqueID:id];
148     if (!device)
149         return { };
150
151     auto source = adoptRef(*new AVVideoCaptureSource(device, id));
152     if (constraints) {
153         auto result = source->applyConstraints(*constraints);
154         if (result)
155             return WTFMove(result.value().first);
156     }
157
158     return CaptureSourceOrError(WTFMove(source));
159 }
160
161 AVVideoCaptureSource::AVVideoCaptureSource(AVCaptureDeviceTypedef* device, const AtomicString& id)
162     : RealtimeMediaSource(id, Type::Video, device.localizedName)
163     , m_objcObserver(adoptNS([[WebCoreAVVideoCaptureSourceObserver alloc] initWithCallback:this]))
164     , m_device(device)
165 {
166     struct VideoPreset {
167         bool symbolAvailable;
168         NSString* name;
169         int width;
170         int height;
171     };
172
173     static const VideoPreset presets[] = {
174 #if PLATFORM(IOS)
175         { canLoadAVCaptureSessionPreset3840x2160(), AVCaptureSessionPreset3840x2160, 3840, 2160  },
176         { canLoadAVCaptureSessionPreset1920x1080(), AVCaptureSessionPreset1920x1080, 1920, 1080 },
177 #endif
178         { canLoadAVCaptureSessionPreset1280x720(), AVCaptureSessionPreset1280x720, 1280, 720 },
179         { canLoadAVCaptureSessionPreset960x540(), AVCaptureSessionPreset960x540, 960, 540 },
180         { canLoadAVCaptureSessionPreset640x480(), AVCaptureSessionPreset640x480, 640, 480 },
181         { canLoadAVCaptureSessionPreset352x288(), AVCaptureSessionPreset352x288, 352, 288 },
182         { canLoadAVCaptureSessionPreset320x240(), AVCaptureSessionPreset320x240, 320, 240 },
183     };
184
185     auto* presetsMap = &videoPresets();
186     for (auto& preset : presets) {
187         if (!preset.symbolAvailable || !preset.name || ![device supportsAVCaptureSessionPreset:preset.name])
188             continue;
189
190         presetsMap->add(String(preset.name), IntSize(preset.width, preset.height));
191     }
192
193 #if PLATFORM(IOS)
194     static_assert(static_cast<int>(InterruptionReason::VideoNotAllowedInBackground) == static_cast<int>(AVCaptureSessionInterruptionReasonVideoDeviceNotAvailableInBackground), "InterruptionReason::VideoNotAllowedInBackground is not AVCaptureSessionInterruptionReasonVideoDeviceNotAvailableInBackground as expected");
195     static_assert(static_cast<int>(InterruptionReason::VideoNotAllowedInSideBySide) == AVCaptureSessionInterruptionReasonVideoDeviceNotAvailableWithMultipleForegroundApps, "InterruptionReason::VideoNotAllowedInSideBySide is not AVCaptureSessionInterruptionReasonVideoDeviceNotAvailableWithMultipleForegroundApps as expected");
196     static_assert(static_cast<int>(InterruptionReason::VideoInUse) == AVCaptureSessionInterruptionReasonVideoDeviceInUseByAnotherClient, "InterruptionReason::VideoInUse is not AVCaptureSessionInterruptionReasonVideoDeviceInUseByAnotherClient as expected");
197     static_assert(static_cast<int>(InterruptionReason::AudioInUse) == AVCaptureSessionInterruptionReasonAudioDeviceInUseByAnotherClient, "InterruptionReason::AudioInUse is not AVCaptureSessionInterruptionReasonAudioDeviceInUseByAnotherClient as expected");
198 #endif
199
200     setPersistentID(String(device.uniqueID));
201 }
202
203 AVVideoCaptureSource::~AVVideoCaptureSource()
204 {
205 #if PLATFORM(IOS)
206     RealtimeMediaSourceCenterMac::videoCaptureSourceFactory().unsetActiveSource(*this);
207 #endif
208     [m_objcObserver disconnect];
209
210     if (!m_session)
211         return;
212
213     [m_session removeObserver:m_objcObserver.get() forKeyPath:@"rate"];
214     if ([m_session isRunning])
215         [m_session stopRunning];
216
217 }
218
219 void AVVideoCaptureSource::startProducingData()
220 {
221     if (!m_session) {
222         if (!setupSession())
223             return;
224     }
225
226     if ([m_session isRunning])
227         return;
228
229     [m_objcObserver addNotificationObservers];
230     [m_session startRunning];
231 }
232
233 void AVVideoCaptureSource::stopProducingData()
234 {
235     if (!m_session)
236         return;
237
238     [m_objcObserver removeNotificationObservers];
239
240     if ([m_session isRunning])
241         [m_session stopRunning];
242
243     m_interruption = InterruptionReason::None;
244 #if PLATFORM(IOS)
245     m_session = nullptr;
246 #endif
247 }
248
249 static void updateSizeMinMax(int& min, int& max, int value)
250 {
251     min = std::min<int>(min, value);
252     max = std::max<int>(max, value);
253 }
254
255 static void updateAspectRatioMinMax(double& min, double& max, double value)
256 {
257     min = std::min<double>(min, value);
258     max = std::max<double>(max, value);
259 }
260
261 void AVVideoCaptureSource::beginConfiguration()
262 {
263     if (m_session)
264         [m_session beginConfiguration];
265 }
266
267 void AVVideoCaptureSource::commitConfiguration()
268 {
269     if (m_session)
270         [m_session commitConfiguration];
271 }
272
273 void AVVideoCaptureSource::settingsDidChange(OptionSet<RealtimeMediaSourceSettings::Flag> settings)
274 {
275     m_currentSettings = std::nullopt;
276     RealtimeMediaSource::settingsDidChange(settings);
277 }
278
279 const RealtimeMediaSourceSettings& AVVideoCaptureSource::settings() const
280 {
281     if (m_currentSettings)
282         return *m_currentSettings;
283
284     RealtimeMediaSourceSettings settings;
285     if ([device() position] == AVCaptureDevicePositionFront)
286         settings.setFacingMode(RealtimeMediaSourceSettings::User);
287     else if ([device() position] == AVCaptureDevicePositionBack)
288         settings.setFacingMode(RealtimeMediaSourceSettings::Environment);
289     else
290         settings.setFacingMode(RealtimeMediaSourceSettings::Unknown);
291
292     auto maxFrameDuration = [device() activeVideoMaxFrameDuration];
293     settings.setFrameRate(maxFrameDuration.timescale / maxFrameDuration.value);
294     settings.setWidth(m_width);
295     settings.setHeight(m_height);
296     settings.setDeviceId(id());
297
298     RealtimeMediaSourceSupportedConstraints supportedConstraints;
299     supportedConstraints.setSupportsDeviceId(true);
300     supportedConstraints.setSupportsFacingMode([device() position] != AVCaptureDevicePositionUnspecified);
301     supportedConstraints.setSupportsWidth(true);
302     supportedConstraints.setSupportsHeight(true);
303     supportedConstraints.setSupportsAspectRatio(true);
304     supportedConstraints.setSupportsFrameRate(true);
305
306     settings.setSupportedConstraints(supportedConstraints);
307
308     m_currentSettings = WTFMove(settings);
309
310     return *m_currentSettings;
311 }
312
313 const RealtimeMediaSourceCapabilities& AVVideoCaptureSource::capabilities() const
314 {
315     if (m_capabilities)
316         return *m_capabilities;
317
318     RealtimeMediaSourceCapabilities capabilities(settings().supportedConstraints());
319     capabilities.setDeviceId(id());
320     AVCaptureDeviceTypedef *videoDevice = device();
321
322     if ([videoDevice position] == AVCaptureDevicePositionFront)
323         capabilities.addFacingMode(RealtimeMediaSourceSettings::User);
324     if ([videoDevice position] == AVCaptureDevicePositionBack)
325         capabilities.addFacingMode(RealtimeMediaSourceSettings::Environment);
326
327     Float64 lowestFrameRateRange = std::numeric_limits<double>::infinity();
328     Float64 highestFrameRateRange = 0;
329     int minimumWidth = std::numeric_limits<int>::infinity();
330     int maximumWidth = 0;
331     int minimumHeight = std::numeric_limits<int>::infinity();
332     int maximumHeight = 0;
333     double minimumAspectRatio = std::numeric_limits<double>::infinity();
334     double maximumAspectRatio = 0;
335
336     for (AVCaptureDeviceFormatType *format in [videoDevice formats]) {
337
338         for (AVFrameRateRangeType *range in [format videoSupportedFrameRateRanges]) {
339             lowestFrameRateRange = std::min<Float64>(lowestFrameRateRange, range.minFrameRate);
340             highestFrameRateRange = std::max<Float64>(highestFrameRateRange, range.maxFrameRate);
341         }
342     }
343
344     for (auto& preset : m_supportedPresets) {
345         auto values = preset.value;
346         updateSizeMinMax(minimumWidth, maximumWidth, values.width());
347         updateSizeMinMax(minimumHeight, maximumHeight, values.height());
348         updateAspectRatioMinMax(minimumAspectRatio, maximumAspectRatio, static_cast<double>(values.width()) / values.height());
349     }
350     capabilities.setFrameRate(CapabilityValueOrRange(lowestFrameRateRange, highestFrameRateRange));
351     capabilities.setWidth(CapabilityValueOrRange(minimumWidth, maximumWidth));
352     capabilities.setHeight(CapabilityValueOrRange(minimumHeight, maximumHeight));
353     capabilities.setAspectRatio(CapabilityValueOrRange(minimumAspectRatio, maximumAspectRatio));
354
355     m_capabilities = WTFMove(capabilities);
356
357     return *m_capabilities;
358 }
359
360 IntSize AVVideoCaptureSource::sizeForPreset(NSString* preset)
361 {
362     if (!preset)
363         return { };
364
365     auto& presets = videoPresets();
366     auto it = presets.find(String(preset));
367     if (it != presets.end())
368         return { it->value.width(), it->value.height() };
369
370     return { };
371 }
372
373 bool AVVideoCaptureSource::setPreset(NSString *preset)
374 {
375     if (!session()) {
376         m_pendingPreset = preset;
377         return true;
378     }
379
380     auto size = sizeForPreset(preset);
381     if (m_presetSize == size)
382         return true;
383
384     m_presetSize = size;
385
386     @try {
387         session().sessionPreset = preset;
388 #if PLATFORM(MAC)
389         auto settingsDictionary = @{
390             (__bridge NSString *)kCVPixelBufferPixelFormatTypeKey: @(videoCaptureFormat),
391             (__bridge NSString *)kCVPixelBufferWidthKey: @(size.width()),
392             (__bridge NSString *)kCVPixelBufferHeightKey: @(size.height())
393         };
394         [m_videoOutput setVideoSettings:settingsDictionary];
395 #endif
396     } @catch(NSException *exception) {
397         LOG(Media, "AVVideoCaptureSource::setPreset(%p), exception thrown configuring device: <%s> %s", this, [[exception name] UTF8String], [[exception reason] UTF8String]);
398         return false;
399     }
400
401     return true;
402 }
403
404 void AVVideoCaptureSource::setFrameRate(double rate)
405 {
406     using namespace PAL;
407     double epsilon = 0.00001;
408     AVFrameRateRangeType *bestFrameRateRange = nil;
409     for (AVFrameRateRangeType *frameRateRange in [[device() activeFormat] videoSupportedFrameRateRanges]) {
410         if (rate + epsilon >= [frameRateRange minFrameRate] && rate - epsilon <= [frameRateRange maxFrameRate]) {
411             if (!bestFrameRateRange || CMTIME_COMPARE_INLINE([frameRateRange minFrameDuration], >, [bestFrameRateRange minFrameDuration]))
412                 bestFrameRateRange = frameRateRange;
413         }
414     }
415
416     if (!bestFrameRateRange || !isFrameRateSupported(rate)) {
417         LOG(Media, "AVVideoCaptureSource::setFrameRate(%p), frame rate %f not supported by video device", this, rate);
418         return;
419     }
420
421     NSError *error = nil;
422     @try {
423         if ([device() lockForConfiguration:&error]) {
424             if (bestFrameRateRange.minFrameRate == bestFrameRateRange.maxFrameRate) {
425                 [device() setActiveVideoMinFrameDuration:[bestFrameRateRange minFrameDuration]];
426                 [device() setActiveVideoMaxFrameDuration:[bestFrameRateRange maxFrameDuration]];
427             } else {
428                 [device() setActiveVideoMinFrameDuration: CMTimeMake(1, rate)];
429                 [device() setActiveVideoMaxFrameDuration: CMTimeMake(1, rate)];
430             }
431             [device() unlockForConfiguration];
432         }
433     } @catch(NSException *exception) {
434         LOG(Media, "AVVideoCaptureSource::setFrameRate(%p), exception thrown configuring device: <%s> %s", this, [[exception name] UTF8String], [[exception reason] UTF8String]);
435         return;
436     }
437
438     if (error) {
439         LOG(Media, "AVVideoCaptureSource::setFrameRate(%p), failed to lock video device for configuration: %s", this, [[error localizedDescription] UTF8String]);
440         return;
441     }
442
443     LOG(Media, "AVVideoCaptureSource::setFrameRate(%p) - set frame rate range to %f", this, rate);
444     return;
445 }
446
447 void AVVideoCaptureSource::applySizeAndFrameRate(std::optional<int> width, std::optional<int> height, std::optional<double> frameRate)
448 {
449     if (width || height)
450         setPreset(bestSessionPresetForVideoDimensions(WTFMove(width), WTFMove(height)));
451
452     if (frameRate)
453         setFrameRate(frameRate.value());
454 }
455
456 static inline int sensorOrientation(AVCaptureVideoOrientation videoOrientation)
457 {
458 #if PLATFORM(IOS)
459     switch (videoOrientation) {
460     case AVCaptureVideoOrientationPortrait:
461         return 180;
462     case AVCaptureVideoOrientationPortraitUpsideDown:
463         return 0;
464     case AVCaptureVideoOrientationLandscapeRight:
465         return 90;
466     case AVCaptureVideoOrientationLandscapeLeft:
467         return -90;
468     }
469 #else
470     switch (videoOrientation) {
471     case AVCaptureVideoOrientationPortrait:
472         return 0;
473     case AVCaptureVideoOrientationPortraitUpsideDown:
474         return 180;
475     case AVCaptureVideoOrientationLandscapeRight:
476         return 90;
477     case AVCaptureVideoOrientationLandscapeLeft:
478         return -90;
479     }
480 #endif
481 }
482
483 static inline int sensorOrientationFromVideoOutput(AVCaptureVideoDataOutputType* videoOutput)
484 {
485     AVCaptureConnectionType* connection = [videoOutput connectionWithMediaType: getAVMediaTypeVideo()];
486     return connection ? sensorOrientation([connection videoOrientation]) : 0;
487 }
488
489 bool AVVideoCaptureSource::setupSession()
490 {
491     if (m_session)
492         return true;
493
494     m_session = adoptNS([allocAVCaptureSessionInstance() init]);
495     [m_session addObserver:m_objcObserver.get() forKeyPath:@"rate" options:NSKeyValueObservingOptionNew context:(void *)nil];
496
497     [m_session beginConfiguration];
498     bool success = setupCaptureSession();
499     [m_session commitConfiguration];
500
501     if (!success)
502         captureFailed();
503
504     return success;
505 }
506
507 bool AVVideoCaptureSource::setupCaptureSession()
508 {
509 #if PLATFORM(IOS)
510     RealtimeMediaSourceCenterMac::videoCaptureSourceFactory().setActiveSource(*this);
511 #endif
512
513     NSError *error = nil;
514     RetainPtr<AVCaptureDeviceInputType> videoIn = adoptNS([allocAVCaptureDeviceInputInstance() initWithDevice:device() error:&error]);
515     if (error) {
516         RELEASE_LOG(Media, "AVVideoCaptureSource::setupCaptureSession(%p), failed to allocate AVCaptureDeviceInput: %s", this, [[error localizedDescription] UTF8String]);
517         return false;
518     }
519
520     if (![session() canAddInput:videoIn.get()]) {
521         RELEASE_LOG(Media, "AVVideoCaptureSource::setupCaptureSession(%p), unable to add video input device", this);
522         return false;
523     }
524     [session() addInput:videoIn.get()];
525
526     m_videoOutput = adoptNS([allocAVCaptureVideoDataOutputInstance() init]);
527     auto settingsDictionary = adoptNS([[NSMutableDictionary alloc] initWithObjectsAndKeys: [NSNumber numberWithInt:videoCaptureFormat], kCVPixelBufferPixelFormatTypeKey, nil]);
528     if (m_pendingPreset) {
529 #if PLATFORM(MAC)
530         auto size = sizeForPreset(m_pendingPreset.get());
531         [settingsDictionary setObject:@(size.width()) forKey:(__bridge NSString *)kCVPixelBufferWidthKey];
532         [settingsDictionary setObject:@(size.height()) forKey:(__bridge NSString *)kCVPixelBufferHeightKey];
533 #endif
534     }
535
536     [m_videoOutput setVideoSettings:settingsDictionary.get()];
537     [m_videoOutput setAlwaysDiscardsLateVideoFrames:YES];
538     [m_videoOutput setSampleBufferDelegate:m_objcObserver.get() queue:globaVideoCaptureSerialQueue()];
539
540     if (![session() canAddOutput:m_videoOutput.get()]) {
541         RELEASE_LOG(Media, "AVVideoCaptureSource::setupCaptureSession(%p), unable to add video sample buffer output delegate", this);
542         return false;
543     }
544     [session() addOutput:m_videoOutput.get()];
545
546 #if PLATFORM(IOS)
547     setPreset(m_pendingPreset.get());
548 #endif
549
550     m_sensorOrientation = sensorOrientationFromVideoOutput(m_videoOutput.get());
551     computeSampleRotation();
552
553     return true;
554 }
555
556 void AVVideoCaptureSource::shutdownCaptureSession()
557 {
558     m_buffer = nullptr;
559     m_width = 0;
560     m_height = 0;
561 }
562
563 void AVVideoCaptureSource::monitorOrientation(OrientationNotifier& notifier)
564 {
565 #if PLATFORM(IOS)
566     notifier.addObserver(*this);
567     orientationChanged(notifier.orientation());
568 #else
569     UNUSED_PARAM(notifier);
570 #endif
571 }
572
573 void AVVideoCaptureSource::orientationChanged(int orientation)
574 {
575     ASSERT(orientation == 0 || orientation == 90 || orientation == -90 || orientation == 180);
576     m_deviceOrientation = orientation;
577     computeSampleRotation();
578 }
579
580 void AVVideoCaptureSource::computeSampleRotation()
581 {
582     bool frontCamera = [device() position] == AVCaptureDevicePositionFront;
583     switch (m_sensorOrientation - m_deviceOrientation) {
584     case 0:
585         m_sampleRotation = MediaSample::VideoRotation::None;
586         break;
587     case 180:
588     case -180:
589         m_sampleRotation = MediaSample::VideoRotation::UpsideDown;
590         break;
591     case 90:
592         m_sampleRotation = frontCamera ? MediaSample::VideoRotation::Left : MediaSample::VideoRotation::Right;
593         break;
594     case -90:
595     case -270:
596         m_sampleRotation = frontCamera ? MediaSample::VideoRotation::Right : MediaSample::VideoRotation::Left;
597         break;
598     default:
599         ASSERT_NOT_REACHED();
600         m_sampleRotation = MediaSample::VideoRotation::None;
601     }
602 }
603
604 void AVVideoCaptureSource::processNewFrame(RetainPtr<CMSampleBufferRef> sampleBuffer, RetainPtr<AVCaptureConnectionType> connection)
605 {
606     if (!isProducingData() || muted())
607         return;
608
609     CMFormatDescriptionRef formatDescription = PAL::CMSampleBufferGetFormatDescription(sampleBuffer.get());
610     if (!formatDescription)
611         return;
612
613     m_buffer = sampleBuffer;
614     CMVideoDimensions dimensions = PAL::CMVideoFormatDescriptionGetDimensions(formatDescription);
615     if (m_sampleRotation == MediaSample::VideoRotation::Left || m_sampleRotation == MediaSample::VideoRotation::Right)
616         std::swap(dimensions.width, dimensions.height);
617
618     if (dimensions.width != m_width || dimensions.height != m_height) {
619         OptionSet<RealtimeMediaSourceSettings::Flag> changed;
620         if (m_width != dimensions.width)
621             changed.add(RealtimeMediaSourceSettings::Flag::Width);
622         if (m_height != dimensions.height)
623             changed.add(RealtimeMediaSourceSettings::Flag::Height);
624
625         m_width = dimensions.width;
626         m_height = dimensions.height;
627         settingsDidChange(changed);
628     }
629
630     videoSampleAvailable(MediaSampleAVFObjC::create(m_buffer.get(), m_sampleRotation, [connection isVideoMirrored]));
631 }
632
633 void AVVideoCaptureSource::captureOutputDidOutputSampleBufferFromConnection(AVCaptureOutputType*, CMSampleBufferRef sampleBuffer, AVCaptureConnectionType* captureConnection)
634 {
635     RetainPtr<CMSampleBufferRef> buffer = sampleBuffer;
636     RetainPtr<AVCaptureConnectionType> connection = captureConnection;
637
638     scheduleDeferredTask([this, buffer, connection] {
639         this->processNewFrame(buffer, connection);
640     });
641 }
642
643 NSString* AVVideoCaptureSource::bestSessionPresetForVideoDimensions(std::optional<int> width, std::optional<int> height)
644 {
645     if (!width && !height)
646         return nil;
647
648     int widthValue = width ? width.value() : 0;
649     int heightValue = height ? height.value() : 0;
650
651     for (auto& preset : videoPresets()) {
652         auto size = preset.value;
653         NSString* name = preset.key;
654
655         if ((!widthValue || widthValue == size.width()) && (!heightValue || heightValue == size.height()))
656             return name;
657     }
658
659     return nil;
660 }
661
662 bool AVVideoCaptureSource::isFrameRateSupported(double frameRate)
663 {
664     double epsilon = 0.001;
665     for (AVFrameRateRangeType *range in [[device() activeFormat] videoSupportedFrameRateRanges]) {
666         if (frameRate + epsilon >= range.minFrameRate && frameRate - epsilon <= range.maxFrameRate)
667             return true;
668     }
669     return false;
670 }
671
672 bool AVVideoCaptureSource::supportsSizeAndFrameRate(std::optional<int> width, std::optional<int> height, std::optional<double> frameRate)
673 {
674     if (!height && !width && !frameRate)
675         return true;
676
677     if ((height || width) && !bestSessionPresetForVideoDimensions(WTFMove(width), WTFMove(height)))
678         return false;
679
680     if (!frameRate)
681         return true;
682
683     return isFrameRateSupported(frameRate.value());
684 }
685
686 void AVVideoCaptureSource::captureSessionIsRunningDidChange(bool state)
687 {
688     scheduleDeferredTask([this, state] {
689         if ((state == m_isRunning) && (state == !muted()))
690             return;
691
692         m_isRunning = state;
693         notifyMutedChange(!m_isRunning);
694     });
695 }
696
697 bool AVVideoCaptureSource::interrupted() const
698 {
699     if (m_interruption != InterruptionReason::None)
700         return true;
701
702     return RealtimeMediaSource::interrupted();
703 }
704
705 #if PLATFORM(IOS)
706 void AVVideoCaptureSource::captureSessionRuntimeError(RetainPtr<NSError> error)
707 {
708     if (!m_isRunning || error.get().code != AVErrorMediaServicesWereReset)
709         return;
710
711     // Try to restart the session, but reset m_isRunning immediately so if it fails we won't try again.
712     [m_session startRunning];
713     m_isRunning = [m_session isRunning];
714 }
715
716 void AVVideoCaptureSource::captureSessionBeginInterruption(RetainPtr<NSNotification> notification)
717 {
718     m_interruption = static_cast<AVVideoCaptureSource::InterruptionReason>([notification.get().userInfo[AVCaptureSessionInterruptionReasonKey] integerValue]);
719 }
720
721 void AVVideoCaptureSource::captureSessionEndInterruption(RetainPtr<NSNotification>)
722 {
723     InterruptionReason reason = m_interruption;
724
725     m_interruption = InterruptionReason::None;
726     if (reason != InterruptionReason::VideoNotAllowedInSideBySide || m_isRunning || !m_session)
727         return;
728
729     [m_session startRunning];
730     m_isRunning = [m_session isRunning];
731 }
732 #endif
733
734 } // namespace WebCore
735
736 @implementation WebCoreAVVideoCaptureSourceObserver
737
738 - (id)initWithCallback:(AVVideoCaptureSource*)callback
739 {
740     self = [super init];
741     if (!self)
742         return nil;
743
744     m_callback = callback;
745
746     return self;
747 }
748
749 - (void)disconnect
750 {
751     [NSObject cancelPreviousPerformRequestsWithTarget:self];
752     [self removeNotificationObservers];
753     m_callback = nullptr;
754 }
755
756 - (void)addNotificationObservers
757 {
758 #if PLATFORM(IOS)
759     ASSERT(m_callback);
760
761     NSNotificationCenter* center = [NSNotificationCenter defaultCenter];
762     AVCaptureSessionType* session = m_callback->session();
763
764     [center addObserver:self selector:@selector(sessionRuntimeError:) name:AVCaptureSessionRuntimeErrorNotification object:session];
765     [center addObserver:self selector:@selector(beginSessionInterrupted:) name:AVCaptureSessionWasInterruptedNotification object:session];
766     [center addObserver:self selector:@selector(endSessionInterrupted:) name:AVCaptureSessionInterruptionEndedNotification object:session];
767 #endif
768 }
769
770 - (void)removeNotificationObservers
771 {
772 #if PLATFORM(IOS)
773     [[NSNotificationCenter defaultCenter] removeObserver:self];
774 #endif
775 }
776
777 - (void)captureOutput:(AVCaptureOutputType*)captureOutput didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer fromConnection:(AVCaptureConnectionType*)connection
778 {
779     if (!m_callback)
780         return;
781
782     m_callback->captureOutputDidOutputSampleBufferFromConnection(captureOutput, sampleBuffer, connection);
783 }
784
785 - (void)observeValueForKeyPath:keyPath ofObject:(id)object change:(NSDictionary*)change context:(void*)context
786 {
787     UNUSED_PARAM(object);
788     UNUSED_PARAM(context);
789
790     if (!m_callback)
791         return;
792
793     id newValue = [change valueForKey:NSKeyValueChangeNewKey];
794
795 #if !LOG_DISABLED
796     bool willChange = [[change valueForKey:NSKeyValueChangeNotificationIsPriorKey] boolValue];
797
798     if (willChange)
799         LOG(Media, "WebCoreAVVideoCaptureSourceObserver::observeValueForKeyPath(%p) - will change, keyPath = %s", self, [keyPath UTF8String]);
800     else {
801         RetainPtr<NSString> valueString = adoptNS([[NSString alloc] initWithFormat:@"%@", newValue]);
802         LOG(Media, "WebCoreAVVideoCaptureSourceObserver::observeValueForKeyPath(%p) - did change, keyPath = %s, value = %s", self, [keyPath UTF8String], [valueString.get() UTF8String]);
803     }
804 #endif
805
806     if ([keyPath isEqualToString:@"running"])
807         m_callback->captureSessionIsRunningDidChange([newValue boolValue]);
808 }
809
810 #if PLATFORM(IOS)
811 - (void)sessionRuntimeError:(NSNotification*)notification
812 {
813     NSError *error = notification.userInfo[AVCaptureSessionErrorKey];
814     LOG(Media, "WebCoreAVVideoCaptureSourceObserver::sessionRuntimeError(%p) - error = %s", self, [[error localizedDescription] UTF8String]);
815
816     if (m_callback)
817         m_callback->captureSessionRuntimeError(error);
818 }
819
820 - (void)beginSessionInterrupted:(NSNotification*)notification
821 {
822     LOG(Media, "WebCoreAVVideoCaptureSourceObserver::beginSessionInterrupted(%p) - reason = %d", self, [notification.userInfo[AVCaptureSessionInterruptionReasonKey] integerValue]);
823
824     if (m_callback)
825         m_callback->captureSessionBeginInterruption(notification);
826 }
827
828 - (void)endSessionInterrupted:(NSNotification*)notification
829 {
830     LOG(Media, "WebCoreAVVideoCaptureSourceObserver::endSessionInterrupted(%p)", self);
831
832     if (m_callback)
833         m_callback->captureSessionEndInterruption(notification);
834 }
835 #endif
836
837 @end
838
839 #endif // ENABLE(MEDIA_STREAM)