dc6a18b54a2064e20aa238815bdc22532887b970
[WebKit-https.git] / Source / WebCore / platform / mediastream / mac / DisplayCaptureSourceCocoa.cpp
1 /*
2  * Copyright (C) 2017-2018 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
17  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24  */
25
26 #include "config.h"
27 #include "DisplayCaptureSourceCocoa.h"
28
29 #if ENABLE(MEDIA_STREAM)
30
31 #include "Logging.h"
32 #include "MediaSampleAVFObjC.h"
33 #include "PixelBufferConformerCV.h"
34 #include "PixelBufferResizer.h"
35 #include "RealtimeMediaSource.h"
36 #include "RealtimeMediaSourceCenter.h"
37 #include "RealtimeMediaSourceSettings.h"
38 #include "RealtimeVideoUtilities.h"
39 #include "Timer.h"
40 #include <CoreMedia/CMSync.h>
41 #include <mach/mach_time.h>
42 #include <pal/avfoundation/MediaTimeAVFoundation.h>
43 #include <pal/cf/CoreMediaSoftLink.h>
44 #include <pal/spi/cf/CoreAudioSPI.h>
45 #include <sys/time.h>
46 #include <wtf/MainThread.h>
47 #include <wtf/NeverDestroyed.h>
48
49 #include "CoreVideoSoftLink.h"
50
51 namespace WebCore {
52 using namespace PAL;
53
54 DisplayCaptureSourceCocoa::DisplayCaptureSourceCocoa(String&& name)
55     : RealtimeMediaSource("", Type::Video, WTFMove(name))
56     , m_timer(RunLoop::current(), this, &DisplayCaptureSourceCocoa::emitFrame)
57 {
58 }
59
60 DisplayCaptureSourceCocoa::~DisplayCaptureSourceCocoa()
61 {
62 #if PLATFORM(IOS)
63     RealtimeMediaSourceCenter::singleton().videoFactory().unsetActiveSource(*this);
64 #endif
65 }
66
67 const RealtimeMediaSourceCapabilities& DisplayCaptureSourceCocoa::capabilities()
68 {
69     if (!m_capabilities) {
70         RealtimeMediaSourceCapabilities capabilities(settings().supportedConstraints());
71
72         // FIXME: what should these be?
73         capabilities.setWidth(CapabilityValueOrRange(72, 2880));
74         capabilities.setHeight(CapabilityValueOrRange(45, 1800));
75         capabilities.setFrameRate(CapabilityValueOrRange(.01, 60.0));
76
77         m_capabilities = WTFMove(capabilities);
78     }
79     return m_capabilities.value();
80 }
81
82 const RealtimeMediaSourceSettings& DisplayCaptureSourceCocoa::settings()
83 {
84     if (!m_currentSettings) {
85         RealtimeMediaSourceSettings settings;
86         settings.setFrameRate(frameRate());
87         auto size = frameSize();
88         if (!size.isEmpty()) {
89             settings.setWidth(size.width());
90             settings.setHeight(size.height());
91         }
92         settings.setDisplaySurface(surfaceType());
93         settings.setLogicalSurface(false);
94
95         RealtimeMediaSourceSupportedConstraints supportedConstraints;
96         supportedConstraints.setSupportsFrameRate(true);
97         supportedConstraints.setSupportsWidth(true);
98         supportedConstraints.setSupportsHeight(true);
99         supportedConstraints.setSupportsDisplaySurface(true);
100         supportedConstraints.setSupportsLogicalSurface(true);
101
102         settings.setSupportedConstraints(supportedConstraints);
103
104         m_currentSettings = WTFMove(settings);
105     }
106     return m_currentSettings.value();
107 }
108
109 void DisplayCaptureSourceCocoa::settingsDidChange(OptionSet<RealtimeMediaSourceSettings::Flag> settings)
110 {
111     if (settings.contains(RealtimeMediaSourceSettings::Flag::FrameRate) && m_timer.isActive())
112         m_timer.startRepeating(1_s / frameRate());
113
114     if (settings.containsAny({ RealtimeMediaSourceSettings::Flag::Width, RealtimeMediaSourceSettings::Flag::Height })) {
115         m_bufferAttributes = nullptr;
116         m_lastSampleBuffer = nullptr;
117     }
118
119     m_currentSettings = { };
120 }
121
122 void DisplayCaptureSourceCocoa::startProducingData()
123 {
124 #if PLATFORM(IOS)
125     RealtimeMediaSourceCenter::singleton().videoFactory().setActiveSource(*this);
126 #endif
127
128     m_startTime = MonotonicTime::now();
129     m_timer.startRepeating(1_s / frameRate());
130 }
131
132 void DisplayCaptureSourceCocoa::stopProducingData()
133 {
134     m_timer.stop();
135     m_elapsedTime += MonotonicTime::now() - m_startTime;
136     m_startTime = MonotonicTime::nan();
137 }
138
139 Seconds DisplayCaptureSourceCocoa::elapsedTime()
140 {
141     if (std::isnan(m_startTime))
142         return m_elapsedTime;
143
144     return m_elapsedTime + (MonotonicTime::now() - m_startTime);
145 }
146
147 IntSize DisplayCaptureSourceCocoa::frameSize() const
148 {
149     IntSize frameSize = size();
150     if (frameSize.isEmpty())
151         frameSize = m_intrinsicSize;
152
153     return frameSize;
154 }
155
156 void DisplayCaptureSourceCocoa::setIntrinsicSize(const IntSize& size)
157 {
158     if (m_intrinsicSize == size)
159         return;
160
161     m_intrinsicSize = size;
162     m_lastSampleBuffer = nullptr;
163 }
164
165 void DisplayCaptureSourceCocoa::emitFrame()
166 {
167     if (muted())
168         return;
169
170     auto pixelBuffer = generateFrame();
171     if (!pixelBuffer)
172         return;
173
174     if (m_lastSampleBuffer && m_lastFullSizedPixelBuffer && CFEqual(m_lastFullSizedPixelBuffer.get(), pixelBuffer.get())) {
175         videoSampleAvailable(MediaSampleAVFObjC::create(m_lastSampleBuffer.get()));
176         return;
177     }
178
179     m_lastFullSizedPixelBuffer = pixelBuffer;
180
181     int width = WTF::safeCast<int>(CVPixelBufferGetWidth(pixelBuffer.get()));
182     int height = WTF::safeCast<int>(CVPixelBufferGetHeight(pixelBuffer.get()));
183     auto requestedSize = frameSize();
184     if (width != requestedSize.width() || height != requestedSize.height()) {
185         if (m_pixelBufferResizer && !m_pixelBufferResizer->canResizeTo(requestedSize))
186             m_pixelBufferResizer = nullptr;
187
188         if (!m_pixelBufferResizer)
189             m_pixelBufferResizer = std::make_unique<PixelBufferResizer>(requestedSize, preferedPixelBufferFormat());
190
191         pixelBuffer = m_pixelBufferResizer->resize(pixelBuffer.get());
192     } else {
193         m_pixelBufferResizer = nullptr;
194
195         auto pixelFormatType = CVPixelBufferGetPixelFormatType(pixelBuffer.get());
196         if (pixelFormatType != preferedPixelBufferFormat()) {
197             if (!m_pixelBufferConformer) {
198                 auto preferredFromat = preferedPixelBufferFormat();
199                 auto conformerAttributes = adoptCF(CFDictionaryCreateMutable(nullptr, 0, &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks));
200                 auto videoType = adoptCF(CFNumberCreate(nullptr,  kCFNumberSInt32Type,  &preferredFromat));
201                 CFDictionarySetValue(conformerAttributes.get(), kCVPixelBufferPixelFormatTypeKey, videoType.get());
202
203                 m_pixelBufferConformer = std::make_unique<PixelBufferConformerCV>(conformerAttributes.get());
204             }
205
206             pixelBuffer = m_pixelBufferConformer->convert(pixelBuffer.get());
207         }
208     }
209     if (!pixelBuffer)
210         return;
211
212     m_lastSampleBuffer = sampleBufferFromPixelBuffer(pixelBuffer.get());
213     if (!m_lastSampleBuffer)
214         return;
215
216     videoSampleAvailable(MediaSampleAVFObjC::create(m_lastSampleBuffer.get()));
217 }
218
219 RetainPtr<CMSampleBufferRef> DisplayCaptureSourceCocoa::sampleBufferFromPixelBuffer(CVPixelBufferRef pixelBuffer)
220 {
221     if (!pixelBuffer)
222         return nullptr;
223
224     CMTime sampleTime = CMTimeMake(((elapsedTime() + 100_ms) * 100).seconds(), 100);
225     CMSampleTimingInfo timingInfo = { kCMTimeInvalid, sampleTime, sampleTime };
226
227     CMVideoFormatDescriptionRef formatDescription = nullptr;
228     auto status = CMVideoFormatDescriptionCreateForImageBuffer(kCFAllocatorDefault, (CVImageBufferRef)pixelBuffer, &formatDescription);
229     if (status) {
230         RELEASE_LOG(Media, "Failed to initialize CMVideoFormatDescription with error code: %d", static_cast<int>(status));
231         return nullptr;
232     }
233
234     CMSampleBufferRef sampleBuffer;
235     status = CMSampleBufferCreateReadyWithImageBuffer(kCFAllocatorDefault, (CVImageBufferRef)pixelBuffer, formatDescription, &timingInfo, &sampleBuffer);
236     CFRelease(formatDescription);
237     if (status) {
238         RELEASE_LOG(Media, "Failed to initialize CMSampleBuffer with error code: %d", static_cast<int>(status));
239         return nullptr;
240     }
241
242     return adoptCF(sampleBuffer);
243 }
244
245 #if HAVE(IOSURFACE) && PLATFORM(MAC)
246 static int32_t roundUpToMacroblockMultiple(int32_t size)
247 {
248     return (size + 15) & ~15;
249 }
250
251 RetainPtr<CVPixelBufferRef> DisplayCaptureSourceCocoa::pixelBufferFromIOSurface(IOSurfaceRef surface)
252 {
253     if (!m_bufferAttributes) {
254         m_bufferAttributes = adoptCF(CFDictionaryCreateMutable(nullptr, 0, &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks));
255
256         auto format = IOSurfaceGetPixelFormat(surface);
257         if (format == kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange || format == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange) {
258
259             // If the width x height isn't a multiple of 16 x 16 and the surface has extra memory in the planes, set pixel buffer attributes to reflect it.
260             auto width = IOSurfaceGetWidth(surface);
261             auto height = IOSurfaceGetHeight(surface);
262             int32_t extendedRight = roundUpToMacroblockMultiple(width) - width;
263             int32_t extendedBottom = roundUpToMacroblockMultiple(height) - height;
264
265             if ((IOSurfaceGetBytesPerRowOfPlane(surface, 0) >= width + extendedRight)
266                 && (IOSurfaceGetBytesPerRowOfPlane(surface, 1) >= width + extendedRight)
267                 && (IOSurfaceGetAllocSize(surface) >= (height + extendedBottom) * IOSurfaceGetBytesPerRowOfPlane(surface, 0) * 3 / 2)) {
268                 auto cfInt = adoptCF(CFNumberCreate(nullptr,  kCFNumberIntType,  &extendedRight));
269                 CFDictionarySetValue(m_bufferAttributes.get(), kCVPixelBufferExtendedPixelsRightKey, cfInt.get());
270                 cfInt = adoptCF(CFNumberCreate(nullptr,  kCFNumberIntType,  &extendedBottom));
271                 CFDictionarySetValue(m_bufferAttributes.get(), kCVPixelBufferExtendedPixelsBottomKey, cfInt.get());
272             }
273         }
274
275         CFDictionarySetValue(m_bufferAttributes.get(), kCVPixelBufferOpenGLCompatibilityKey, kCFBooleanTrue);
276     }
277
278     CVPixelBufferRef pixelBuffer;
279     auto status = CVPixelBufferCreateWithIOSurface(kCFAllocatorDefault, surface, m_bufferAttributes.get(), &pixelBuffer);
280     if (status) {
281         RELEASE_LOG(Media, "CVPixelBufferCreateWithIOSurface failed with error code: %d", static_cast<int>(status));
282         return nullptr;
283     }
284
285     return adoptCF(pixelBuffer);
286 }
287 #endif
288
289 } // namespace WebCore
290
291 #endif // ENABLE(MEDIA_STREAM)