Use existing RGB colorspace instead of creating a new one
[WebKit-https.git] / Source / WebCore / platform / mediastream / mac / ScreenDisplayCaptureSourceMac.mm
1 /*
2  * Copyright (C) 2017 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
17  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24  */
25
26 #include "config.h"
27 #include "ScreenDisplayCaptureSourceMac.h"
28
29 #if ENABLE(MEDIA_STREAM) && PLATFORM(MAC)
30
31 #include "GraphicsContextCG.h"
32 #include "ImageBuffer.h"
33 #include "Logging.h"
34 #include "MediaConstraints.h"
35 #include "MediaSampleAVFObjC.h"
36 #include "NotImplemented.h"
37 #include "PlatformLayer.h"
38 #include "RealtimeMediaSourceSettings.h"
39
40 #include "CoreVideoSoftLink.h"
41
42 extern "C" {
43 size_t CGDisplayModeGetPixelsWide(CGDisplayModeRef);
44 size_t CGDisplayModeGetPixelsHigh(CGDisplayModeRef);
45 }
46
47 namespace WebCore {
48
49 static int32_t roundUpToMacroblockMultiple(int32_t size)
50 {
51     return (size + 15) & ~15;
52 }
53
54 std::optional<CGDirectDisplayID> ScreenDisplayCaptureSourceMac::updateDisplayID(CGDirectDisplayID displayID)
55 {
56     uint32_t displayCount = 0;
57     auto err = CGGetActiveDisplayList(0, nullptr, &displayCount);
58     if (err) {
59         RELEASE_LOG(Media, "CGGetActiveDisplayList() returned error %d when trying to get display count", static_cast<int>(err));
60         return std::nullopt;
61     }
62
63     if (!displayCount) {
64         RELEASE_LOG(Media, "CGGetActiveDisplayList() returned a display count of 0");
65         return std::nullopt;
66     }
67
68     CGDirectDisplayID activeDisplays[displayCount];
69     err = CGGetActiveDisplayList(displayCount, &(activeDisplays[0]), &displayCount);
70     if (err) {
71         RELEASE_LOG(Media, "CGGetActiveDisplayList() returned error %d when trying to get the active display list", static_cast<int>(err));
72         return std::nullopt;
73     }
74
75     auto displayMask = CGDisplayIDToOpenGLDisplayMask(displayID);
76     for (auto display : activeDisplays) {
77         if (displayMask == CGDisplayIDToOpenGLDisplayMask(display))
78             return display;
79     }
80
81     return std::nullopt;
82 }
83
84 CaptureSourceOrError ScreenDisplayCaptureSourceMac::create(const String& deviceID, const MediaConstraints* constraints)
85 {
86     bool ok;
87     auto displayID = deviceID.toUIntStrict(&ok);
88     if (!ok) {
89         RELEASE_LOG(Media, "Display ID does not convert to 32-bit integer");
90         return { };
91     }
92
93     auto actualDisplayID = updateDisplayID(displayID);
94     if (!actualDisplayID)
95         return { };
96
97     auto source = adoptRef(*new ScreenDisplayCaptureSourceMac(actualDisplayID.value()));
98     if (constraints && source->applyConstraints(*constraints))
99         return { };
100
101     return CaptureSourceOrError(WTFMove(source));
102 }
103
104 ScreenDisplayCaptureSourceMac::ScreenDisplayCaptureSourceMac(uint32_t displayID)
105     : DisplayCaptureSourceCocoa("Screen")
106     , m_displayID(displayID)
107 {
108 }
109
110 ScreenDisplayCaptureSourceMac::~ScreenDisplayCaptureSourceMac()
111 {
112     if (m_observingDisplayChanges)
113         CGDisplayRemoveReconfigurationCallback(displayReconfigurationCallBack, this);
114
115     m_currentFrame = nullptr;
116 }
117
118 bool ScreenDisplayCaptureSourceMac::createDisplayStream()
119 {
120     static const int screenQueueMaximumLength = 6;
121
122     auto actualDisplayID = updateDisplayID(m_displayID);
123     if (!actualDisplayID) {
124         captureFailed();
125         return false;
126     }
127
128     if (m_displayID != actualDisplayID.value()) {
129         m_displayID = actualDisplayID.value();
130         RELEASE_LOG(Media, "ScreenDisplayCaptureSourceMac::create(%p), display ID changed to %d", this, static_cast<int>(m_displayID));
131     }
132
133     if (!m_displayStream) {
134
135         if (size().isEmpty()) {
136             CGDisplayModeRef displayMode = CGDisplayCopyDisplayMode(m_displayID);
137             auto screenWidth = CGDisplayModeGetPixelsWide(displayMode);
138             auto screenHeight = CGDisplayModeGetPixelsHigh(displayMode);
139             if (!screenWidth || !screenHeight) {
140                 RELEASE_LOG(Media, "ScreenDisplayCaptureSourceMac::createDisplayStream(%p), unable to get screen width/height", this);
141                 captureFailed();
142                 return false;
143             }
144             setWidth(screenWidth);
145             setHeight(screenHeight);
146             CGDisplayModeRelease(displayMode);
147         }
148
149         if (!m_captureQueue)
150             m_captureQueue = adoptOSObject(dispatch_queue_create("ScreenDisplayCaptureSourceMac Capture Queue", DISPATCH_QUEUE_SERIAL));
151
152         double frameTime = 1 / frameRate();
153         auto frameTimeCF = adoptCF(CFNumberCreate(nullptr,  kCFNumberDoubleType,  &frameTime));
154         int depth = screenQueueMaximumLength;
155         auto depthCF = adoptCF(CFNumberCreate(nullptr,  kCFNumberIntType,  &depth));
156         CFTypeRef keys[] = {
157             kCGDisplayStreamMinimumFrameTime,
158             kCGDisplayStreamQueueDepth,
159             kCGDisplayStreamColorSpace,
160             kCGDisplayStreamShowCursor,
161         };
162         CFTypeRef values[] = {
163             frameTimeCF.get(),
164             depthCF.get(),
165             sRGBColorSpaceRef(),
166             kCFBooleanTrue,
167         };
168         auto streamOptions = adoptCF(CFDictionaryCreate(kCFAllocatorDefault, keys, values, WTF_ARRAY_LENGTH(keys), &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks));
169
170         auto weakThis = m_weakFactory.createWeakPtr(*this);
171         m_frameAvailableBlock = Block_copy(^(CGDisplayStreamFrameStatus status, uint64_t displayTime, IOSurfaceRef frameSurface, CGDisplayStreamUpdateRef updateRef) {
172             if (!weakThis)
173                 return;
174
175             weakThis->frameAvailable(status, displayTime, frameSurface, updateRef);
176         });
177
178         m_displayStream = adoptCF(CGDisplayStreamCreateWithDispatchQueue(m_displayID, size().width(), size().height(), kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, streamOptions.get(), m_captureQueue.get(), m_frameAvailableBlock));
179         if (!m_displayStream) {
180             RELEASE_LOG(Media, "ScreenDisplayCaptureSourceMac::createDisplayStream(%p), CGDisplayStreamCreate failed", this);
181             captureFailed();
182             return false;
183         }
184     }
185
186     if (!m_observingDisplayChanges) {
187         CGDisplayRegisterReconfigurationCallback(displayReconfigurationCallBack, this);
188         m_observingDisplayChanges = true;
189     }
190
191     return true;
192 }
193
194 void ScreenDisplayCaptureSourceMac::startProducingData()
195 {
196     DisplayCaptureSourceCocoa::startProducingData();
197
198     if (m_isRunning)
199         return;
200
201     startDisplayStream();
202 }
203
204 void ScreenDisplayCaptureSourceMac::stopProducingData()
205 {
206     DisplayCaptureSourceCocoa::stopProducingData();
207
208     if (!m_isRunning)
209         return;
210
211     if (m_displayStream)
212         CGDisplayStreamStop(m_displayStream.get());
213
214     m_isRunning = false;
215 }
216
217 RetainPtr<CMSampleBufferRef> ScreenDisplayCaptureSourceMac::sampleBufferFromPixelBuffer(CVPixelBufferRef pixelBuffer)
218 {
219     if (!pixelBuffer)
220         return nullptr;
221
222     CMTime sampleTime = CMTimeMake((elapsedTime() + .1) * 100, 100);
223     CMSampleTimingInfo timingInfo = { kCMTimeInvalid, sampleTime, sampleTime };
224
225     CMVideoFormatDescriptionRef formatDescription = nullptr;
226     auto status = CMVideoFormatDescriptionCreateForImageBuffer(kCFAllocatorDefault, (CVImageBufferRef)pixelBuffer, &formatDescription);
227     if (status) {
228         RELEASE_LOG(Media, "Failed to initialize CMVideoFormatDescription with error code: %d", static_cast<int>(status));
229         return nullptr;
230     }
231
232     CMSampleBufferRef sampleBuffer;
233     status = CMSampleBufferCreateReadyWithImageBuffer(kCFAllocatorDefault, (CVImageBufferRef)pixelBuffer, formatDescription, &timingInfo, &sampleBuffer);
234     CFRelease(formatDescription);
235     if (status) {
236         RELEASE_LOG(Media, "Failed to initialize CMSampleBuffer with error code: %d", static_cast<int>(status));
237         return nullptr;
238     }
239
240     return adoptCF(sampleBuffer);
241 }
242
243 RetainPtr<CVPixelBufferRef> ScreenDisplayCaptureSourceMac::pixelBufferFromIOSurface(IOSurfaceRef surface)
244 {
245     if (!m_bufferAttributes) {
246         m_bufferAttributes = adoptCF(CFDictionaryCreateMutable(nullptr, 0, &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks));
247
248         auto format = IOSurfaceGetPixelFormat(surface);
249         if (format == kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange || format == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange) {
250
251             // If the width x height isn't a multiple of 16 x 16 and the surface has extra memory in the planes, set pixel buffer attributes to reflect it.
252             auto width = IOSurfaceGetWidth(surface);
253             auto height = IOSurfaceGetHeight(surface);
254             int32_t extendedRight = roundUpToMacroblockMultiple(width) - width;
255             int32_t extendedBottom = roundUpToMacroblockMultiple(height) - height;
256
257             if ((IOSurfaceGetBytesPerRowOfPlane(surface, 0) >= width + extendedRight)
258                 && (IOSurfaceGetBytesPerRowOfPlane(surface, 1) >= width + extendedRight)
259                 && (IOSurfaceGetAllocSize(surface) >= (height + extendedBottom) * IOSurfaceGetBytesPerRowOfPlane(surface, 0) * 3 / 2)) {
260                     auto cfInt = adoptCF(CFNumberCreate(nullptr,  kCFNumberIntType,  &extendedRight));
261                     CFDictionarySetValue(m_bufferAttributes.get(), kCVPixelBufferExtendedPixelsRightKey, cfInt.get());
262                     cfInt = adoptCF(CFNumberCreate(nullptr,  kCFNumberIntType,  &extendedBottom));
263                     CFDictionarySetValue(m_bufferAttributes.get(), kCVPixelBufferExtendedPixelsBottomKey, cfInt.get());
264             }
265         }
266
267         CFDictionarySetValue(m_bufferAttributes.get(), kCVPixelBufferOpenGLCompatibilityKey, kCFBooleanTrue);
268     }
269
270     CVPixelBufferRef pixelBuffer;
271     auto status = CVPixelBufferCreateWithIOSurface(kCFAllocatorDefault, surface, m_bufferAttributes.get(), &pixelBuffer);
272     if (status) {
273         RELEASE_LOG(Media, "Failed to initialize CMVideoFormatDescription with error code: %d", static_cast<int>(status));
274         return nullptr;
275     }
276
277     return adoptCF(pixelBuffer);
278 }
279
280 void ScreenDisplayCaptureSourceMac::generateFrame()
281 {
282     if (!m_currentFrame.ioSurface())
283         return;
284
285     DisplaySurface currentFrame;
286     {
287         LockHolder lock(m_currentFrameMutex);
288         currentFrame = m_currentFrame.ioSurface();
289     }
290
291     auto pixelBuffer = pixelBufferFromIOSurface(currentFrame.ioSurface());
292     if (!pixelBuffer)
293         return;
294
295     auto sampleBuffer = sampleBufferFromPixelBuffer(pixelBuffer.get());
296     if (!sampleBuffer)
297         return;
298
299     videoSampleAvailable(MediaSampleAVFObjC::create(sampleBuffer.get()));
300 }
301
302 void ScreenDisplayCaptureSourceMac::startDisplayStream()
303 {
304     auto actualDisplayID = updateDisplayID(m_displayID);
305     if (!actualDisplayID)
306         return;
307
308     if (m_displayID != actualDisplayID.value()) {
309         m_displayID = actualDisplayID.value();
310         RELEASE_LOG(Media, "ScreenDisplayCaptureSourceMac::create(%p), display ID changed to %d", this, static_cast<int>(m_displayID));
311     }
312
313     if (!m_displayStream && !createDisplayStream())
314         return;
315
316     auto err = CGDisplayStreamStart(m_displayStream.get());
317     if (err) {
318         RELEASE_LOG(Media, "ScreenDisplayCaptureSourceMac::startProducingData(%p), CGDisplayStreamStart failed with error %d", this, static_cast<int>(err));
319         captureFailed();
320         return;
321     }
322
323     m_isRunning = true;
324 }
325
326 bool ScreenDisplayCaptureSourceMac::applySize(const IntSize& newSize)
327 {
328     if (size() == newSize)
329         return true;
330
331     m_bufferAttributes = nullptr;
332     m_displayStream = nullptr;
333     return true;
334 }
335
336 bool ScreenDisplayCaptureSourceMac::applyFrameRate(double rate)
337 {
338     if (frameRate() != rate) {
339         m_bufferAttributes = nullptr;
340         m_displayStream = nullptr;
341     }
342
343     return DisplayCaptureSourceCocoa::applyFrameRate(rate);
344 }
345
346 void ScreenDisplayCaptureSourceMac::commitConfiguration()
347 {
348     if (m_isRunning && !m_displayStream)
349         startDisplayStream();
350 }
351
352 void ScreenDisplayCaptureSourceMac::displayWasReconfigured(CGDirectDisplayID, CGDisplayChangeSummaryFlags)
353 {
354     // FIXME: implement!
355 }
356
357 void ScreenDisplayCaptureSourceMac::displayReconfigurationCallBack(CGDirectDisplayID display, CGDisplayChangeSummaryFlags flags, void *userInfo)
358 {
359     if (userInfo)
360         reinterpret_cast<ScreenDisplayCaptureSourceMac *>(userInfo)->displayWasReconfigured(display, flags);
361 }
362
363 void ScreenDisplayCaptureSourceMac::frameAvailable(CGDisplayStreamFrameStatus status, uint64_t displayTime, IOSurfaceRef frameSurface, CGDisplayStreamUpdateRef updateRef)
364 {
365     switch (status) {
366     case kCGDisplayStreamFrameStatusFrameComplete:
367         break;
368
369     case kCGDisplayStreamFrameStatusFrameIdle:
370         break;
371
372     case kCGDisplayStreamFrameStatusFrameBlank:
373         RELEASE_LOG(Media, "ScreenDisplayCaptureSourceMac::frameAvailable(%p), kCGDisplayStreamFrameStatusFrameBlank", this);
374         break;
375
376     case kCGDisplayStreamFrameStatusStopped:
377         RELEASE_LOG(Media, "ScreenDisplayCaptureSourceMac::frameAvailable(%p), kCGDisplayStreamFrameStatusStopped", this);
378         break;
379     }
380
381     if (!frameSurface || !displayTime)
382         return;
383
384     size_t count;
385     auto* rects = CGDisplayStreamUpdateGetRects(updateRef, kCGDisplayStreamUpdateDirtyRects, &count);
386     if (!rects || !count)
387         return;
388
389     LockHolder lock(m_currentFrameMutex);
390     m_lastFrameTime = monotonicallyIncreasingTime();
391     m_currentFrame = frameSurface;
392 }
393
394 } // namespace WebCore
395
396 #endif // ENABLE(MEDIA_STREAM) && PLATFORM(MAC)