[MediaStream] Add Mac screen capture source
[WebKit-https.git] / Source / WebCore / platform / mediastream / mac / ScreenDisplayCaptureSourceMac.mm
1 /*
2  * Copyright (C) 2017 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
17  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24  */
25
26 #include "config.h"
27 #include "ScreenDisplayCaptureSourceMac.h"
28
29 #if ENABLE(MEDIA_STREAM) && PLATFORM(MAC)
30
31 #include "GraphicsContextCG.h"
32 #include "ImageBuffer.h"
33 #include "Logging.h"
34 #include "MediaConstraints.h"
35 #include "MediaSampleAVFObjC.h"
36 #include "NotImplemented.h"
37 #include "PlatformLayer.h"
38 #include "RealtimeMediaSourceSettings.h"
39
40 #include "CoreVideoSoftLink.h"
41
42 extern "C" {
43 size_t CGDisplayModeGetPixelsWide(CGDisplayModeRef);
44 size_t CGDisplayModeGetPixelsHigh(CGDisplayModeRef);
45 }
46
47 namespace WebCore {
48
49 static int32_t roundUpToMacroblockMultiple(int32_t size)
50 {
51     return (size + 15) & ~15;
52 }
53
54 std::optional<CGDirectDisplayID> ScreenDisplayCaptureSourceMac::updateDisplayID(CGDirectDisplayID displayID)
55 {
56     uint32_t displayCount = 0;
57     auto err = CGGetActiveDisplayList(0, nullptr, &displayCount);
58     if (err) {
59         RELEASE_LOG(Media, "CGGetActiveDisplayList() returned error %d when trying to get display count", static_cast<int>(err));
60         return std::nullopt;
61     }
62
63     if (!displayCount) {
64         RELEASE_LOG(Media, "CGGetActiveDisplayList() returned a display count of 0");
65         return std::nullopt;
66     }
67
68     CGDirectDisplayID activeDisplays[displayCount];
69     err = CGGetActiveDisplayList(displayCount, &(activeDisplays[0]), &displayCount);
70     if (err) {
71         RELEASE_LOG(Media, "CGGetActiveDisplayList() returned error %d when trying to get the active display list", static_cast<int>(err));
72         return std::nullopt;
73     }
74
75     auto displayMask = CGDisplayIDToOpenGLDisplayMask(displayID);
76     for (auto display : activeDisplays) {
77         if (displayMask == CGDisplayIDToOpenGLDisplayMask(display))
78             return display;
79     }
80
81     return std::nullopt;
82 }
83
84 CaptureSourceOrError ScreenDisplayCaptureSourceMac::create(const String& deviceID, const MediaConstraints* constraints)
85 {
86     bool ok;
87     auto displayID = deviceID.toUIntStrict(&ok);
88     if (!ok) {
89         RELEASE_LOG(Media, "Display ID does not convert to 32-bit integer");
90         return { };
91     }
92
93     auto actualDisplayID = updateDisplayID(displayID);
94     if (!actualDisplayID)
95         return { };
96
97     auto source = adoptRef(*new ScreenDisplayCaptureSourceMac(actualDisplayID.value()));
98     if (constraints && source->applyConstraints(*constraints))
99         return { };
100
101     return CaptureSourceOrError(WTFMove(source));
102 }
103
104 ScreenDisplayCaptureSourceMac::ScreenDisplayCaptureSourceMac(uint32_t displayID)
105     : DisplayCaptureSourceCocoa("Screen")
106     , m_displayID(displayID)
107 {
108 }
109
110 ScreenDisplayCaptureSourceMac::~ScreenDisplayCaptureSourceMac()
111 {
112     if (m_observingDisplayChanges)
113         CGDisplayRemoveReconfigurationCallback(displayReconfigurationCallBack, this);
114
115     m_currentFrame = nullptr;
116 }
117
118 bool ScreenDisplayCaptureSourceMac::createDisplayStream()
119 {
120     static const int screenQueueMaximumLength = 6;
121
122     auto actualDisplayID = updateDisplayID(m_displayID);
123     if (!actualDisplayID) {
124         captureFailed();
125         return false;
126     }
127
128     if (m_displayID != actualDisplayID.value()) {
129         m_displayID = actualDisplayID.value();
130         RELEASE_LOG(Media, "ScreenDisplayCaptureSourceMac::create(%p), display ID changed to %d", this, static_cast<int>(m_displayID));
131     }
132
133     if (!m_displayStream) {
134
135         if (size().isEmpty()) {
136             CGDisplayModeRef displayMode = CGDisplayCopyDisplayMode(m_displayID);
137             auto screenWidth = CGDisplayModeGetPixelsWide(displayMode);
138             auto screenHeight = CGDisplayModeGetPixelsHigh(displayMode);
139             if (!screenWidth || !screenHeight) {
140                 RELEASE_LOG(Media, "ScreenDisplayCaptureSourceMac::createDisplayStream(%p), unable to get screen width/height", this);
141                 captureFailed();
142                 return false;
143             }
144             setWidth(screenWidth);
145             setHeight(screenHeight);
146             CGDisplayModeRelease(displayMode);
147         }
148
149         if (!m_captureQueue)
150             m_captureQueue = adoptOSObject(dispatch_queue_create("ScreenDisplayCaptureSourceMac Capture Queue", DISPATCH_QUEUE_SERIAL));
151
152         static CGColorSpaceRef deviceRGBColorSpace = CGColorSpaceCreateDeviceRGB();
153         double frameTime = 1 / frameRate();
154         auto frameTimeCF = adoptCF(CFNumberCreate(nullptr,  kCFNumberDoubleType,  &frameTime));
155         int depth = screenQueueMaximumLength;
156         auto depthCF = adoptCF(CFNumberCreate(nullptr,  kCFNumberIntType,  &depth));
157         CFTypeRef keys[] = {
158             kCGDisplayStreamMinimumFrameTime,
159             kCGDisplayStreamQueueDepth,
160             kCGDisplayStreamColorSpace,
161             kCGDisplayStreamShowCursor,
162         };
163         CFTypeRef values[] = {
164             frameTimeCF.get(),
165             depthCF.get(),
166             deviceRGBColorSpace,
167             kCFBooleanTrue,
168         };
169         auto streamOptions = adoptCF(CFDictionaryCreate(kCFAllocatorDefault, keys, values, WTF_ARRAY_LENGTH(keys), &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks));
170
171         auto weakThis = m_weakFactory.createWeakPtr(*this);
172         m_frameAvailableBlock = Block_copy(^(CGDisplayStreamFrameStatus status, uint64_t displayTime, IOSurfaceRef frameSurface, CGDisplayStreamUpdateRef updateRef) {
173             if (!weakThis)
174                 return;
175
176             weakThis->frameAvailable(status, displayTime, frameSurface, updateRef);
177         });
178
179         m_displayStream = adoptCF(CGDisplayStreamCreateWithDispatchQueue(m_displayID, size().width(), size().height(), kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, streamOptions.get(), m_captureQueue.get(), m_frameAvailableBlock));
180         if (!m_displayStream) {
181             RELEASE_LOG(Media, "ScreenDisplayCaptureSourceMac::createDisplayStream(%p), CGDisplayStreamCreate failed", this);
182             captureFailed();
183             return false;
184         }
185     }
186
187     if (!m_observingDisplayChanges) {
188         CGDisplayRegisterReconfigurationCallback(displayReconfigurationCallBack, this);
189         m_observingDisplayChanges = true;
190     }
191
192     return true;
193 }
194
195 void ScreenDisplayCaptureSourceMac::startProducingData()
196 {
197     DisplayCaptureSourceCocoa::startProducingData();
198
199     if (m_isRunning)
200         return;
201
202     startDisplayStream();
203 }
204
205 void ScreenDisplayCaptureSourceMac::stopProducingData()
206 {
207     DisplayCaptureSourceCocoa::stopProducingData();
208
209     if (!m_isRunning)
210         return;
211
212     if (m_displayStream)
213         CGDisplayStreamStop(m_displayStream.get());
214
215     m_isRunning = false;
216 }
217
218 RetainPtr<CMSampleBufferRef> ScreenDisplayCaptureSourceMac::sampleBufferFromPixelBuffer(CVPixelBufferRef pixelBuffer)
219 {
220     if (!pixelBuffer)
221         return nullptr;
222
223     CMTime sampleTime = CMTimeMake((elapsedTime() + .1) * 100, 100);
224     CMSampleTimingInfo timingInfo = { kCMTimeInvalid, sampleTime, sampleTime };
225
226     CMVideoFormatDescriptionRef formatDescription = nullptr;
227     auto status = CMVideoFormatDescriptionCreateForImageBuffer(kCFAllocatorDefault, (CVImageBufferRef)pixelBuffer, &formatDescription);
228     if (status) {
229         RELEASE_LOG(Media, "Failed to initialize CMVideoFormatDescription with error code: %d", static_cast<int>(status));
230         return nullptr;
231     }
232
233     CMSampleBufferRef sampleBuffer;
234     status = CMSampleBufferCreateReadyWithImageBuffer(kCFAllocatorDefault, (CVImageBufferRef)pixelBuffer, formatDescription, &timingInfo, &sampleBuffer);
235     CFRelease(formatDescription);
236     if (status) {
237         RELEASE_LOG(Media, "Failed to initialize CMSampleBuffer with error code: %d", static_cast<int>(status));
238         return nullptr;
239     }
240
241     return adoptCF(sampleBuffer);
242 }
243
244 RetainPtr<CVPixelBufferRef> ScreenDisplayCaptureSourceMac::pixelBufferFromIOSurface(IOSurfaceRef surface)
245 {
246     if (!m_bufferAttributes) {
247         m_bufferAttributes = adoptCF(CFDictionaryCreateMutable(nullptr, 0, &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks));
248
249         auto format = IOSurfaceGetPixelFormat(surface);
250         if (format == kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange || format == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange) {
251
252             // If the width x height isn't a multiple of 16 x 16 and the surface has extra memory in the planes, set pixel buffer attributes to reflect it.
253             auto width = IOSurfaceGetWidth(surface);
254             auto height = IOSurfaceGetHeight(surface);
255             int32_t extendedRight = roundUpToMacroblockMultiple(width) - width;
256             int32_t extendedBottom = roundUpToMacroblockMultiple(height) - height;
257
258             if ((IOSurfaceGetBytesPerRowOfPlane(surface, 0) >= width + extendedRight)
259                 && (IOSurfaceGetBytesPerRowOfPlane(surface, 1) >= width + extendedRight)
260                 && (IOSurfaceGetAllocSize(surface) >= (height + extendedBottom) * IOSurfaceGetBytesPerRowOfPlane(surface, 0) * 3 / 2)) {
261                     auto cfInt = adoptCF(CFNumberCreate(nullptr,  kCFNumberIntType,  &extendedRight));
262                     CFDictionarySetValue(m_bufferAttributes.get(), kCVPixelBufferExtendedPixelsRightKey, cfInt.get());
263                     cfInt = adoptCF(CFNumberCreate(nullptr,  kCFNumberIntType,  &extendedBottom));
264                     CFDictionarySetValue(m_bufferAttributes.get(), kCVPixelBufferExtendedPixelsBottomKey, cfInt.get());
265             }
266         }
267
268         CFDictionarySetValue(m_bufferAttributes.get(), kCVPixelBufferOpenGLCompatibilityKey, kCFBooleanTrue);
269     }
270
271     CVPixelBufferRef pixelBuffer;
272     auto status = CVPixelBufferCreateWithIOSurface(kCFAllocatorDefault, surface, m_bufferAttributes.get(), &pixelBuffer);
273     if (status) {
274         RELEASE_LOG(Media, "Failed to initialize CMVideoFormatDescription with error code: %d", static_cast<int>(status));
275         return nullptr;
276     }
277
278     return adoptCF(pixelBuffer);
279 }
280
281 void ScreenDisplayCaptureSourceMac::generateFrame()
282 {
283     if (!m_currentFrame.ioSurface())
284         return;
285
286     DisplaySurface currentFrame;
287     {
288         LockHolder lock(m_currentFrameMutex);
289         currentFrame = m_currentFrame.ioSurface();
290     }
291
292     auto pixelBuffer = pixelBufferFromIOSurface(currentFrame.ioSurface());
293     if (!pixelBuffer)
294         return;
295
296     auto sampleBuffer = sampleBufferFromPixelBuffer(pixelBuffer.get());
297     if (!sampleBuffer)
298         return;
299
300     videoSampleAvailable(MediaSampleAVFObjC::create(sampleBuffer.get()));
301 }
302
303 void ScreenDisplayCaptureSourceMac::startDisplayStream()
304 {
305     auto actualDisplayID = updateDisplayID(m_displayID);
306     if (!actualDisplayID)
307         return;
308
309     if (m_displayID != actualDisplayID.value()) {
310         m_displayID = actualDisplayID.value();
311         RELEASE_LOG(Media, "ScreenDisplayCaptureSourceMac::create(%p), display ID changed to %d", this, static_cast<int>(m_displayID));
312     }
313
314     if (!m_displayStream && !createDisplayStream())
315         return;
316
317     auto err = CGDisplayStreamStart(m_displayStream.get());
318     if (err) {
319         RELEASE_LOG(Media, "ScreenDisplayCaptureSourceMac::startProducingData(%p), CGDisplayStreamStart failed with error %d", this, static_cast<int>(err));
320         captureFailed();
321         return;
322     }
323
324     m_isRunning = true;
325 }
326
327 bool ScreenDisplayCaptureSourceMac::applySize(const IntSize& newSize)
328 {
329     if (size() == newSize)
330         return true;
331
332     m_bufferAttributes = nullptr;
333     m_displayStream = nullptr;
334     return true;
335 }
336
337 bool ScreenDisplayCaptureSourceMac::applyFrameRate(double rate)
338 {
339     if (frameRate() != rate) {
340         m_bufferAttributes = nullptr;
341         m_displayStream = nullptr;
342     }
343
344     return DisplayCaptureSourceCocoa::applyFrameRate(rate);
345 }
346
347 void ScreenDisplayCaptureSourceMac::commitConfiguration()
348 {
349     if (m_isRunning && !m_displayStream)
350         startDisplayStream();
351 }
352
353 void ScreenDisplayCaptureSourceMac::displayWasReconfigured(CGDirectDisplayID, CGDisplayChangeSummaryFlags)
354 {
355     // FIXME: implement!
356 }
357
358 void ScreenDisplayCaptureSourceMac::displayReconfigurationCallBack(CGDirectDisplayID display, CGDisplayChangeSummaryFlags flags, void *userInfo)
359 {
360     if (userInfo)
361         reinterpret_cast<ScreenDisplayCaptureSourceMac *>(userInfo)->displayWasReconfigured(display, flags);
362 }
363
364 void ScreenDisplayCaptureSourceMac::frameAvailable(CGDisplayStreamFrameStatus status, uint64_t displayTime, IOSurfaceRef frameSurface, CGDisplayStreamUpdateRef updateRef)
365 {
366     switch (status) {
367     case kCGDisplayStreamFrameStatusFrameComplete:
368         break;
369
370     case kCGDisplayStreamFrameStatusFrameIdle:
371         break;
372
373     case kCGDisplayStreamFrameStatusFrameBlank:
374         RELEASE_LOG(Media, "ScreenDisplayCaptureSourceMac::frameAvailable(%p), kCGDisplayStreamFrameStatusFrameBlank", this);
375         break;
376
377     case kCGDisplayStreamFrameStatusStopped:
378         RELEASE_LOG(Media, "ScreenDisplayCaptureSourceMac::frameAvailable(%p), kCGDisplayStreamFrameStatusStopped", this);
379         break;
380     }
381
382     if (!frameSurface || !displayTime)
383         return;
384
385     size_t count;
386     auto* rects = CGDisplayStreamUpdateGetRects(updateRef, kCGDisplayStreamUpdateDirtyRects, &count);
387     if (!rects || !count)
388         return;
389
390     LockHolder lock(m_currentFrameMutex);
391     m_lastFrameTime = monotonicallyIncreasingTime();
392     m_currentFrame = frameSurface;
393 }
394
395 } // namespace WebCore
396
397 #endif // ENABLE(MEDIA_STREAM) && PLATFORM(MAC)