[WTF] Add makeUnique<T>, which ensures T is fast-allocated, makeUnique / makeUniqueWi...
[WebKit-https.git] / Source / WebKit / WebProcess / cocoa / UserMediaCaptureManager.cpp
1 /*
2  * Copyright (C) 2017-2018 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
14  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
15  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
17  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
18  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
19  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
20  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
21  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
22  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
23  * THE POSSIBILITY OF SUCH DAMAGE.
24  */
25
26 #include "config.h"
27 #include "UserMediaCaptureManager.h"
28
29 #if PLATFORM(COCOA) && ENABLE(MEDIA_STREAM)
30
31 #include "SharedRingBufferStorage.h"
32 #include "UserMediaCaptureManagerMessages.h"
33 #include "UserMediaCaptureManagerProxyMessages.h"
34 #include "WebCoreArgumentCoders.h"
35 #include "WebProcess.h"
36 #include "WebProcessCreationParameters.h"
37 #include <WebCore/CaptureDevice.h>
38 #include <WebCore/ImageTransferSessionVT.h>
39 #include <WebCore/MediaConstraints.h>
40 #include <WebCore/MockRealtimeMediaSourceCenter.h>
41 #include <WebCore/RealtimeMediaSourceCenter.h>
42 #include <WebCore/RemoteVideoSample.h>
43 #include <WebCore/WebAudioBufferList.h>
44 #include <WebCore/WebAudioSourceProviderAVFObjC.h>
45 #include <wtf/Assertions.h>
46
47 namespace WebKit {
48 using namespace PAL;
49 using namespace WebCore;
50
51 static uint64_t nextSessionID()
52 {
53     static uint64_t nextID = 0;
54     return ++nextID;
55 }
56
57 class UserMediaCaptureManager::Source : public RealtimeMediaSource {
58 public:
59     Source(String&& sourceID, Type type, CaptureDevice::DeviceType deviceType, String&& name, String&& hashSalt, uint64_t id, UserMediaCaptureManager& manager)
60         : RealtimeMediaSource(type, WTFMove(name), WTFMove(sourceID), WTFMove(hashSalt))
61         , m_id(id)
62         , m_manager(manager)
63         , m_deviceType(deviceType)
64     {
65         ASSERT(deviceType != CaptureDevice::DeviceType::Unknown);
66         if (type == Type::Audio)
67             m_ringBuffer = makeUnique<CARingBuffer>(makeUniqueRef<SharedRingBufferStorage>(nullptr));
68     }
69
70     ~Source()
71     {
72         if (type() == Type::Audio)
73             storage().invalidate();
74     }
75
76     SharedRingBufferStorage& storage()
77     {
78         ASSERT(type() == Type::Audio);
79         return static_cast<SharedRingBufferStorage&>(m_ringBuffer->storage());
80     }
81
82     const RealtimeMediaSourceCapabilities& capabilities() final
83     {
84         if (!m_capabilities)
85             m_capabilities = m_manager.capabilities(m_id);
86         return m_capabilities.value();
87     }
88
89     const RealtimeMediaSourceSettings& settings() final { return m_settings; }
90     void setSettings(RealtimeMediaSourceSettings&& settings)
91     {
92         auto changed = m_settings.difference(settings);
93         m_settings = WTFMove(settings);
94         notifySettingsDidChangeObservers(changed);
95     }
96
97     const CAAudioStreamDescription& description() const { return m_description; }
98     void setStorage(const SharedMemory::Handle& handle, const WebCore::CAAudioStreamDescription& description, uint64_t numberOfFrames)
99     {
100         ASSERT(type() == Type::Audio);
101         m_description = description;
102
103         if (handle.isNull()) {
104             m_ringBuffer->deallocate();
105             storage().setReadOnly(false);
106             storage().setStorage(nullptr);
107             return;
108         }
109
110         RefPtr<SharedMemory> memory = SharedMemory::map(handle, SharedMemory::Protection::ReadOnly);
111         storage().setStorage(WTFMove(memory));
112         storage().setReadOnly(true);
113
114         m_ringBuffer->allocate(description, numberOfFrames);
115     }
116
117     void setRingBufferFrameBounds(uint64_t startFrame, uint64_t endFrame)
118     {
119         ASSERT(type() == Type::Audio);
120         m_ringBuffer->setCurrentFrameBounds(startFrame, endFrame);
121     }
122
123     void audioSamplesAvailable(MediaTime time, uint64_t numberOfFrames)
124     {
125         ASSERT(type() == Type::Audio);
126         WebAudioBufferList audioData(m_description, numberOfFrames);
127         m_ringBuffer->fetch(audioData.list(), numberOfFrames, time.timeValue());
128
129         RealtimeMediaSource::audioSamplesAvailable(time, audioData, m_description, numberOfFrames);
130     }
131
132 #if HAVE(IOSURFACE)
133     void remoteVideoSampleAvailable(RemoteVideoSample&& remoteSample)
134     {
135         ASSERT(type() == Type::Video);
136
137         auto remoteSampleSize = remoteSample.size();
138         setIntrinsicSize(remoteSampleSize);
139
140         auto videoSampleSize = IntSize(m_settings.width(), m_settings.height());
141         if (videoSampleSize.isZero())
142             videoSampleSize = remoteSampleSize;
143         else if (!videoSampleSize.height())
144             videoSampleSize.setHeight(videoSampleSize.width() * (remoteSampleSize.height() / static_cast<double>(remoteSampleSize.width())));
145         else if (!videoSampleSize.width())
146             videoSampleSize.setWidth(videoSampleSize.height() * (remoteSampleSize.width() / static_cast<double>(remoteSampleSize.height())));
147
148         if (!m_imageTransferSession || m_imageTransferSession->pixelFormat() != remoteSample.videoFormat())
149             m_imageTransferSession = ImageTransferSessionVT::create(remoteSample.videoFormat());
150
151         if (!m_imageTransferSession) {
152             ASSERT_NOT_REACHED();
153             return;
154         }
155
156         auto sampleRef = m_imageTransferSession->createMediaSample(remoteSample.surface(), remoteSample.time(), videoSampleSize);
157         if (!sampleRef) {
158             ASSERT_NOT_REACHED();
159             return;
160         }
161
162         RealtimeMediaSource::videoSampleAvailable(*sampleRef);
163     }
164 #endif
165
166     void applyConstraintsSucceeded(const WebCore::RealtimeMediaSourceSettings& settings)
167     {
168         setSettings(WebCore::RealtimeMediaSourceSettings(settings));
169
170         auto callback = m_pendingApplyConstraintsCallbacks.takeFirst();
171         callback({ });
172     }
173
174     void applyConstraintsFailed(String&& failedConstraint, String&& errorMessage)
175     {
176         auto callback = m_pendingApplyConstraintsCallbacks.takeFirst();
177         callback(ApplyConstraintsError { WTFMove(failedConstraint), WTFMove(errorMessage) });
178     }
179
180 private:
181     void startProducingData() final { m_manager.startProducingData(m_id); }
182     void stopProducingData() final { m_manager.stopProducingData(m_id); }
183     bool isCaptureSource() const final { return true; }
184     CaptureDevice::DeviceType deviceType() const final { return m_deviceType; }
185
186     // RealtimeMediaSource
187     void beginConfiguration() final { }
188     void commitConfiguration() final { }
189     void hasEnded() final { m_manager.sourceEnded(m_id); }
190
191     void applyConstraints(const WebCore::MediaConstraints& constraints, ApplyConstraintsHandler&& completionHandler) final
192     {
193         m_manager.applyConstraints(m_id, constraints);
194         m_pendingApplyConstraintsCallbacks.append(WTFMove(completionHandler));
195     }
196
197     uint64_t m_id;
198     UserMediaCaptureManager& m_manager;
199     mutable Optional<RealtimeMediaSourceCapabilities> m_capabilities;
200     RealtimeMediaSourceSettings m_settings;
201
202     CAAudioStreamDescription m_description;
203     std::unique_ptr<CARingBuffer> m_ringBuffer;
204
205     std::unique_ptr<ImageTransferSessionVT> m_imageTransferSession;
206     CaptureDevice::DeviceType m_deviceType { CaptureDevice::DeviceType::Unknown };
207
208     Deque<ApplyConstraintsHandler> m_pendingApplyConstraintsCallbacks;
209 };
210
211 UserMediaCaptureManager::UserMediaCaptureManager(WebProcess& process)
212     : m_process(process)
213 {
214     m_process.addMessageReceiver(Messages::UserMediaCaptureManager::messageReceiverName(), *this);
215 }
216
217 UserMediaCaptureManager::~UserMediaCaptureManager()
218 {
219     RealtimeMediaSourceCenter::singleton().unsetAudioCaptureFactory(*this);
220     RealtimeMediaSourceCenter::singleton().unsetDisplayCaptureFactory(*this);
221     RealtimeMediaSourceCenter::singleton().unsetVideoCaptureFactory(*this);
222     m_process.removeMessageReceiver(Messages::UserMediaCaptureManager::messageReceiverName());
223 }
224
225 const char* UserMediaCaptureManager::supplementName()
226 {
227     return "UserMediaCaptureManager";
228 }
229
230 void UserMediaCaptureManager::initialize(const WebProcessCreationParameters& parameters)
231 {
232     MockRealtimeMediaSourceCenter::singleton().setMockAudioCaptureEnabled(!parameters.shouldCaptureAudioInUIProcess);
233     MockRealtimeMediaSourceCenter::singleton().setMockVideoCaptureEnabled(!parameters.shouldCaptureVideoInUIProcess);
234     MockRealtimeMediaSourceCenter::singleton().setMockDisplayCaptureEnabled(!parameters.shouldCaptureDisplayInUIProcess);
235
236     if (parameters.shouldCaptureAudioInUIProcess)
237         RealtimeMediaSourceCenter::singleton().setAudioCaptureFactory(*this);
238     if (parameters.shouldCaptureVideoInUIProcess)
239         RealtimeMediaSourceCenter::singleton().setVideoCaptureFactory(*this);
240     if (parameters.shouldCaptureDisplayInUIProcess)
241         RealtimeMediaSourceCenter::singleton().setDisplayCaptureFactory(*this);
242 }
243
244 WebCore::CaptureSourceOrError UserMediaCaptureManager::createCaptureSource(const CaptureDevice& device, String&& hashSalt, const WebCore::MediaConstraints* constraints)
245 {
246     if (!constraints)
247         return { };
248
249     uint64_t id = nextSessionID();
250     RealtimeMediaSourceSettings settings;
251     String errorMessage;
252     bool succeeded;
253     if (!m_process.sendSync(Messages::UserMediaCaptureManagerProxy::CreateMediaSourceForCaptureDeviceWithConstraints(id, device, hashSalt, *constraints), Messages::UserMediaCaptureManagerProxy::CreateMediaSourceForCaptureDeviceWithConstraints::Reply(succeeded, errorMessage, settings), 0))
254         return WTFMove(errorMessage);
255
256     auto type = device.type() == CaptureDevice::DeviceType::Microphone ? WebCore::RealtimeMediaSource::Type::Audio : WebCore::RealtimeMediaSource::Type::Video;
257     auto source = adoptRef(*new Source(String::number(id), type, device.type(), String { settings.label() }, WTFMove(hashSalt), id, *this));
258     source->setSettings(WTFMove(settings));
259     m_sources.add(id, source.copyRef());
260     return WebCore::CaptureSourceOrError(WTFMove(source));
261 }
262
263 void UserMediaCaptureManager::sourceStopped(uint64_t id)
264 {
265     if (auto source = m_sources.get(id)) {
266         source->stop();
267         sourceEnded(id);
268     }
269 }
270
271 void UserMediaCaptureManager::captureFailed(uint64_t id)
272 {
273     if (auto source = m_sources.get(id)) {
274         source->captureFailed();
275         sourceEnded(id);
276     }
277 }
278
279 void UserMediaCaptureManager::sourceMutedChanged(uint64_t id, bool muted)
280 {
281     if (auto source = m_sources.get(id))
282         source->setMuted(muted);
283 }
284
285 void UserMediaCaptureManager::sourceSettingsChanged(uint64_t id, const RealtimeMediaSourceSettings& settings)
286 {
287     if (auto source = m_sources.get(id))
288         source->setSettings(RealtimeMediaSourceSettings(settings));
289 }
290
291 void UserMediaCaptureManager::storageChanged(uint64_t id, const SharedMemory::Handle& handle, const WebCore::CAAudioStreamDescription& description, uint64_t numberOfFrames)
292 {
293     if (auto source = m_sources.get(id))
294         source->setStorage(handle, description, numberOfFrames);
295 }
296
297 void UserMediaCaptureManager::ringBufferFrameBoundsChanged(uint64_t id, uint64_t startFrame, uint64_t endFrame)
298 {
299     if (auto source = m_sources.get(id))
300         source->setRingBufferFrameBounds(startFrame, endFrame);
301 }
302
303 void UserMediaCaptureManager::audioSamplesAvailable(uint64_t id, MediaTime time, uint64_t numberOfFrames, uint64_t startFrame, uint64_t endFrame)
304 {
305     if (auto source = m_sources.get(id)) {
306         source->setRingBufferFrameBounds(startFrame, endFrame);
307         source->audioSamplesAvailable(time, numberOfFrames);
308     }
309 }
310
311 #if HAVE(IOSURFACE)
312 void UserMediaCaptureManager::remoteVideoSampleAvailable(uint64_t id, RemoteVideoSample&& sample)
313 {
314     if (auto source = m_sources.get(id))
315         source->remoteVideoSampleAvailable(WTFMove(sample));
316 }
317 #else
318 NO_RETURN_DUE_TO_ASSERT void UserMediaCaptureManager::remoteVideoSampleAvailable(uint64_t, RemoteVideoSample&&)
319 {
320     ASSERT_NOT_REACHED();
321 }
322 #endif
323
324 void UserMediaCaptureManager::startProducingData(uint64_t id)
325 {
326     m_process.send(Messages::UserMediaCaptureManagerProxy::StartProducingData(id), 0);
327 }
328
329 void UserMediaCaptureManager::stopProducingData(uint64_t id)
330 {
331     m_process.send(Messages::UserMediaCaptureManagerProxy::StopProducingData(id), 0);
332 }
333
334 WebCore::RealtimeMediaSourceCapabilities UserMediaCaptureManager::capabilities(uint64_t id)
335 {
336     WebCore::RealtimeMediaSourceCapabilities capabilities;
337     m_process.sendSync(Messages::UserMediaCaptureManagerProxy::Capabilities(id), Messages::UserMediaCaptureManagerProxy::Capabilities::Reply(capabilities), 0);
338     return capabilities;
339 }
340
341 void UserMediaCaptureManager::setMuted(uint64_t id, bool muted)
342 {
343     m_process.send(Messages::UserMediaCaptureManagerProxy::SetMuted(id, muted), 0);
344 }
345
346 void UserMediaCaptureManager::applyConstraints(uint64_t id, const WebCore::MediaConstraints& constraints)
347 {
348     m_process.send(Messages::UserMediaCaptureManagerProxy::ApplyConstraints(id, constraints), 0);
349 }
350
351 void UserMediaCaptureManager::sourceEnded(uint64_t id)
352 {
353     m_process.send(Messages::UserMediaCaptureManagerProxy::End(id), 0);
354     m_sources.remove(id);
355 }
356
357 void UserMediaCaptureManager::applyConstraintsSucceeded(uint64_t id, const WebCore::RealtimeMediaSourceSettings& settings)
358 {
359     if (auto source = m_sources.get(id))
360         source->applyConstraintsSucceeded(settings);
361 }
362
363 void UserMediaCaptureManager::applyConstraintsFailed(uint64_t id, String&& failedConstraint, String&& message)
364 {
365     if (auto source = m_sources.get(id))
366         source->applyConstraintsFailed(WTFMove(failedConstraint), WTFMove(message));
367 }
368
369 #if PLATFORM(IOS_FAMILY)
370 void UserMediaCaptureManager::setAudioCapturePageState(bool interrupted, bool pageMuted)
371 {
372     if (auto* activeSource = static_cast<AudioCaptureFactory*>(this)->activeSource())
373         activeSource->setInterrupted(interrupted, pageMuted);
374 }
375
376 void UserMediaCaptureManager::setVideoCapturePageState(bool interrupted, bool pageMuted)
377 {
378     if (auto* activeSource = static_cast<VideoCaptureFactory*>(this)->activeSource())
379         activeSource->setInterrupted(interrupted, pageMuted);
380 }
381 #endif
382
383 }
384
385 #endif