7236a5b5d756a3c46d288ded086f11fc567244aa
[WebKit-https.git] / Source / WebKit / WebProcess / cocoa / UserMediaCaptureManager.cpp
1 /*
2  * Copyright (C) 2017-2018 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
14  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
15  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
17  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
18  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
19  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
20  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
21  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
22  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
23  * THE POSSIBILITY OF SUCH DAMAGE.
24  */
25
26 #include "config.h"
27 #include "UserMediaCaptureManager.h"
28
29 #if PLATFORM(COCOA) && ENABLE(MEDIA_STREAM)
30
31 #include "SharedRingBufferStorage.h"
32 #include "UserMediaCaptureManagerMessages.h"
33 #include "UserMediaCaptureManagerProxyMessages.h"
34 #include "WebCoreArgumentCoders.h"
35 #include "WebProcess.h"
36 #include "WebProcessCreationParameters.h"
37 #include <WebCore/CaptureDevice.h>
38 #include <WebCore/ImageTransferSessionVT.h>
39 #include <WebCore/MediaConstraints.h>
40 #include <WebCore/RealtimeMediaSourceCenter.h>
41 #include <WebCore/RemoteVideoSample.h>
42 #include <WebCore/WebAudioBufferList.h>
43 #include <WebCore/WebAudioSourceProviderAVFObjC.h>
44 #include <wtf/Assertions.h>
45
46 namespace WebKit {
47 using namespace PAL;
48 using namespace WebCore;
49
50 static uint64_t nextSessionID()
51 {
52     static uint64_t nextID = 0;
53     return ++nextID;
54 }
55
56 class UserMediaCaptureManager::Source : public RealtimeMediaSource {
57 public:
58     Source(String&& sourceID, Type type, String&& name, String&& hashSalt, uint64_t id, UserMediaCaptureManager& manager)
59         : RealtimeMediaSource(type, WTFMove(name), WTFMove(sourceID), WTFMove(hashSalt))
60         , m_id(id)
61         , m_manager(manager)
62     {
63         if (type == Type::Audio)
64             m_ringBuffer = std::make_unique<CARingBuffer>(makeUniqueRef<SharedRingBufferStorage>(nullptr));
65     }
66
67     ~Source()
68     {
69         if (type() == Type::Audio)
70             storage().invalidate();
71     }
72
73     SharedRingBufferStorage& storage()
74     {
75         ASSERT(type() == Type::Audio);
76         return static_cast<SharedRingBufferStorage&>(m_ringBuffer->storage());
77     }
78
79     const RealtimeMediaSourceCapabilities& capabilities() final
80     {
81         if (!m_capabilities)
82             m_capabilities = m_manager.capabilities(m_id);
83         return m_capabilities.value();
84     }
85
86     const RealtimeMediaSourceSettings& settings() final { return m_settings; }
87     void setSettings(RealtimeMediaSourceSettings&& settings)
88     {
89         auto changed = m_settings.difference(settings);
90         m_settings = WTFMove(settings);
91         notifySettingsDidChangeObservers(changed);
92     }
93
94     const CAAudioStreamDescription& description() const { return m_description; }
95     void setStorage(const SharedMemory::Handle& handle, const WebCore::CAAudioStreamDescription& description, uint64_t numberOfFrames)
96     {
97         ASSERT(type() == Type::Audio);
98         m_description = description;
99
100         if (handle.isNull()) {
101             m_ringBuffer->deallocate();
102             storage().setReadOnly(false);
103             storage().setStorage(nullptr);
104             return;
105         }
106
107         RefPtr<SharedMemory> memory = SharedMemory::map(handle, SharedMemory::Protection::ReadOnly);
108         storage().setStorage(WTFMove(memory));
109         storage().setReadOnly(true);
110
111         m_ringBuffer->allocate(description, numberOfFrames);
112     }
113
114     void setRingBufferFrameBounds(uint64_t startFrame, uint64_t endFrame)
115     {
116         ASSERT(type() == Type::Audio);
117         m_ringBuffer->setCurrentFrameBounds(startFrame, endFrame);
118     }
119
120     void audioSamplesAvailable(MediaTime time, uint64_t numberOfFrames)
121     {
122         ASSERT(type() == Type::Audio);
123         WebAudioBufferList audioData(m_description, numberOfFrames);
124         m_ringBuffer->fetch(audioData.list(), numberOfFrames, time.timeValue());
125
126         RealtimeMediaSource::audioSamplesAvailable(time, audioData, m_description, numberOfFrames);
127     }
128
129 #if HAVE(IOSURFACE)
130     void remoteVideoSampleAvailable(RemoteVideoSample&& remoteSample)
131     {
132         ASSERT(type() == Type::Video);
133
134         auto videoSampleSize = IntSize(m_settings.width(), m_settings.height());
135         if (videoSampleSize.isEmpty())
136             videoSampleSize = remoteSample.size();
137
138         if (!m_imageTransferSession)
139             m_imageTransferSession = ImageTransferSessionVT::create(remoteSample.videoFormat());
140
141         if (!m_imageTransferSession) {
142             ASSERT_NOT_REACHED();
143             return;
144         }
145
146         auto sampleRef = m_imageTransferSession->createMediaSample(remoteSample.surface(), remoteSample.time(), videoSampleSize);
147         if (!sampleRef) {
148             ASSERT_NOT_REACHED();
149             return;
150         }
151
152         RealtimeMediaSource::videoSampleAvailable(*sampleRef);
153     }
154 #endif
155
156     void applyConstraintsSucceeded(const WebCore::RealtimeMediaSourceSettings& settings)
157     {
158         auto callbacks = m_pendingApplyConstraintsCallbacks.takeFirst();
159         setSettings(WebCore::RealtimeMediaSourceSettings(settings));
160         callbacks.successHandler();
161     }
162
163     void applyConstraintsFailed(const String& failedConstraint, const String& errorMessage)
164     {
165         auto callbacks = m_pendingApplyConstraintsCallbacks.takeFirst();
166         callbacks.failureHandler(failedConstraint, errorMessage);
167     }
168
169 private:
170     void startProducingData() final { m_manager.startProducingData(m_id); }
171     void stopProducingData() final { m_manager.stopProducingData(m_id); }
172     bool isCaptureSource() const final { return true; }
173
174     // RealtimeMediaSource
175     void beginConfiguration() final { }
176     void commitConfiguration() final { }
177
178     void applyConstraints(const WebCore::MediaConstraints& constraints, SuccessHandler&& successHandler, FailureHandler&& failureHandler) final {
179         m_manager.applyConstraints(m_id, constraints);
180         m_pendingApplyConstraintsCallbacks.append({ WTFMove(successHandler), WTFMove(failureHandler)});
181     }
182
183     uint64_t m_id;
184     UserMediaCaptureManager& m_manager;
185     mutable std::optional<RealtimeMediaSourceCapabilities> m_capabilities;
186     RealtimeMediaSourceSettings m_settings;
187
188     CAAudioStreamDescription m_description;
189     std::unique_ptr<CARingBuffer> m_ringBuffer;
190
191     std::unique_ptr<ImageTransferSessionVT> m_imageTransferSession;
192
193     struct ApplyConstraintsCallback {
194         SuccessHandler successHandler;
195         FailureHandler failureHandler;
196     };
197     Deque<ApplyConstraintsCallback> m_pendingApplyConstraintsCallbacks;
198 };
199
200 UserMediaCaptureManager::UserMediaCaptureManager(WebProcess& process)
201     : m_process(process)
202 {
203     m_process.addMessageReceiver(Messages::UserMediaCaptureManager::messageReceiverName(), *this);
204 }
205
206 UserMediaCaptureManager::~UserMediaCaptureManager()
207 {
208     RealtimeMediaSourceCenter::unsetAudioFactory(*this);
209     RealtimeMediaSourceCenter::unsetDisplayCaptureFactory(*this);
210     m_process.removeMessageReceiver(Messages::UserMediaCaptureManager::messageReceiverName());
211 }
212
213 const char* UserMediaCaptureManager::supplementName()
214 {
215     return "UserMediaCaptureManager";
216 }
217
218 void UserMediaCaptureManager::initialize(const WebProcessCreationParameters& parameters)
219 {
220     if (parameters.shouldCaptureAudioInUIProcess)
221         RealtimeMediaSourceCenter::setAudioFactory(*this);
222     if (parameters.shouldCaptureDisplayInUIProcess)
223         RealtimeMediaSourceCenter::setDisplayCaptureFactory(*this);
224 }
225
226 WebCore::CaptureSourceOrError UserMediaCaptureManager::createCaptureSource(const CaptureDevice& device, String&& hashSalt, const WebCore::MediaConstraints* constraints)
227 {
228     if (!constraints)
229         return { };
230
231     uint64_t id = nextSessionID();
232     RealtimeMediaSourceSettings settings;
233     String errorMessage;
234     bool succeeded;
235     if (!m_process.sendSync(Messages::UserMediaCaptureManagerProxy::CreateMediaSourceForCaptureDeviceWithConstraints(id, device, hashSalt, *constraints), Messages::UserMediaCaptureManagerProxy::CreateMediaSourceForCaptureDeviceWithConstraints::Reply(succeeded, errorMessage, settings), 0))
236         return WTFMove(errorMessage);
237
238     auto type = device.type() == CaptureDevice::DeviceType::Microphone ? WebCore::RealtimeMediaSource::Type::Audio : WebCore::RealtimeMediaSource::Type::Video;
239     auto source = adoptRef(*new Source(String::number(id), type, String { settings.label() }, WTFMove(hashSalt), id, *this));
240     source->setSettings(WTFMove(settings));
241     m_sources.set(id, source.copyRef());
242     return WebCore::CaptureSourceOrError(WTFMove(source));
243 }
244
245 void UserMediaCaptureManager::sourceStopped(uint64_t id)
246 {
247     ASSERT(m_sources.contains(id));
248     m_sources.get(id)->stop();
249 }
250
251 void UserMediaCaptureManager::captureFailed(uint64_t id)
252 {
253     ASSERT(m_sources.contains(id));
254     m_sources.get(id)->captureFailed();
255 }
256
257 void UserMediaCaptureManager::sourceMutedChanged(uint64_t id, bool muted)
258 {
259     ASSERT(m_sources.contains(id));
260     m_sources.get(id)->setMuted(muted);
261 }
262
263 void UserMediaCaptureManager::sourceSettingsChanged(uint64_t id, const RealtimeMediaSourceSettings& settings)
264 {
265     ASSERT(m_sources.contains(id));
266     m_sources.get(id)->setSettings(RealtimeMediaSourceSettings(settings));
267 }
268
269 void UserMediaCaptureManager::storageChanged(uint64_t id, const SharedMemory::Handle& handle, const WebCore::CAAudioStreamDescription& description, uint64_t numberOfFrames)
270 {
271     ASSERT(m_sources.contains(id));
272     m_sources.get(id)->setStorage(handle, description, numberOfFrames);
273 }
274
275 void UserMediaCaptureManager::ringBufferFrameBoundsChanged(uint64_t id, uint64_t startFrame, uint64_t endFrame)
276 {
277     ASSERT(m_sources.contains(id));
278     m_sources.get(id)->setRingBufferFrameBounds(startFrame, endFrame);
279 }
280
281 void UserMediaCaptureManager::audioSamplesAvailable(uint64_t id, MediaTime time, uint64_t numberOfFrames, uint64_t startFrame, uint64_t endFrame)
282 {
283     ASSERT(m_sources.contains(id));
284     auto& source = *m_sources.get(id);
285     source.setRingBufferFrameBounds(startFrame, endFrame);
286     source.audioSamplesAvailable(time, numberOfFrames);
287 }
288
289 #if HAVE(IOSURFACE)
290 void UserMediaCaptureManager::remoteVideoSampleAvailable(uint64_t id, RemoteVideoSample&& sample)
291 {
292     ASSERT(m_sources.contains(id));
293     m_sources.get(id)->remoteVideoSampleAvailable(WTFMove(sample));
294 }
295 #else
296 NO_RETURN_DUE_TO_ASSERT void UserMediaCaptureManager::remoteVideoSampleAvailable(uint64_t, RemoteVideoSample&&)
297 {
298     ASSERT_NOT_REACHED();
299 }
300 #endif
301
302 void UserMediaCaptureManager::startProducingData(uint64_t id)
303 {
304     m_process.send(Messages::UserMediaCaptureManagerProxy::StartProducingData(id), 0);
305 }
306
307 void UserMediaCaptureManager::stopProducingData(uint64_t id)
308 {
309     m_process.send(Messages::UserMediaCaptureManagerProxy::StopProducingData(id), 0);
310 }
311
312 WebCore::RealtimeMediaSourceCapabilities UserMediaCaptureManager::capabilities(uint64_t id)
313 {
314     WebCore::RealtimeMediaSourceCapabilities capabilities;
315     m_process.sendSync(Messages::UserMediaCaptureManagerProxy::Capabilities(id), Messages::UserMediaCaptureManagerProxy::Capabilities::Reply(capabilities), 0);
316     return capabilities;
317 }
318
319 void UserMediaCaptureManager::setMuted(uint64_t id, bool muted)
320 {
321     m_process.send(Messages::UserMediaCaptureManagerProxy::SetMuted(id, muted), 0);
322 }
323
324 void UserMediaCaptureManager::applyConstraints(uint64_t id, const WebCore::MediaConstraints& constraints)
325 {
326     m_process.send(Messages::UserMediaCaptureManagerProxy::ApplyConstraints(id, constraints), 0);
327 }
328
329 void UserMediaCaptureManager::applyConstraintsSucceeded(uint64_t id, const WebCore::RealtimeMediaSourceSettings& settings)
330 {
331     ASSERT(m_sources.contains(id));
332     auto& source = *m_sources.get(id);
333     source.applyConstraintsSucceeded(settings);
334 }
335
336 void UserMediaCaptureManager::applyConstraintsFailed(uint64_t id, const String& failedConstraint, const String& message)
337 {
338     ASSERT(m_sources.contains(id));
339     auto& source = *m_sources.get(id);
340     source.applyConstraintsFailed(failedConstraint, message);
341 }
342
343 }
344
345 #endif