63432878258b752439ea053808caafcd197970aa
[WebKit-https.git] / Source / WebCore / platform / mediarecorder / cocoa / MediaRecorderPrivateWriterCocoa.mm
1 /*
2  * Copyright (C) 2018 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
14  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
15  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
17  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
18  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
19  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
20  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
21  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
22  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
23  * THE POSSIBILITY OF SUCH DAMAGE.
24  */
25
26 #include "config.h"
27 #include "MediaRecorderPrivateWriterCocoa.h"
28
29 #if ENABLE(MEDIA_STREAM) && USE(AVFOUNDATION)
30
31 #include "AudioStreamDescription.h"
32 #include "FileSystem.h"
33 #include "Logging.h"
34 #include "WebAudioBufferList.h"
35 #include <AVFoundation/AVAssetWriter.h>
36 #include <AVFoundation/AVAssetWriterInput.h>
37 #include <pal/cf/CoreMediaSoftLink.h>
38
39 typedef AVAssetWriter AVAssetWriterType;
40 typedef AVAssetWriterInput AVAssetWriterInputType;
41
42 SOFT_LINK_FRAMEWORK_OPTIONAL(AVFoundation)
43
44 SOFT_LINK_CLASS(AVFoundation, AVAssetWriter)
45 SOFT_LINK_CLASS(AVFoundation, AVAssetWriterInput)
46
47 SOFT_LINK_CONSTANT(AVFoundation, AVFileTypeMPEG4, NSString *)
48 SOFT_LINK_CONSTANT(AVFoundation, AVVideoCodecKey, NSString *)
49 SOFT_LINK_CONSTANT(AVFoundation, AVVideoCodecH264, NSString *)
50 SOFT_LINK_CONSTANT(AVFoundation, AVVideoWidthKey, NSString *)
51 SOFT_LINK_CONSTANT(AVFoundation, AVVideoHeightKey, NSString *)
52 SOFT_LINK_CONSTANT(AVFoundation, AVMediaTypeVideo, NSString *)
53 SOFT_LINK_CONSTANT(AVFoundation, AVMediaTypeAudio, NSString *)
54 SOFT_LINK_CONSTANT(AVFoundation, AVEncoderBitRateKey, NSString *)
55 SOFT_LINK_CONSTANT(AVFoundation, AVFormatIDKey, NSString *)
56 SOFT_LINK_CONSTANT(AVFoundation, AVNumberOfChannelsKey, NSString *)
57 SOFT_LINK_CONSTANT(AVFoundation, AVSampleRateKey, NSString *)
58
59 SOFT_LINK_CONSTANT(AVFoundation, AVVideoExpectedSourceFrameRateKey, NSString *)
60 SOFT_LINK_CONSTANT(AVFoundation, AVVideoProfileLevelKey, NSString *)
61 SOFT_LINK_CONSTANT(AVFoundation, AVVideoAverageBitRateKey, NSString *)
62 SOFT_LINK_CONSTANT(AVFoundation, AVVideoMaxKeyFrameIntervalKey, NSString *)
63 SOFT_LINK_CONSTANT(AVFoundation, AVVideoProfileLevelH264MainAutoLevel, NSString *)
64 SOFT_LINK_CONSTANT(AVFoundation, AVVideoCompressionPropertiesKey, NSString *)
65
66 #define AVFileTypeMPEG4 getAVFileTypeMPEG4()
67 #define AVMediaTypeAudio getAVMediaTypeAudio()
68 #define AVMediaTypeVideo getAVMediaTypeVideo()
69 #define AVVideoCodecKey getAVVideoCodecKey()
70 #define AVVideoCodecH264 getAVVideoCodecH264()
71 #define AVVideoWidthKey getAVVideoWidthKey()
72 #define AVVideoHeightKey getAVVideoHeightKey()
73 #define AVEncoderBitRateKey getAVEncoderBitRateKey()
74 #define AVFormatIDKey getAVFormatIDKey()
75 #define AVNumberOfChannelsKey getAVNumberOfChannelsKey()
76 #define AVSampleRateKey getAVSampleRateKey()
77
78 #define AVVideoExpectedSourceFrameRateKey getAVVideoExpectedSourceFrameRateKey()
79 #define AVVideoProfileLevelKey getAVVideoProfileLevelKey()
80 #define AVVideoAverageBitRateKey getAVVideoAverageBitRateKey()
81 #define AVVideoMaxKeyFrameIntervalKey getAVVideoMaxKeyFrameIntervalKey()
82 #define AVVideoProfileLevelH264MainAutoLevel getAVVideoProfileLevelH264MainAutoLevel()
83 #define AVVideoCompressionPropertiesKey getAVVideoCompressionPropertiesKey()
84
85 namespace WebCore {
86
87 using namespace PAL;
88
89 MediaRecorderPrivateWriter::~MediaRecorderPrivateWriter()
90 {
91     clear();
92 }
93
94 void MediaRecorderPrivateWriter::clear()
95 {
96     if (m_videoInput) {
97         m_videoInput.clear();
98         dispatch_release(m_videoPullQueue);
99     }
100     if (m_audioInput) {
101         m_audioInput.clear();
102         dispatch_release(m_audioPullQueue);
103     }
104     if (m_writer)
105         m_writer.clear();
106 }
107
108 bool MediaRecorderPrivateWriter::setupWriter()
109 {
110     ASSERT(!m_writer);
111     
112     NSString *directory = FileSystem::createTemporaryDirectory(@"videos");
113     NSString *filename = [NSString stringWithFormat:@"/%lld.mp4", CMClockGetTime(CMClockGetHostTimeClock()).value];
114     NSString *path = [directory stringByAppendingString:filename];
115
116     NSURL *outputURL = [NSURL fileURLWithPath:path];
117     m_path = [path UTF8String];
118     NSError *error = nil;
119     m_writer = adoptNS([allocAVAssetWriterInstance() initWithURL:outputURL fileType:AVFileTypeMPEG4 error:&error]);
120     if (error) {
121         RELEASE_LOG_ERROR(MediaStream, "create AVAssetWriter instance failed with error code %ld", (long)error.code);
122         m_writer = nullptr;
123         return false;
124     }
125     return true;
126 }
127
128 bool MediaRecorderPrivateWriter::setVideoInput(int width, int height)
129 {
130     ASSERT(!m_videoInput);
131     
132     NSDictionary *compressionProperties = @{
133         AVVideoAverageBitRateKey : [NSNumber numberWithInt:width * height * 12],
134         AVVideoExpectedSourceFrameRateKey : @(30),
135         AVVideoMaxKeyFrameIntervalKey : @(120),
136         AVVideoProfileLevelKey : AVVideoProfileLevelH264MainAutoLevel
137     };
138
139     NSDictionary *videoSettings = @{
140         AVVideoCodecKey: AVVideoCodecH264,
141         AVVideoWidthKey: [NSNumber numberWithInt:width],
142         AVVideoHeightKey: [NSNumber numberWithInt:height],
143         AVVideoCompressionPropertiesKey: compressionProperties
144     };
145     
146     m_videoInput = adoptNS([allocAVAssetWriterInputInstance() initWithMediaType:AVMediaTypeVideo outputSettings:videoSettings sourceFormatHint:nil]);
147     [m_videoInput setExpectsMediaDataInRealTime:true];
148     
149     if (![m_writer canAddInput:m_videoInput.get()]) {
150         m_videoInput = nullptr;
151         RELEASE_LOG_ERROR(MediaStream, "the video input is not allowed to add to the AVAssetWriter");
152         return false;
153     }
154     [m_writer addInput:m_videoInput.get()];
155     m_videoPullQueue = dispatch_queue_create("WebCoreVideoRecordingPullBufferQueue", DISPATCH_QUEUE_SERIAL);
156     return true;
157 }
158
159 bool MediaRecorderPrivateWriter::setAudioInput()
160 {
161     ASSERT(!m_audioInput);
162
163     NSDictionary *audioSettings = @{
164         AVEncoderBitRateKey : @(28000),
165         AVFormatIDKey : @(kAudioFormatMPEG4AAC),
166         AVNumberOfChannelsKey : @(1),
167         AVSampleRateKey : @(22050)
168     };
169
170     m_audioInput = adoptNS([allocAVAssetWriterInputInstance() initWithMediaType:AVMediaTypeAudio outputSettings:audioSettings sourceFormatHint:nil]);
171     [m_audioInput setExpectsMediaDataInRealTime:true];
172     
173     if (![m_writer canAddInput:m_audioInput.get()]) {
174         m_audioInput = nullptr;
175         RELEASE_LOG_ERROR(MediaStream, "the audio input is not allowed to add to the AVAssetWriter");
176         return false;
177     }
178     [m_writer addInput:m_audioInput.get()];
179     m_audioPullQueue = dispatch_queue_create("WebCoreAudioRecordingPullBufferQueue", DISPATCH_QUEUE_SERIAL);
180     return true;
181 }
182
183 static inline CMSampleBufferRef copySampleBufferWithCurrentTimeStamp(CMSampleBufferRef originalBuffer)
184 {
185     CMTime startTime = CMClockGetTime(CMClockGetHostTimeClock());
186     CMItemCount count = 0;
187     CMSampleBufferGetSampleTimingInfoArray(originalBuffer, 0, nil, &count);
188     
189     Vector<CMSampleTimingInfo> timeInfo(count);
190     CMSampleBufferGetSampleTimingInfoArray(originalBuffer, count, timeInfo.data(), &count);
191     
192     for (CMItemCount i = 0; i < count; i++) {
193         timeInfo[i].decodeTimeStamp = kCMTimeInvalid;
194         timeInfo[i].presentationTimeStamp = startTime;
195     }
196     
197     CMSampleBufferRef newBuffer;
198     CMSampleBufferCreateCopyWithNewTiming(kCFAllocatorDefault, originalBuffer, count, timeInfo.data(), &newBuffer);
199     return newBuffer;
200 }
201
202 void MediaRecorderPrivateWriter::appendVideoSampleBuffer(CMSampleBufferRef sampleBuffer)
203 {
204     ASSERT(m_videoInput);
205     if (m_isStopped)
206         return;
207
208     if (!m_hasStartedWriting) {
209         if (![m_writer startWriting]) {
210             m_isStopped = true;
211             RELEASE_LOG_ERROR(MediaStream, "create AVAssetWriter instance failed with error code %ld", (long)[m_writer error]);
212             return;
213         }
214         [m_writer startSessionAtSourceTime:CMClockGetTime(CMClockGetHostTimeClock())];
215         m_hasStartedWriting = true;
216         RefPtr<MediaRecorderPrivateWriter> protectedThis = this;
217         [m_videoInput requestMediaDataWhenReadyOnQueue:m_videoPullQueue usingBlock:[this, protectedThis = WTFMove(protectedThis)] {
218             do {
219                 if (![m_videoInput isReadyForMoreMediaData])
220                     break;
221                 auto locker = holdLock(m_videoLock);
222                 if (m_videoBufferPool.isEmpty())
223                     break;
224                 auto buffer = m_videoBufferPool.takeFirst();
225                 locker.unlockEarly();
226                 if (![m_videoInput appendSampleBuffer:buffer.get()])
227                     break;
228             } while (true);
229             if (m_isStopped && m_videoBufferPool.isEmpty()) {
230                 [m_videoInput markAsFinished];
231                 m_finishWritingVideoSemaphore.signal();
232             }
233         }];
234         return;
235     }
236     CMSampleBufferRef bufferWithCurrentTime = copySampleBufferWithCurrentTimeStamp(sampleBuffer);
237     auto locker = holdLock(m_videoLock);
238     m_videoBufferPool.append(retainPtr(bufferWithCurrentTime));
239 }
240
241 void MediaRecorderPrivateWriter::appendAudioSampleBuffer(const PlatformAudioData& data, const AudioStreamDescription& description, const WTF::MediaTime&, size_t sampleCount)
242 {
243     ASSERT(m_audioInput);
244     if ((!m_hasStartedWriting && m_videoInput) || m_isStopped)
245         return;
246     CMSampleBufferRef sampleBuffer;
247     CMFormatDescriptionRef format;
248     OSStatus error;
249     auto& basicDescription = *WTF::get<const AudioStreamBasicDescription*>(description.platformDescription().description);
250     error = CMAudioFormatDescriptionCreate(kCFAllocatorDefault, &basicDescription, 0, NULL, 0, NULL, NULL, &format);
251     if (m_isFirstAudioSample) {
252         if (!m_videoInput) {
253             // audio-only recording.
254             if (![m_writer startWriting]) {
255                 m_isStopped = true;
256                 return;
257             }
258             [m_writer startSessionAtSourceTime:CMClockGetTime(CMClockGetHostTimeClock())];
259             m_hasStartedWriting = true;
260         }
261         m_isFirstAudioSample = false;
262         RefPtr<MediaRecorderPrivateWriter> protectedThis = this;
263         [m_audioInput requestMediaDataWhenReadyOnQueue:m_audioPullQueue usingBlock:[this, protectedThis] {
264             do {
265                 if (![m_audioInput isReadyForMoreMediaData])
266                     break;
267                 auto locker = holdLock(m_audioLock);
268                 if (m_audioBufferPool.isEmpty())
269                     break;
270                 auto buffer = m_audioBufferPool.takeFirst();
271                 locker.unlockEarly();
272                 [m_audioInput appendSampleBuffer:buffer.get()];
273             } while (true);
274             if (m_isStopped && m_audioBufferPool.isEmpty()) {
275                 [m_audioInput markAsFinished];
276                 m_finishWritingAudioSemaphore.signal();
277             }
278         }];
279     }
280     CMTime startTime = CMClockGetTime(CMClockGetHostTimeClock());
281
282     error = CMAudioSampleBufferCreateWithPacketDescriptions(kCFAllocatorDefault, NULL, false, NULL, NULL, format, sampleCount, startTime, NULL, &sampleBuffer);
283     if (error)
284         return;
285     error = CMSampleBufferSetDataBufferFromAudioBufferList(sampleBuffer, kCFAllocatorDefault, kCFAllocatorDefault, 0, downcast<WebAudioBufferList>(data).list());
286     if (error)
287         return;
288
289     auto locker = holdLock(m_audioLock);
290     m_audioBufferPool.append(retainPtr(sampleBuffer));
291 }
292
293 void MediaRecorderPrivateWriter::stopRecording()
294 {
295     m_isStopped = true;
296     if (!m_hasStartedWriting)
297         return;
298     ASSERT([m_writer status] == AVAssetWriterStatusWriting);
299     if (m_videoInput)
300         m_finishWritingVideoSemaphore.wait();
301
302     if (m_audioInput)
303         m_finishWritingAudioSemaphore.wait();
304     auto weakPtr = makeWeakPtr(*this);
305     [m_writer finishWritingWithCompletionHandler:[this, weakPtr] {
306         m_finishWritingSemaphore.signal();
307         callOnMainThread([this, weakPtr] {
308             if (!weakPtr)
309                 return;
310             m_isStopped = false;
311             m_hasStartedWriting = false;
312             m_isFirstAudioSample = true;
313             clear();
314         });
315     }];
316 }
317
318 RefPtr<SharedBuffer> MediaRecorderPrivateWriter::fetchData()
319 {
320     if ((m_path.isEmpty() && !m_isStopped) || !m_hasStartedWriting)
321         return nullptr;
322     
323     m_finishWritingSemaphore.wait();
324     return SharedBuffer::createWithContentsOfFile(m_path);
325 }
326
327 } // namespace WebCore
328
329 #endif // ENABLE(MEDIA_STREAM) && USE(AVFOUNDATION)