MediaRecorderPrivateAVFImpl should have a Ref<MediaRecorderPrivateWriter> as member
[WebKit-https.git] / Source / WebCore / platform / mediarecorder / cocoa / MediaRecorderPrivateWriterCocoa.mm
1 /*
2  * Copyright (C) 2018 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
14  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
15  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
17  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
18  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
19  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
20  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
21  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
22  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
23  * THE POSSIBILITY OF SUCH DAMAGE.
24  */
25
26 #include "config.h"
27 #include "MediaRecorderPrivateWriterCocoa.h"
28
29 #if ENABLE(MEDIA_STREAM) && USE(AVFOUNDATION)
30
31 #include "AudioStreamDescription.h"
32 #include "FileSystem.h"
33 #include "Logging.h"
34 #include "MediaStreamTrackPrivate.h"
35 #include "WebAudioBufferList.h"
36 #include <AVFoundation/AVAssetWriter.h>
37 #include <AVFoundation/AVAssetWriterInput.h>
38 #include <pal/cf/CoreMediaSoftLink.h>
39
40 typedef AVAssetWriter AVAssetWriterType;
41 typedef AVAssetWriterInput AVAssetWriterInputType;
42
43 SOFT_LINK_FRAMEWORK_OPTIONAL(AVFoundation)
44
45 SOFT_LINK_CLASS(AVFoundation, AVAssetWriter)
46 SOFT_LINK_CLASS(AVFoundation, AVAssetWriterInput)
47
48 SOFT_LINK_CONSTANT(AVFoundation, AVFileTypeMPEG4, NSString *)
49 SOFT_LINK_CONSTANT(AVFoundation, AVVideoCodecKey, NSString *)
50 SOFT_LINK_CONSTANT(AVFoundation, AVVideoCodecH264, NSString *)
51 SOFT_LINK_CONSTANT(AVFoundation, AVVideoWidthKey, NSString *)
52 SOFT_LINK_CONSTANT(AVFoundation, AVVideoHeightKey, NSString *)
53 SOFT_LINK_CONSTANT(AVFoundation, AVMediaTypeVideo, NSString *)
54 SOFT_LINK_CONSTANT(AVFoundation, AVMediaTypeAudio, NSString *)
55 SOFT_LINK_CONSTANT(AVFoundation, AVEncoderBitRateKey, NSString *)
56 SOFT_LINK_CONSTANT(AVFoundation, AVFormatIDKey, NSString *)
57 SOFT_LINK_CONSTANT(AVFoundation, AVNumberOfChannelsKey, NSString *)
58 SOFT_LINK_CONSTANT(AVFoundation, AVSampleRateKey, NSString *)
59
60 SOFT_LINK_CONSTANT(AVFoundation, AVVideoExpectedSourceFrameRateKey, NSString *)
61 SOFT_LINK_CONSTANT(AVFoundation, AVVideoProfileLevelKey, NSString *)
62 SOFT_LINK_CONSTANT(AVFoundation, AVVideoAverageBitRateKey, NSString *)
63 SOFT_LINK_CONSTANT(AVFoundation, AVVideoMaxKeyFrameIntervalKey, NSString *)
64 SOFT_LINK_CONSTANT(AVFoundation, AVVideoProfileLevelH264MainAutoLevel, NSString *)
65 SOFT_LINK_CONSTANT(AVFoundation, AVVideoCompressionPropertiesKey, NSString *)
66
67 #define AVFileTypeMPEG4 getAVFileTypeMPEG4()
68 #define AVMediaTypeAudio getAVMediaTypeAudio()
69 #define AVMediaTypeVideo getAVMediaTypeVideo()
70 #define AVVideoCodecKey getAVVideoCodecKey()
71 #define AVVideoCodecH264 getAVVideoCodecH264()
72 #define AVVideoWidthKey getAVVideoWidthKey()
73 #define AVVideoHeightKey getAVVideoHeightKey()
74 #define AVEncoderBitRateKey getAVEncoderBitRateKey()
75 #define AVFormatIDKey getAVFormatIDKey()
76 #define AVNumberOfChannelsKey getAVNumberOfChannelsKey()
77 #define AVSampleRateKey getAVSampleRateKey()
78
79 #define AVVideoExpectedSourceFrameRateKey getAVVideoExpectedSourceFrameRateKey()
80 #define AVVideoProfileLevelKey getAVVideoProfileLevelKey()
81 #define AVVideoAverageBitRateKey getAVVideoAverageBitRateKey()
82 #define AVVideoMaxKeyFrameIntervalKey getAVVideoMaxKeyFrameIntervalKey()
83 #define AVVideoProfileLevelH264MainAutoLevel getAVVideoProfileLevelH264MainAutoLevel()
84 #define AVVideoCompressionPropertiesKey getAVVideoCompressionPropertiesKey()
85
86 namespace WebCore {
87
88 using namespace PAL;
89
90 RefPtr<MediaRecorderPrivateWriter> MediaRecorderPrivateWriter::create(const MediaStreamTrackPrivate* audioTrack, const MediaStreamTrackPrivate* videoTrack)
91 {
92     NSString *directory = FileSystem::createTemporaryDirectory(@"videos");
93     NSString *filename = [NSString stringWithFormat:@"/%lld.mp4", CMClockGetTime(CMClockGetHostTimeClock()).value];
94     NSString *path = [directory stringByAppendingString:filename];
95
96     NSURL *outputURL = [NSURL fileURLWithPath:path];
97     String filePath = [path UTF8String];
98     NSError *error = nil;
99     auto avAssetWriter = adoptNS([allocAVAssetWriterInstance() initWithURL:outputURL fileType:AVFileTypeMPEG4 error:&error]);
100     if (error) {
101         RELEASE_LOG_ERROR(MediaStream, "create AVAssetWriter instance failed with error code %ld", (long)error.code);
102         return nullptr;
103     }
104
105     auto writer = adoptRef(*new MediaRecorderPrivateWriter(WTFMove(avAssetWriter), WTFMove(filePath)));
106
107     if (audioTrack && !writer->setAudioInput())
108         return nullptr;
109
110     if (videoTrack) {
111         auto& settings = videoTrack->settings();
112         if (!writer->setVideoInput(settings.width(), settings.height()))
113             return nullptr;
114     }
115
116     return WTFMove(writer);
117 }
118
119 MediaRecorderPrivateWriter::MediaRecorderPrivateWriter(RetainPtr<AVAssetWriter>&& avAssetWriter, String&& filePath)
120     : m_writer(WTFMove(avAssetWriter))
121     , m_path(WTFMove(filePath))
122 {
123 }
124
125 MediaRecorderPrivateWriter::~MediaRecorderPrivateWriter()
126 {
127     clear();
128 }
129
130 void MediaRecorderPrivateWriter::clear()
131 {
132     if (m_videoInput) {
133         m_videoInput.clear();
134         dispatch_release(m_videoPullQueue);
135     }
136     if (m_audioInput) {
137         m_audioInput.clear();
138         dispatch_release(m_audioPullQueue);
139     }
140     if (m_writer)
141         m_writer.clear();
142 }
143
144 bool MediaRecorderPrivateWriter::setVideoInput(int width, int height)
145 {
146     ASSERT(!m_videoInput);
147     
148     NSDictionary *compressionProperties = @{
149         AVVideoAverageBitRateKey : [NSNumber numberWithInt:width * height * 12],
150         AVVideoExpectedSourceFrameRateKey : @(30),
151         AVVideoMaxKeyFrameIntervalKey : @(120),
152         AVVideoProfileLevelKey : AVVideoProfileLevelH264MainAutoLevel
153     };
154
155     NSDictionary *videoSettings = @{
156         AVVideoCodecKey: AVVideoCodecH264,
157         AVVideoWidthKey: [NSNumber numberWithInt:width],
158         AVVideoHeightKey: [NSNumber numberWithInt:height],
159         AVVideoCompressionPropertiesKey: compressionProperties
160     };
161     
162     m_videoInput = adoptNS([allocAVAssetWriterInputInstance() initWithMediaType:AVMediaTypeVideo outputSettings:videoSettings sourceFormatHint:nil]);
163     [m_videoInput setExpectsMediaDataInRealTime:true];
164     
165     if (![m_writer canAddInput:m_videoInput.get()]) {
166         m_videoInput = nullptr;
167         RELEASE_LOG_ERROR(MediaStream, "the video input is not allowed to add to the AVAssetWriter");
168         return false;
169     }
170     [m_writer addInput:m_videoInput.get()];
171     m_videoPullQueue = dispatch_queue_create("WebCoreVideoRecordingPullBufferQueue", DISPATCH_QUEUE_SERIAL);
172     return true;
173 }
174
175 bool MediaRecorderPrivateWriter::setAudioInput()
176 {
177     ASSERT(!m_audioInput);
178
179     NSDictionary *audioSettings = @{
180         AVEncoderBitRateKey : @(28000),
181         AVFormatIDKey : @(kAudioFormatMPEG4AAC),
182         AVNumberOfChannelsKey : @(1),
183         AVSampleRateKey : @(22050)
184     };
185
186     m_audioInput = adoptNS([allocAVAssetWriterInputInstance() initWithMediaType:AVMediaTypeAudio outputSettings:audioSettings sourceFormatHint:nil]);
187     [m_audioInput setExpectsMediaDataInRealTime:true];
188     
189     if (![m_writer canAddInput:m_audioInput.get()]) {
190         m_audioInput = nullptr;
191         RELEASE_LOG_ERROR(MediaStream, "the audio input is not allowed to add to the AVAssetWriter");
192         return false;
193     }
194     [m_writer addInput:m_audioInput.get()];
195     m_audioPullQueue = dispatch_queue_create("WebCoreAudioRecordingPullBufferQueue", DISPATCH_QUEUE_SERIAL);
196     return true;
197 }
198
199 static inline CMSampleBufferRef copySampleBufferWithCurrentTimeStamp(CMSampleBufferRef originalBuffer)
200 {
201     CMTime startTime = CMClockGetTime(CMClockGetHostTimeClock());
202     CMItemCount count = 0;
203     CMSampleBufferGetSampleTimingInfoArray(originalBuffer, 0, nil, &count);
204     
205     Vector<CMSampleTimingInfo> timeInfo(count);
206     CMSampleBufferGetSampleTimingInfoArray(originalBuffer, count, timeInfo.data(), &count);
207     
208     for (CMItemCount i = 0; i < count; i++) {
209         timeInfo[i].decodeTimeStamp = kCMTimeInvalid;
210         timeInfo[i].presentationTimeStamp = startTime;
211     }
212     
213     CMSampleBufferRef newBuffer;
214     CMSampleBufferCreateCopyWithNewTiming(kCFAllocatorDefault, originalBuffer, count, timeInfo.data(), &newBuffer);
215     return newBuffer;
216 }
217
218 void MediaRecorderPrivateWriter::appendVideoSampleBuffer(CMSampleBufferRef sampleBuffer)
219 {
220     ASSERT(m_videoInput);
221     if (m_isStopped)
222         return;
223
224     if (!m_hasStartedWriting) {
225         if (![m_writer startWriting]) {
226             m_isStopped = true;
227             RELEASE_LOG_ERROR(MediaStream, "create AVAssetWriter instance failed with error code %ld", (long)[m_writer error]);
228             return;
229         }
230         [m_writer startSessionAtSourceTime:CMClockGetTime(CMClockGetHostTimeClock())];
231         m_hasStartedWriting = true;
232         RefPtr<MediaRecorderPrivateWriter> protectedThis = this;
233         [m_videoInput requestMediaDataWhenReadyOnQueue:m_videoPullQueue usingBlock:[this, protectedThis = WTFMove(protectedThis)] {
234             do {
235                 if (![m_videoInput isReadyForMoreMediaData])
236                     break;
237                 auto locker = holdLock(m_videoLock);
238                 if (m_videoBufferPool.isEmpty())
239                     break;
240                 auto buffer = m_videoBufferPool.takeFirst();
241                 locker.unlockEarly();
242                 if (![m_videoInput appendSampleBuffer:buffer.get()])
243                     break;
244             } while (true);
245             if (m_isStopped && m_videoBufferPool.isEmpty()) {
246                 [m_videoInput markAsFinished];
247                 m_finishWritingVideoSemaphore.signal();
248             }
249         }];
250         return;
251     }
252     CMSampleBufferRef bufferWithCurrentTime = copySampleBufferWithCurrentTimeStamp(sampleBuffer);
253     auto locker = holdLock(m_videoLock);
254     m_videoBufferPool.append(retainPtr(bufferWithCurrentTime));
255 }
256
257 void MediaRecorderPrivateWriter::appendAudioSampleBuffer(const PlatformAudioData& data, const AudioStreamDescription& description, const WTF::MediaTime&, size_t sampleCount)
258 {
259     ASSERT(m_audioInput);
260     if ((!m_hasStartedWriting && m_videoInput) || m_isStopped)
261         return;
262     CMSampleBufferRef sampleBuffer;
263     CMFormatDescriptionRef format;
264     OSStatus error;
265     auto& basicDescription = *WTF::get<const AudioStreamBasicDescription*>(description.platformDescription().description);
266     error = CMAudioFormatDescriptionCreate(kCFAllocatorDefault, &basicDescription, 0, NULL, 0, NULL, NULL, &format);
267     if (m_isFirstAudioSample) {
268         if (!m_videoInput) {
269             // audio-only recording.
270             if (![m_writer startWriting]) {
271                 m_isStopped = true;
272                 return;
273             }
274             [m_writer startSessionAtSourceTime:CMClockGetTime(CMClockGetHostTimeClock())];
275             m_hasStartedWriting = true;
276         }
277         m_isFirstAudioSample = false;
278         RefPtr<MediaRecorderPrivateWriter> protectedThis = this;
279         [m_audioInput requestMediaDataWhenReadyOnQueue:m_audioPullQueue usingBlock:[this, protectedThis = WTFMove(protectedThis)] {
280             do {
281                 if (![m_audioInput isReadyForMoreMediaData])
282                     break;
283                 auto locker = holdLock(m_audioLock);
284                 if (m_audioBufferPool.isEmpty())
285                     break;
286                 auto buffer = m_audioBufferPool.takeFirst();
287                 locker.unlockEarly();
288                 [m_audioInput appendSampleBuffer:buffer.get()];
289             } while (true);
290             if (m_isStopped && m_audioBufferPool.isEmpty()) {
291                 [m_audioInput markAsFinished];
292                 m_finishWritingAudioSemaphore.signal();
293             }
294         }];
295     }
296     CMTime startTime = CMClockGetTime(CMClockGetHostTimeClock());
297
298     error = CMAudioSampleBufferCreateWithPacketDescriptions(kCFAllocatorDefault, NULL, false, NULL, NULL, format, sampleCount, startTime, NULL, &sampleBuffer);
299     if (error)
300         return;
301     error = CMSampleBufferSetDataBufferFromAudioBufferList(sampleBuffer, kCFAllocatorDefault, kCFAllocatorDefault, 0, downcast<WebAudioBufferList>(data).list());
302     if (error)
303         return;
304
305     auto locker = holdLock(m_audioLock);
306     m_audioBufferPool.append(retainPtr(sampleBuffer));
307 }
308
309 void MediaRecorderPrivateWriter::stopRecording()
310 {
311     m_isStopped = true;
312     if (!m_hasStartedWriting)
313         return;
314     ASSERT([m_writer status] == AVAssetWriterStatusWriting);
315     if (m_videoInput)
316         m_finishWritingVideoSemaphore.wait();
317
318     if (m_audioInput)
319         m_finishWritingAudioSemaphore.wait();
320     auto weakPtr = makeWeakPtr(*this);
321     [m_writer finishWritingWithCompletionHandler:[this, weakPtr] {
322         m_finishWritingSemaphore.signal();
323         callOnMainThread([this, weakPtr] {
324             if (!weakPtr)
325                 return;
326             m_isStopped = false;
327             m_hasStartedWriting = false;
328             m_isFirstAudioSample = true;
329             clear();
330         });
331     }];
332 }
333
334 RefPtr<SharedBuffer> MediaRecorderPrivateWriter::fetchData()
335 {
336     if ((m_path.isEmpty() && !m_isStopped) || !m_hasStartedWriting)
337         return nullptr;
338     
339     m_finishWritingSemaphore.wait();
340     return SharedBuffer::createWithContentsOfFile(m_path);
341 }
342
343 } // namespace WebCore
344
345 #endif // ENABLE(MEDIA_STREAM) && USE(AVFOUNDATION)