75d46567869841dc04cd9dad39af9158bd717284
[WebKit-https.git] / Source / WebCore / platform / mediarecorder / cocoa / MediaRecorderPrivateWriterCocoa.mm
1 /*
2  * Copyright (C) 2018 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
14  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
15  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
17  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
18  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
19  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
20  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
21  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
22  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
23  * THE POSSIBILITY OF SUCH DAMAGE.
24  */
25
26 #include "config.h"
27 #include "MediaRecorderPrivateWriterCocoa.h"
28
29 #if ENABLE(MEDIA_STREAM) && USE(AVFOUNDATION)
30
31 #include "AudioStreamDescription.h"
32 #include "Logging.h"
33 #include "MediaStreamTrackPrivate.h"
34 #include "WebAudioBufferList.h"
35 #include <AVFoundation/AVAssetWriter.h>
36 #include <AVFoundation/AVAssetWriterInput.h>
37 #include <pal/cf/CoreMediaSoftLink.h>
38 #include <wtf/FileSystem.h>
39
40 #import <pal/cocoa/AVFoundationSoftLink.h>
41
42 #undef AVEncoderBitRateKey
43 #define AVEncoderBitRateKey getAVEncoderBitRateKeyWithFallback()
44 #undef AVFormatIDKey
45 #define AVFormatIDKey getAVFormatIDKeyWithFallback()
46 #undef AVNumberOfChannelsKey
47 #define AVNumberOfChannelsKey getAVNumberOfChannelsKeyWithFallback()
48 #undef AVSampleRateKey
49 #define AVSampleRateKey getAVSampleRateKeyWithFallback()
50
51 namespace WebCore {
52
53 using namespace PAL;
54
55 static NSString *getAVFormatIDKeyWithFallback()
56 {
57     if (PAL::canLoad_AVFoundation_AVFormatIDKey())
58         return PAL::get_AVFoundation_AVFormatIDKey();
59
60     RELEASE_LOG_ERROR(Media, "Failed to load AVFormatIDKey");
61     return @"AVFormatIDKey";
62 }
63
64 static NSString *getAVNumberOfChannelsKeyWithFallback()
65 {
66     if (PAL::canLoad_AVFoundation_AVNumberOfChannelsKey())
67         return PAL::get_AVFoundation_AVNumberOfChannelsKey();
68
69     RELEASE_LOG_ERROR(Media, "Failed to load AVNumberOfChannelsKey");
70     return @"AVNumberOfChannelsKey";
71 }
72
73 static NSString *getAVSampleRateKeyWithFallback()
74 {
75     if (PAL::canLoad_AVFoundation_AVSampleRateKey())
76         return PAL::get_AVFoundation_AVSampleRateKey();
77
78     RELEASE_LOG_ERROR(Media, "Failed to load AVSampleRateKey");
79     return @"AVSampleRateKey";
80 }
81
82 static NSString *getAVEncoderBitRateKeyWithFallback()
83 {
84     if (PAL::canLoad_AVFoundation_AVEncoderBitRateKey())
85         return PAL::get_AVFoundation_AVEncoderBitRateKey();
86
87     RELEASE_LOG_ERROR(Media, "Failed to load AVEncoderBitRateKey");
88     return @"AVEncoderBitRateKey";
89 }
90
91 RefPtr<MediaRecorderPrivateWriter> MediaRecorderPrivateWriter::create(const MediaStreamTrackPrivate* audioTrack, const MediaStreamTrackPrivate* videoTrack)
92 {
93     NSString *directory = FileSystem::createTemporaryDirectory(@"videos");
94     NSString *filename = [NSString stringWithFormat:@"/%lld.mp4", CMClockGetTime(CMClockGetHostTimeClock()).value];
95     NSString *path = [directory stringByAppendingString:filename];
96
97     NSURL *outputURL = [NSURL fileURLWithPath:path];
98     String filePath = [path UTF8String];
99     NSError *error = nil;
100     auto avAssetWriter = adoptNS([PAL::allocAVAssetWriterInstance() initWithURL:outputURL fileType:AVFileTypeMPEG4 error:&error]);
101     if (error) {
102         RELEASE_LOG_ERROR(MediaStream, "create AVAssetWriter instance failed with error code %ld", (long)error.code);
103         return nullptr;
104     }
105
106     auto writer = adoptRef(*new MediaRecorderPrivateWriter(WTFMove(avAssetWriter), WTFMove(filePath)));
107
108     if (audioTrack && !writer->setAudioInput())
109         return nullptr;
110
111     if (videoTrack) {
112         auto& settings = videoTrack->settings();
113         if (!writer->setVideoInput(settings.width(), settings.height()))
114             return nullptr;
115     }
116
117     return WTFMove(writer);
118 }
119
120 MediaRecorderPrivateWriter::MediaRecorderPrivateWriter(RetainPtr<AVAssetWriter>&& avAssetWriter, String&& filePath)
121     : m_writer(WTFMove(avAssetWriter))
122     , m_path(WTFMove(filePath))
123 {
124 }
125
126 MediaRecorderPrivateWriter::~MediaRecorderPrivateWriter()
127 {
128     clear();
129 }
130
131 void MediaRecorderPrivateWriter::clear()
132 {
133     if (m_videoInput) {
134         m_videoInput.clear();
135         dispatch_release(m_videoPullQueue);
136     }
137     if (m_audioInput) {
138         m_audioInput.clear();
139         dispatch_release(m_audioPullQueue);
140     }
141     if (m_writer)
142         m_writer.clear();
143 }
144
145 bool MediaRecorderPrivateWriter::setVideoInput(int width, int height)
146 {
147     ASSERT(!m_videoInput);
148     
149     NSDictionary *compressionProperties = @{
150         AVVideoAverageBitRateKey : [NSNumber numberWithInt:width * height * 12],
151         AVVideoExpectedSourceFrameRateKey : @(30),
152         AVVideoMaxKeyFrameIntervalKey : @(120),
153         AVVideoProfileLevelKey : AVVideoProfileLevelH264MainAutoLevel
154     };
155
156     NSDictionary *videoSettings = @{
157         AVVideoCodecKey: AVVideoCodecH264,
158         AVVideoWidthKey: [NSNumber numberWithInt:width],
159         AVVideoHeightKey: [NSNumber numberWithInt:height],
160         AVVideoCompressionPropertiesKey: compressionProperties
161     };
162     
163     m_videoInput = adoptNS([PAL::allocAVAssetWriterInputInstance() initWithMediaType:AVMediaTypeVideo outputSettings:videoSettings sourceFormatHint:nil]);
164     [m_videoInput setExpectsMediaDataInRealTime:true];
165     
166     if (![m_writer canAddInput:m_videoInput.get()]) {
167         m_videoInput = nullptr;
168         RELEASE_LOG_ERROR(MediaStream, "the video input is not allowed to add to the AVAssetWriter");
169         return false;
170     }
171     [m_writer addInput:m_videoInput.get()];
172     m_videoPullQueue = dispatch_queue_create("WebCoreVideoRecordingPullBufferQueue", DISPATCH_QUEUE_SERIAL);
173     return true;
174 }
175
176 bool MediaRecorderPrivateWriter::setAudioInput()
177 {
178     ASSERT(!m_audioInput);
179
180     NSDictionary *audioSettings = @{
181         AVEncoderBitRateKey : @(28000),
182         AVFormatIDKey : @(kAudioFormatMPEG4AAC),
183         AVNumberOfChannelsKey : @(1),
184         AVSampleRateKey : @(22050)
185     };
186
187     m_audioInput = adoptNS([PAL::allocAVAssetWriterInputInstance() initWithMediaType:AVMediaTypeAudio outputSettings:audioSettings sourceFormatHint:nil]);
188     [m_audioInput setExpectsMediaDataInRealTime:true];
189     
190     if (![m_writer canAddInput:m_audioInput.get()]) {
191         m_audioInput = nullptr;
192         RELEASE_LOG_ERROR(MediaStream, "the audio input is not allowed to add to the AVAssetWriter");
193         return false;
194     }
195     [m_writer addInput:m_audioInput.get()];
196     m_audioPullQueue = dispatch_queue_create("WebCoreAudioRecordingPullBufferQueue", DISPATCH_QUEUE_SERIAL);
197     return true;
198 }
199
200 static inline RetainPtr<CMSampleBufferRef> copySampleBufferWithCurrentTimeStamp(CMSampleBufferRef originalBuffer)
201 {
202     CMTime startTime = CMClockGetTime(CMClockGetHostTimeClock());
203     CMItemCount count = 0;
204     CMSampleBufferGetSampleTimingInfoArray(originalBuffer, 0, nil, &count);
205     
206     Vector<CMSampleTimingInfo> timeInfo(count);
207     CMSampleBufferGetSampleTimingInfoArray(originalBuffer, count, timeInfo.data(), &count);
208     
209     for (CMItemCount i = 0; i < count; i++) {
210         timeInfo[i].decodeTimeStamp = kCMTimeInvalid;
211         timeInfo[i].presentationTimeStamp = startTime;
212     }
213     
214     CMSampleBufferRef newBuffer = nullptr;
215     auto error = CMSampleBufferCreateCopyWithNewTiming(kCFAllocatorDefault, originalBuffer, count, timeInfo.data(), &newBuffer);
216     if (error)
217         return nullptr;
218     return adoptCF(newBuffer);
219 }
220
221 void MediaRecorderPrivateWriter::appendVideoSampleBuffer(CMSampleBufferRef sampleBuffer)
222 {
223     ASSERT(m_videoInput);
224     if (m_isStopped)
225         return;
226
227     if (!m_hasStartedWriting) {
228         if (![m_writer startWriting]) {
229             m_isStopped = true;
230             RELEASE_LOG_ERROR(MediaStream, "create AVAssetWriter instance failed with error code %ld", (long)[m_writer error]);
231             return;
232         }
233         [m_writer startSessionAtSourceTime:CMClockGetTime(CMClockGetHostTimeClock())];
234         m_hasStartedWriting = true;
235         RefPtr<MediaRecorderPrivateWriter> protectedThis = this;
236         [m_videoInput requestMediaDataWhenReadyOnQueue:m_videoPullQueue usingBlock:[this, protectedThis = WTFMove(protectedThis)] {
237             do {
238                 if (![m_videoInput isReadyForMoreMediaData])
239                     break;
240                 auto locker = holdLock(m_videoLock);
241                 if (m_videoBufferPool.isEmpty())
242                     break;
243                 auto buffer = m_videoBufferPool.takeFirst();
244                 locker.unlockEarly();
245                 if (![m_videoInput appendSampleBuffer:buffer.get()])
246                     break;
247             } while (true);
248             if (m_isStopped && m_videoBufferPool.isEmpty()) {
249                 [m_videoInput markAsFinished];
250                 m_finishWritingVideoSemaphore.signal();
251             }
252         }];
253         return;
254     }
255     auto bufferWithCurrentTime = copySampleBufferWithCurrentTimeStamp(sampleBuffer);
256     if (!bufferWithCurrentTime)
257         return;
258
259     auto locker = holdLock(m_videoLock);
260     m_videoBufferPool.append(WTFMove(bufferWithCurrentTime));
261 }
262
263 static inline RetainPtr<CMFormatDescriptionRef> createAudioFormatDescription(const AudioStreamDescription& description)
264 {
265     auto basicDescription = WTF::get<const AudioStreamBasicDescription*>(description.platformDescription().description);
266     CMFormatDescriptionRef format = nullptr;
267     auto error = CMAudioFormatDescriptionCreate(kCFAllocatorDefault, basicDescription, 0, NULL, 0, NULL, NULL, &format);
268     if (error)
269         return nullptr;
270     return adoptCF(format);
271 }
272
273 static inline RetainPtr<CMSampleBufferRef> createAudioSampleBufferWithPacketDescriptions(CMFormatDescriptionRef format, size_t sampleCount)
274 {
275     CMTime startTime = CMClockGetTime(CMClockGetHostTimeClock());
276     CMSampleBufferRef sampleBuffer = nullptr;
277     auto error = CMAudioSampleBufferCreateWithPacketDescriptions(kCFAllocatorDefault, NULL, false, NULL, NULL, format, sampleCount, startTime, NULL, &sampleBuffer);
278     if (error)
279         return nullptr;
280     return adoptCF(sampleBuffer);
281 }
282
283 void MediaRecorderPrivateWriter::appendAudioSampleBuffer(const PlatformAudioData& data, const AudioStreamDescription& description, const WTF::MediaTime&, size_t sampleCount)
284 {
285     ASSERT(m_audioInput);
286     if ((!m_hasStartedWriting && m_videoInput) || m_isStopped)
287         return;
288     auto format = createAudioFormatDescription(description);
289     if (!format)
290         return;
291     if (m_isFirstAudioSample) {
292         if (!m_videoInput) {
293             // audio-only recording.
294             if (![m_writer startWriting]) {
295                 m_isStopped = true;
296                 return;
297             }
298             [m_writer startSessionAtSourceTime:CMClockGetTime(CMClockGetHostTimeClock())];
299             m_hasStartedWriting = true;
300         }
301         m_isFirstAudioSample = false;
302         RefPtr<MediaRecorderPrivateWriter> protectedThis = this;
303         [m_audioInput requestMediaDataWhenReadyOnQueue:m_audioPullQueue usingBlock:[this, protectedThis = WTFMove(protectedThis)] {
304             do {
305                 if (![m_audioInput isReadyForMoreMediaData])
306                     break;
307                 auto locker = holdLock(m_audioLock);
308                 if (m_audioBufferPool.isEmpty())
309                     break;
310                 auto buffer = m_audioBufferPool.takeFirst();
311                 locker.unlockEarly();
312                 [m_audioInput appendSampleBuffer:buffer.get()];
313             } while (true);
314             if (m_isStopped && m_audioBufferPool.isEmpty()) {
315                 [m_audioInput markAsFinished];
316                 m_finishWritingAudioSemaphore.signal();
317             }
318         }];
319     }
320
321     auto sampleBuffer = createAudioSampleBufferWithPacketDescriptions(format.get(), sampleCount);
322     if (!sampleBuffer)
323         return;
324     auto error = CMSampleBufferSetDataBufferFromAudioBufferList(sampleBuffer.get(), kCFAllocatorDefault, kCFAllocatorDefault, 0, downcast<WebAudioBufferList>(data).list());
325     if (error)
326         return;
327
328     auto locker = holdLock(m_audioLock);
329     m_audioBufferPool.append(WTFMove(sampleBuffer));
330 }
331
332 void MediaRecorderPrivateWriter::stopRecording()
333 {
334     m_isStopped = true;
335     if (!m_hasStartedWriting)
336         return;
337     ASSERT([m_writer status] == AVAssetWriterStatusWriting);
338     if (m_videoInput)
339         m_finishWritingVideoSemaphore.wait();
340
341     if (m_audioInput)
342         m_finishWritingAudioSemaphore.wait();
343     auto weakPtr = makeWeakPtr(*this);
344     [m_writer finishWritingWithCompletionHandler:[this, weakPtr] {
345         m_finishWritingSemaphore.signal();
346         callOnMainThread([this, weakPtr] {
347             if (!weakPtr)
348                 return;
349             m_isStopped = false;
350             m_hasStartedWriting = false;
351             m_isFirstAudioSample = true;
352             clear();
353         });
354     }];
355 }
356
357 RefPtr<SharedBuffer> MediaRecorderPrivateWriter::fetchData()
358 {
359     if ((m_path.isEmpty() && !m_isStopped) || !m_hasStartedWriting)
360         return nullptr;
361     
362     m_finishWritingSemaphore.wait();
363     return SharedBuffer::createWithContentsOfFile(m_path);
364 }
365
366 } // namespace WebCore
367
368 #endif // ENABLE(MEDIA_STREAM) && USE(AVFOUNDATION)