Unreviewed, rolling out r244627.
[WebKit-https.git] / Source / WebCore / platform / mediarecorder / cocoa / MediaRecorderPrivateWriterCocoa.mm
1 /*
2  * Copyright (C) 2018 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
14  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
15  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
17  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
18  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
19  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
20  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
21  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
22  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
23  * THE POSSIBILITY OF SUCH DAMAGE.
24  */
25
26 #include "config.h"
27 #include "MediaRecorderPrivateWriterCocoa.h"
28
29 #if ENABLE(MEDIA_STREAM) && USE(AVFOUNDATION)
30
31 #include "AudioStreamDescription.h"
32 #include "Logging.h"
33 #include "MediaStreamTrackPrivate.h"
34 #include "WebAudioBufferList.h"
35 #include <AVFoundation/AVAssetWriter.h>
36 #include <AVFoundation/AVAssetWriterInput.h>
37 #include <pal/cf/CoreMediaSoftLink.h>
38 #include <wtf/FileSystem.h>
39
40 typedef AVAssetWriter AVAssetWriterType;
41 typedef AVAssetWriterInput AVAssetWriterInputType;
42
43 SOFT_LINK_FRAMEWORK_OPTIONAL(AVFoundation)
44
45 SOFT_LINK_CLASS(AVFoundation, AVAssetWriter)
46 SOFT_LINK_CLASS(AVFoundation, AVAssetWriterInput)
47
48 SOFT_LINK_CONSTANT(AVFoundation, AVFileTypeMPEG4, NSString *)
49 SOFT_LINK_CONSTANT(AVFoundation, AVVideoCodecKey, NSString *)
50 SOFT_LINK_CONSTANT(AVFoundation, AVVideoCodecH264, NSString *)
51 SOFT_LINK_CONSTANT(AVFoundation, AVVideoWidthKey, NSString *)
52 SOFT_LINK_CONSTANT(AVFoundation, AVVideoHeightKey, NSString *)
53 SOFT_LINK_CONSTANT(AVFoundation, AVMediaTypeVideo, NSString *)
54 SOFT_LINK_CONSTANT(AVFoundation, AVMediaTypeAudio, NSString *)
55
56 SOFT_LINK_CONSTANT(AVFoundation, AVVideoExpectedSourceFrameRateKey, NSString *)
57 SOFT_LINK_CONSTANT(AVFoundation, AVVideoProfileLevelKey, NSString *)
58 SOFT_LINK_CONSTANT(AVFoundation, AVVideoAverageBitRateKey, NSString *)
59 SOFT_LINK_CONSTANT(AVFoundation, AVVideoMaxKeyFrameIntervalKey, NSString *)
60 SOFT_LINK_CONSTANT(AVFoundation, AVVideoProfileLevelH264MainAutoLevel, NSString *)
61 SOFT_LINK_CONSTANT(AVFoundation, AVVideoCompressionPropertiesKey, NSString *)
62
63 #define AVFileTypeMPEG4 getAVFileTypeMPEG4()
64 #define AVMediaTypeAudio getAVMediaTypeAudio()
65 #define AVMediaTypeVideo getAVMediaTypeVideo()
66 #define AVVideoCodecKey getAVVideoCodecKey()
67 #define AVVideoCodecH264 getAVVideoCodecH264()
68 #define AVVideoWidthKey getAVVideoWidthKey()
69 #define AVVideoHeightKey getAVVideoHeightKey()
70
71 #define AVVideoExpectedSourceFrameRateKey getAVVideoExpectedSourceFrameRateKey()
72 #define AVVideoProfileLevelKey getAVVideoProfileLevelKey()
73 #define AVVideoAverageBitRateKey getAVVideoAverageBitRateKey()
74 #define AVVideoMaxKeyFrameIntervalKey getAVVideoMaxKeyFrameIntervalKey()
75 #define AVVideoProfileLevelH264MainAutoLevel getAVVideoProfileLevelH264MainAutoLevel()
76 #define AVVideoCompressionPropertiesKey getAVVideoCompressionPropertiesKey()
77
78 SOFT_LINK_CONSTANT_MAY_FAIL(AVFoundation, AVEncoderBitRateKey, NSString *)
79 SOFT_LINK_CONSTANT_MAY_FAIL(AVFoundation, AVFormatIDKey, NSString *)
80 SOFT_LINK_CONSTANT_MAY_FAIL(AVFoundation, AVNumberOfChannelsKey, NSString *)
81 SOFT_LINK_CONSTANT_MAY_FAIL(AVFoundation, AVSampleRateKey, NSString *)
82
83 #define AVEncoderBitRateKey getAVEncoderBitRateKeyWithFallback()
84 #define AVFormatIDKey getAVFormatIDKeyWithFallback()
85 #define AVNumberOfChannelsKey getAVNumberOfChannelsKeyWithFallback()
86 #define AVSampleRateKey getAVSampleRateKeyWithFallback()
87
88 namespace WebCore {
89
90 using namespace PAL;
91
92 static NSString *getAVFormatIDKeyWithFallback()
93 {
94     if (canLoadAVFormatIDKey())
95         return getAVFormatIDKey();
96
97     RELEASE_LOG_ERROR(Media, "Failed to load AVFormatIDKey");
98     return @"AVFormatIDKey";
99 }
100
101 static NSString *getAVNumberOfChannelsKeyWithFallback()
102 {
103     if (canLoadAVNumberOfChannelsKey())
104         return getAVNumberOfChannelsKey();
105
106     RELEASE_LOG_ERROR(Media, "Failed to load AVNumberOfChannelsKey");
107     return @"AVNumberOfChannelsKey";
108 }
109
110 static NSString *getAVSampleRateKeyWithFallback()
111 {
112     if (canLoadAVSampleRateKey())
113         return getAVSampleRateKey();
114
115     RELEASE_LOG_ERROR(Media, "Failed to load AVSampleRateKey");
116     return @"AVSampleRateKey";
117 }
118
119 static NSString *getAVEncoderBitRateKeyWithFallback()
120 {
121     if (canLoadAVEncoderBitRateKey())
122         return getAVEncoderBitRateKey();
123
124     RELEASE_LOG_ERROR(Media, "Failed to load AVEncoderBitRateKey");
125     return @"AVEncoderBitRateKey";
126 }
127
128 RefPtr<MediaRecorderPrivateWriter> MediaRecorderPrivateWriter::create(const MediaStreamTrackPrivate* audioTrack, const MediaStreamTrackPrivate* videoTrack)
129 {
130     NSString *directory = FileSystem::createTemporaryDirectory(@"videos");
131     NSString *filename = [NSString stringWithFormat:@"/%lld.mp4", CMClockGetTime(CMClockGetHostTimeClock()).value];
132     NSString *path = [directory stringByAppendingString:filename];
133
134     NSURL *outputURL = [NSURL fileURLWithPath:path];
135     String filePath = [path UTF8String];
136     NSError *error = nil;
137     auto avAssetWriter = adoptNS([allocAVAssetWriterInstance() initWithURL:outputURL fileType:AVFileTypeMPEG4 error:&error]);
138     if (error) {
139         RELEASE_LOG_ERROR(MediaStream, "create AVAssetWriter instance failed with error code %ld", (long)error.code);
140         return nullptr;
141     }
142
143     auto writer = adoptRef(*new MediaRecorderPrivateWriter(WTFMove(avAssetWriter), WTFMove(filePath)));
144
145     if (audioTrack && !writer->setAudioInput())
146         return nullptr;
147
148     if (videoTrack) {
149         auto& settings = videoTrack->settings();
150         if (!writer->setVideoInput(settings.width(), settings.height()))
151             return nullptr;
152     }
153
154     return WTFMove(writer);
155 }
156
157 MediaRecorderPrivateWriter::MediaRecorderPrivateWriter(RetainPtr<AVAssetWriter>&& avAssetWriter, String&& filePath)
158     : m_writer(WTFMove(avAssetWriter))
159     , m_path(WTFMove(filePath))
160 {
161 }
162
163 MediaRecorderPrivateWriter::~MediaRecorderPrivateWriter()
164 {
165     clear();
166 }
167
168 void MediaRecorderPrivateWriter::clear()
169 {
170     if (m_videoInput) {
171         m_videoInput.clear();
172         dispatch_release(m_videoPullQueue);
173     }
174     if (m_audioInput) {
175         m_audioInput.clear();
176         dispatch_release(m_audioPullQueue);
177     }
178     if (m_writer)
179         m_writer.clear();
180 }
181
182 bool MediaRecorderPrivateWriter::setVideoInput(int width, int height)
183 {
184     ASSERT(!m_videoInput);
185     
186     NSDictionary *compressionProperties = @{
187         AVVideoAverageBitRateKey : [NSNumber numberWithInt:width * height * 12],
188         AVVideoExpectedSourceFrameRateKey : @(30),
189         AVVideoMaxKeyFrameIntervalKey : @(120),
190         AVVideoProfileLevelKey : AVVideoProfileLevelH264MainAutoLevel
191     };
192
193     NSDictionary *videoSettings = @{
194         AVVideoCodecKey: AVVideoCodecH264,
195         AVVideoWidthKey: [NSNumber numberWithInt:width],
196         AVVideoHeightKey: [NSNumber numberWithInt:height],
197         AVVideoCompressionPropertiesKey: compressionProperties
198     };
199     
200     m_videoInput = adoptNS([allocAVAssetWriterInputInstance() initWithMediaType:AVMediaTypeVideo outputSettings:videoSettings sourceFormatHint:nil]);
201     [m_videoInput setExpectsMediaDataInRealTime:true];
202     
203     if (![m_writer canAddInput:m_videoInput.get()]) {
204         m_videoInput = nullptr;
205         RELEASE_LOG_ERROR(MediaStream, "the video input is not allowed to add to the AVAssetWriter");
206         return false;
207     }
208     [m_writer addInput:m_videoInput.get()];
209     m_videoPullQueue = dispatch_queue_create("WebCoreVideoRecordingPullBufferQueue", DISPATCH_QUEUE_SERIAL);
210     return true;
211 }
212
213 bool MediaRecorderPrivateWriter::setAudioInput()
214 {
215     ASSERT(!m_audioInput);
216
217     NSDictionary *audioSettings = @{
218         AVEncoderBitRateKey : @(28000),
219         AVFormatIDKey : @(kAudioFormatMPEG4AAC),
220         AVNumberOfChannelsKey : @(1),
221         AVSampleRateKey : @(22050)
222     };
223
224     m_audioInput = adoptNS([allocAVAssetWriterInputInstance() initWithMediaType:AVMediaTypeAudio outputSettings:audioSettings sourceFormatHint:nil]);
225     [m_audioInput setExpectsMediaDataInRealTime:true];
226     
227     if (![m_writer canAddInput:m_audioInput.get()]) {
228         m_audioInput = nullptr;
229         RELEASE_LOG_ERROR(MediaStream, "the audio input is not allowed to add to the AVAssetWriter");
230         return false;
231     }
232     [m_writer addInput:m_audioInput.get()];
233     m_audioPullQueue = dispatch_queue_create("WebCoreAudioRecordingPullBufferQueue", DISPATCH_QUEUE_SERIAL);
234     return true;
235 }
236
237 static inline RetainPtr<CMSampleBufferRef> copySampleBufferWithCurrentTimeStamp(CMSampleBufferRef originalBuffer)
238 {
239     CMTime startTime = CMClockGetTime(CMClockGetHostTimeClock());
240     CMItemCount count = 0;
241     CMSampleBufferGetSampleTimingInfoArray(originalBuffer, 0, nil, &count);
242     
243     Vector<CMSampleTimingInfo> timeInfo(count);
244     CMSampleBufferGetSampleTimingInfoArray(originalBuffer, count, timeInfo.data(), &count);
245     
246     for (CMItemCount i = 0; i < count; i++) {
247         timeInfo[i].decodeTimeStamp = kCMTimeInvalid;
248         timeInfo[i].presentationTimeStamp = startTime;
249     }
250     
251     CMSampleBufferRef newBuffer = nullptr;
252     auto error = CMSampleBufferCreateCopyWithNewTiming(kCFAllocatorDefault, originalBuffer, count, timeInfo.data(), &newBuffer);
253     if (error)
254         return nullptr;
255     return adoptCF(newBuffer);
256 }
257
258 void MediaRecorderPrivateWriter::appendVideoSampleBuffer(CMSampleBufferRef sampleBuffer)
259 {
260     ASSERT(m_videoInput);
261     if (m_isStopped)
262         return;
263
264     if (!m_hasStartedWriting) {
265         if (![m_writer startWriting]) {
266             m_isStopped = true;
267             RELEASE_LOG_ERROR(MediaStream, "create AVAssetWriter instance failed with error code %ld", (long)[m_writer error]);
268             return;
269         }
270         [m_writer startSessionAtSourceTime:CMClockGetTime(CMClockGetHostTimeClock())];
271         m_hasStartedWriting = true;
272         RefPtr<MediaRecorderPrivateWriter> protectedThis = this;
273         [m_videoInput requestMediaDataWhenReadyOnQueue:m_videoPullQueue usingBlock:[this, protectedThis = WTFMove(protectedThis)] {
274             do {
275                 if (![m_videoInput isReadyForMoreMediaData])
276                     break;
277                 auto locker = holdLock(m_videoLock);
278                 if (m_videoBufferPool.isEmpty())
279                     break;
280                 auto buffer = m_videoBufferPool.takeFirst();
281                 locker.unlockEarly();
282                 if (![m_videoInput appendSampleBuffer:buffer.get()])
283                     break;
284             } while (true);
285             if (m_isStopped && m_videoBufferPool.isEmpty()) {
286                 [m_videoInput markAsFinished];
287                 m_finishWritingVideoSemaphore.signal();
288             }
289         }];
290         return;
291     }
292     auto bufferWithCurrentTime = copySampleBufferWithCurrentTimeStamp(sampleBuffer);
293     if (!bufferWithCurrentTime)
294         return;
295
296     auto locker = holdLock(m_videoLock);
297     m_videoBufferPool.append(WTFMove(bufferWithCurrentTime));
298 }
299
300 static inline RetainPtr<CMFormatDescriptionRef> createAudioFormatDescription(const AudioStreamDescription& description)
301 {
302     auto basicDescription = WTF::get<const AudioStreamBasicDescription*>(description.platformDescription().description);
303     CMFormatDescriptionRef format = nullptr;
304     auto error = CMAudioFormatDescriptionCreate(kCFAllocatorDefault, basicDescription, 0, NULL, 0, NULL, NULL, &format);
305     if (error)
306         return nullptr;
307     return adoptCF(format);
308 }
309
310 static inline RetainPtr<CMSampleBufferRef> createAudioSampleBufferWithPacketDescriptions(CMFormatDescriptionRef format, size_t sampleCount)
311 {
312     CMTime startTime = CMClockGetTime(CMClockGetHostTimeClock());
313     CMSampleBufferRef sampleBuffer = nullptr;
314     auto error = CMAudioSampleBufferCreateWithPacketDescriptions(kCFAllocatorDefault, NULL, false, NULL, NULL, format, sampleCount, startTime, NULL, &sampleBuffer);
315     if (error)
316         return nullptr;
317     return adoptCF(sampleBuffer);
318 }
319
320 void MediaRecorderPrivateWriter::appendAudioSampleBuffer(const PlatformAudioData& data, const AudioStreamDescription& description, const WTF::MediaTime&, size_t sampleCount)
321 {
322     ASSERT(m_audioInput);
323     if ((!m_hasStartedWriting && m_videoInput) || m_isStopped)
324         return;
325     auto format = createAudioFormatDescription(description);
326     if (!format)
327         return;
328     if (m_isFirstAudioSample) {
329         if (!m_videoInput) {
330             // audio-only recording.
331             if (![m_writer startWriting]) {
332                 m_isStopped = true;
333                 return;
334             }
335             [m_writer startSessionAtSourceTime:CMClockGetTime(CMClockGetHostTimeClock())];
336             m_hasStartedWriting = true;
337         }
338         m_isFirstAudioSample = false;
339         RefPtr<MediaRecorderPrivateWriter> protectedThis = this;
340         [m_audioInput requestMediaDataWhenReadyOnQueue:m_audioPullQueue usingBlock:[this, protectedThis = WTFMove(protectedThis)] {
341             do {
342                 if (![m_audioInput isReadyForMoreMediaData])
343                     break;
344                 auto locker = holdLock(m_audioLock);
345                 if (m_audioBufferPool.isEmpty())
346                     break;
347                 auto buffer = m_audioBufferPool.takeFirst();
348                 locker.unlockEarly();
349                 [m_audioInput appendSampleBuffer:buffer.get()];
350             } while (true);
351             if (m_isStopped && m_audioBufferPool.isEmpty()) {
352                 [m_audioInput markAsFinished];
353                 m_finishWritingAudioSemaphore.signal();
354             }
355         }];
356     }
357
358     auto sampleBuffer = createAudioSampleBufferWithPacketDescriptions(format.get(), sampleCount);
359     if (!sampleBuffer)
360         return;
361     auto error = CMSampleBufferSetDataBufferFromAudioBufferList(sampleBuffer.get(), kCFAllocatorDefault, kCFAllocatorDefault, 0, downcast<WebAudioBufferList>(data).list());
362     if (error)
363         return;
364
365     auto locker = holdLock(m_audioLock);
366     m_audioBufferPool.append(WTFMove(sampleBuffer));
367 }
368
369 void MediaRecorderPrivateWriter::stopRecording()
370 {
371     m_isStopped = true;
372     if (!m_hasStartedWriting)
373         return;
374     ASSERT([m_writer status] == AVAssetWriterStatusWriting);
375     if (m_videoInput)
376         m_finishWritingVideoSemaphore.wait();
377
378     if (m_audioInput)
379         m_finishWritingAudioSemaphore.wait();
380     auto weakPtr = makeWeakPtr(*this);
381     [m_writer finishWritingWithCompletionHandler:[this, weakPtr] {
382         m_finishWritingSemaphore.signal();
383         callOnMainThread([this, weakPtr] {
384             if (!weakPtr)
385                 return;
386             m_isStopped = false;
387             m_hasStartedWriting = false;
388             m_isFirstAudioSample = true;
389             clear();
390         });
391     }];
392 }
393
394 RefPtr<SharedBuffer> MediaRecorderPrivateWriter::fetchData()
395 {
396     if ((m_path.isEmpty() && !m_isStopped) || !m_hasStartedWriting)
397         return nullptr;
398     
399     m_finishWritingSemaphore.wait();
400     return SharedBuffer::createWithContentsOfFile(m_path);
401 }
402
403 } // namespace WebCore
404
405 #endif // ENABLE(MEDIA_STREAM) && USE(AVFOUNDATION)