MediaRecorder stopRecorder() returns empty Blob after first use
[WebKit-https.git] / Source / WebCore / platform / mediarecorder / cocoa / MediaRecorderPrivateWriterCocoa.mm
1 /*
2  * Copyright (C) 2018 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
14  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
15  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
17  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
18  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
19  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
20  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
21  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
22  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
23  * THE POSSIBILITY OF SUCH DAMAGE.
24  */
25
26 #include "config.h"
27 #include "MediaRecorderPrivateWriterCocoa.h"
28
29 #if ENABLE(MEDIA_STREAM) && HAVE(AVASSETWRITERDELEGATE)
30
31 #include "AudioSampleBufferCompressor.h"
32 #include "AudioStreamDescription.h"
33 #include "Logging.h"
34 #include "MediaStreamTrackPrivate.h"
35 #include "VideoSampleBufferCompressor.h"
36 #include "WebAudioBufferList.h"
37 #include <AVFoundation/AVAssetWriter.h>
38 #include <AVFoundation/AVAssetWriterInput.h>
39 #include <AVFoundation/AVAssetWriter_Private.h>
40 #include <pal/avfoundation/MediaTimeAVFoundation.h>
41 #include <wtf/BlockPtr.h>
42 #include <wtf/CompletionHandler.h>
43 #include <wtf/FileSystem.h>
44 #include <wtf/cf/TypeCastsCF.h>
45
46 #include <pal/cf/CoreMediaSoftLink.h>
47 #include <pal/cocoa/AVFoundationSoftLink.h>
48
49 @interface WebAVAssetWriterDelegate : NSObject <AVAssetWriterDelegate> {
50     WeakPtr<WebCore::MediaRecorderPrivateWriter> m_writer;
51 }
52
53 - (instancetype)initWithWriter:(WebCore::MediaRecorderPrivateWriter*)writer;
54 - (void)close;
55
56 @end
57
58 @implementation WebAVAssetWriterDelegate {
59 };
60
61 - (instancetype)initWithWriter:(WebCore::MediaRecorderPrivateWriter*)writer
62 {
63     ASSERT(isMainThread());
64     self = [super init];
65     if (self)
66         self->m_writer = makeWeakPtr(writer);
67
68     return self;
69 }
70
71 - (void)assetWriter:(AVAssetWriter *)assetWriter didProduceFragmentedHeaderData:(NSData *)fragmentedHeaderData
72 {
73     UNUSED_PARAM(assetWriter);
74     if (!isMainThread()) {
75         if (auto size = [fragmentedHeaderData length]) {
76             callOnMainThread([protectedSelf = RetainPtr<WebAVAssetWriterDelegate>(self), buffer = WebCore::SharedBuffer::create(static_cast<const char*>([fragmentedHeaderData bytes]), size)]() mutable {
77                 if (protectedSelf->m_writer)
78                     protectedSelf->m_writer->appendData(WTFMove(buffer));
79             });
80         }
81         return;
82     }
83
84     if (m_writer)
85         m_writer->appendData(static_cast<const char*>([fragmentedHeaderData bytes]), [fragmentedHeaderData length]);
86 }
87
88 - (void)assetWriter:(AVAssetWriter *)assetWriter didProduceFragmentedMediaData:(NSData *)fragmentedMediaData fragmentedMediaDataReport:(AVFragmentedMediaDataReport *)fragmentedMediaDataReport
89 {
90     UNUSED_PARAM(assetWriter);
91     UNUSED_PARAM(fragmentedMediaDataReport);
92     if (!isMainThread()) {
93         if (auto size = [fragmentedMediaData length]) {
94             callOnMainThread([protectedSelf = RetainPtr<WebAVAssetWriterDelegate>(self), buffer = WebCore::SharedBuffer::create(static_cast<const char*>([fragmentedMediaData bytes]), size)]() mutable {
95                 if (protectedSelf->m_writer)
96                     protectedSelf->m_writer->appendData(WTFMove(buffer));
97             });
98         }
99         return;
100     }
101
102     if (m_writer)
103         m_writer->appendData(static_cast<const char*>([fragmentedMediaData bytes]), [fragmentedMediaData length]);
104 }
105
106 - (void)close
107 {
108     m_writer = nullptr;
109 }
110
111 @end
112
113 namespace WebCore {
114
115 using namespace PAL;
116
117 RefPtr<MediaRecorderPrivateWriter> MediaRecorderPrivateWriter::create(bool hasAudio, int width, int height)
118 {
119     auto writer = adoptRef(*new MediaRecorderPrivateWriter(hasAudio, width && height));
120     if (!writer->initialize())
121         return nullptr;
122     return writer;
123 }
124
125 RefPtr<MediaRecorderPrivateWriter> MediaRecorderPrivateWriter::create(const MediaStreamTrackPrivate* audioTrack, const MediaStreamTrackPrivate* videoTrack)
126 {
127     int width = 0, height = 0;
128     if (videoTrack) {
129         auto& settings = videoTrack->settings();
130         width = settings.width();
131         height = settings.height();
132     }
133     return create(!!audioTrack, width, height);
134 }
135
136 void MediaRecorderPrivateWriter::compressedVideoOutputBufferCallback(void *mediaRecorderPrivateWriter, CMBufferQueueTriggerToken)
137 {
138     callOnMainThread([weakWriter = makeWeakPtr(static_cast<MediaRecorderPrivateWriter*>(mediaRecorderPrivateWriter))] {
139         if (weakWriter)
140             weakWriter->processNewCompressedVideoSampleBuffers();
141     });
142 }
143
144 void MediaRecorderPrivateWriter::compressedAudioOutputBufferCallback(void *mediaRecorderPrivateWriter, CMBufferQueueTriggerToken)
145 {
146     callOnMainThread([weakWriter = makeWeakPtr(static_cast<MediaRecorderPrivateWriter*>(mediaRecorderPrivateWriter))] {
147         if (weakWriter)
148             weakWriter->processNewCompressedAudioSampleBuffers();
149     });
150 }
151
152 MediaRecorderPrivateWriter::MediaRecorderPrivateWriter(bool hasAudio, bool hasVideo)
153     : m_hasAudio(hasAudio)
154     , m_hasVideo(hasVideo)
155 {
156 }
157
158 MediaRecorderPrivateWriter::~MediaRecorderPrivateWriter()
159 {
160     clear();
161 }
162
163 bool MediaRecorderPrivateWriter::initialize()
164 {
165     NSError *error = nil;
166     ALLOW_DEPRECATED_DECLARATIONS_BEGIN
167     m_writer = adoptNS([PAL::allocAVAssetWriterInstance() initWithFileType:AVFileTypeMPEG4 error:&error]);
168     ALLOW_DEPRECATED_DECLARATIONS_END
169     if (error) {
170         RELEASE_LOG_ERROR(MediaStream, "create AVAssetWriter instance failed with error code %ld", (long)error.code);
171         return false;
172     }
173
174     m_writerDelegate = adoptNS([[WebAVAssetWriterDelegate alloc] initWithWriter: this]);
175     [m_writer.get() setDelegate:m_writerDelegate.get()];
176
177     if (m_hasAudio) {
178         m_audioCompressor = AudioSampleBufferCompressor::create(compressedAudioOutputBufferCallback, this);
179         if (!m_audioCompressor)
180             return false;
181     }
182     if (m_hasVideo) {
183         m_videoCompressor = VideoSampleBufferCompressor::create(kCMVideoCodecType_H264, compressedVideoOutputBufferCallback, this);
184         if (!m_videoCompressor)
185             return false;
186     }
187     return true;
188 }
189
190 void MediaRecorderPrivateWriter::processNewCompressedVideoSampleBuffers()
191 {
192     ASSERT(m_hasVideo);
193     if (!m_videoFormatDescription) {
194         m_videoFormatDescription = CMSampleBufferGetFormatDescription(m_videoCompressor->getOutputSampleBuffer());
195
196         if (m_hasAudio && !m_audioFormatDescription)
197             return;
198
199         startAssetWriter();
200     }
201
202     if (!m_hasStartedWriting)
203         return;
204     appendCompressedSampleBuffers();
205 }
206
207 void MediaRecorderPrivateWriter::processNewCompressedAudioSampleBuffers()
208 {
209     ASSERT(m_hasAudio);
210     if (!m_audioFormatDescription) {
211         m_audioFormatDescription = CMSampleBufferGetFormatDescription(m_audioCompressor->getOutputSampleBuffer());
212         if (m_hasVideo && !m_videoFormatDescription)
213             return;
214
215         startAssetWriter();
216     }
217
218     if (!m_hasStartedWriting)
219         return;
220     appendCompressedSampleBuffers();
221 }
222
223 void MediaRecorderPrivateWriter::startAssetWriter()
224 {
225     if (m_hasVideo) {
226         m_videoAssetWriterInput = adoptNS([PAL::allocAVAssetWriterInputInstance() initWithMediaType:AVMediaTypeVideo outputSettings:nil sourceFormatHint:m_videoFormatDescription.get()]);
227         [m_videoAssetWriterInput setExpectsMediaDataInRealTime:true];
228         if (![m_writer.get() canAddInput:m_videoAssetWriterInput.get()]) {
229             RELEASE_LOG_ERROR(MediaStream, "MediaRecorderPrivateWriter::startAssetWriter failed canAddInput for video");
230             return;
231         }
232         [m_writer.get() addInput:m_videoAssetWriterInput.get()];
233     }
234
235     if (m_hasAudio) {
236         m_audioAssetWriterInput = adoptNS([PAL::allocAVAssetWriterInputInstance() initWithMediaType:AVMediaTypeAudio outputSettings:nil sourceFormatHint:m_audioFormatDescription.get()]);
237         [m_audioAssetWriterInput setExpectsMediaDataInRealTime:true];
238         if (![m_writer.get() canAddInput:m_audioAssetWriterInput.get()]) {
239             RELEASE_LOG_ERROR(MediaStream, "MediaRecorderPrivateWriter::startAssetWriter failed canAddInput for audio");
240             return;
241         }
242         [m_writer.get() addInput:m_audioAssetWriterInput.get()];
243     }
244
245     if (![m_writer.get() startWriting]) {
246         RELEASE_LOG_ERROR(MediaStream, "MediaRecorderPrivateWriter::startAssetWriter failed startWriting");
247         return;
248     }
249
250     [m_writer.get() startSessionAtSourceTime:kCMTimeZero];
251
252     appendCompressedSampleBuffers();
253
254     m_hasStartedWriting = true;
255 }
256
257 bool MediaRecorderPrivateWriter::appendCompressedAudioSampleBuffer()
258 {
259     if (!m_audioCompressor)
260         return false;
261
262     if (![m_audioAssetWriterInput isReadyForMoreMediaData])
263         return false;
264
265     auto buffer = m_audioCompressor->takeOutputSampleBuffer();
266     if (!buffer)
267         return false;
268
269     [m_audioAssetWriterInput.get() appendSampleBuffer:buffer.get()];
270     return true;
271 }
272
273 bool MediaRecorderPrivateWriter::appendCompressedVideoSampleBuffer()
274 {
275     if (!m_videoCompressor)
276         return false;
277
278     if (![m_videoAssetWriterInput isReadyForMoreMediaData])
279         return false;
280
281     auto buffer = m_videoCompressor->takeOutputSampleBuffer();
282     if (!buffer)
283         return false;
284
285     m_lastVideoPresentationTime = CMSampleBufferGetPresentationTimeStamp(buffer.get());
286     m_lastVideoDecodingTime = CMSampleBufferGetDecodeTimeStamp(buffer.get());
287     m_hasEncodedVideoSamples = true;
288
289     [m_videoAssetWriterInput.get() appendSampleBuffer:buffer.get()];
290     return true;
291 }
292
293 void MediaRecorderPrivateWriter::appendCompressedSampleBuffers()
294 {
295     while (appendCompressedVideoSampleBuffer() || appendCompressedAudioSampleBuffer()) { };
296 }
297
298 static inline void appendEndsPreviousSampleDurationMarker(AVAssetWriterInput *assetWriterInput, CMTime presentationTimeStamp, CMTime decodingTimeStamp)
299 {
300     CMSampleTimingInfo timingInfo = { kCMTimeInvalid, presentationTimeStamp, decodingTimeStamp};
301
302     CMSampleBufferRef buffer = NULL;
303     auto error = CMSampleBufferCreate(kCFAllocatorDefault, NULL, true, NULL, NULL, NULL, 0, 1, &timingInfo, 0, NULL, &buffer);
304     if (error) {
305         RELEASE_LOG_ERROR(MediaStream, "MediaRecorderPrivateWriter appendEndsPreviousSampleDurationMarker failed CMSampleBufferCreate with %d", error);
306         return;
307     }
308     auto sampleBuffer = adoptCF(buffer);
309
310     CMSetAttachment(sampleBuffer.get(), kCMSampleBufferAttachmentKey_EndsPreviousSampleDuration, kCFBooleanTrue, kCMAttachmentMode_ShouldPropagate);
311     if (![assetWriterInput appendSampleBuffer:sampleBuffer.get()])
312         RELEASE_LOG_ERROR(MediaStream, "MediaRecorderPrivateWriter appendSampleBuffer to writer input failed");
313 }
314
315 void MediaRecorderPrivateWriter::appendEndOfVideoSampleDurationIfNeeded(CompletionHandler<void()>&& completionHandler)
316 {
317     if (!m_hasEncodedVideoSamples) {
318         completionHandler();
319         return;
320     }
321     if ([m_videoAssetWriterInput isReadyForMoreMediaData]) {
322         appendEndsPreviousSampleDurationMarker(m_videoAssetWriterInput.get(), m_lastVideoPresentationTime, m_lastVideoDecodingTime);
323         completionHandler();
324         return;
325     }
326
327     auto block = makeBlockPtr([this, weakThis = makeWeakPtr(this), completionHandler = WTFMove(completionHandler)]() mutable {
328         if (weakThis) {
329             appendEndsPreviousSampleDurationMarker(m_videoAssetWriterInput.get(), m_lastVideoPresentationTime, m_lastVideoDecodingTime);
330             [m_videoAssetWriterInput markAsFinished];
331         }
332         completionHandler();
333     });
334     [m_videoAssetWriterInput requestMediaDataWhenReadyOnQueue:dispatch_get_main_queue() usingBlock:block.get()];
335 }
336
337 void MediaRecorderPrivateWriter::flushCompressedSampleBuffers(CompletionHandler<void()>&& completionHandler)
338 {
339     appendCompressedSampleBuffers();
340     appendEndOfVideoSampleDurationIfNeeded(WTFMove(completionHandler));
341 }
342
343 void MediaRecorderPrivateWriter::clear()
344 {
345     if (m_writer)
346         m_writer.clear();
347
348     m_data = nullptr;
349     if (auto completionHandler = WTFMove(m_fetchDataCompletionHandler))
350         completionHandler(nullptr);
351 }
352
353
354 static inline RetainPtr<CMSampleBufferRef> copySampleBufferWithCurrentTimeStamp(CMSampleBufferRef originalBuffer)
355 {
356     CMTime startTime = CMClockGetTime(CMClockGetHostTimeClock());
357     CMItemCount count = 0;
358     CMSampleBufferGetSampleTimingInfoArray(originalBuffer, 0, nil, &count);
359
360     Vector<CMSampleTimingInfo> timeInfo(count);
361     CMSampleBufferGetSampleTimingInfoArray(originalBuffer, count, timeInfo.data(), &count);
362
363     for (auto i = 0; i < count; i++) {
364         timeInfo[i].decodeTimeStamp = kCMTimeInvalid;
365         timeInfo[i].presentationTimeStamp = startTime;
366     }
367
368     CMSampleBufferRef newBuffer = nullptr;
369     if (auto error = CMSampleBufferCreateCopyWithNewTiming(kCFAllocatorDefault, originalBuffer, count, timeInfo.data(), &newBuffer)) {
370         RELEASE_LOG_ERROR(MediaStream, "MediaRecorderPrivateWriter CMSampleBufferCreateCopyWithNewTiming failed with %d", error);
371         return nullptr;
372     }
373     return adoptCF(newBuffer);
374 }
375
376 void MediaRecorderPrivateWriter::appendVideoSampleBuffer(CMSampleBufferRef sampleBuffer)
377 {
378     // FIXME: We should not set the timestamps if they are already set.
379     if (auto bufferWithCurrentTime = copySampleBufferWithCurrentTimeStamp(sampleBuffer))
380         m_videoCompressor->addSampleBuffer(bufferWithCurrentTime.get());
381 }
382
383 static inline RetainPtr<CMFormatDescriptionRef> createAudioFormatDescription(const AudioStreamDescription& description)
384 {
385     auto basicDescription = WTF::get<const AudioStreamBasicDescription*>(description.platformDescription().description);
386     CMFormatDescriptionRef format = nullptr;
387     auto error = CMAudioFormatDescriptionCreate(kCFAllocatorDefault, basicDescription, 0, NULL, 0, NULL, NULL, &format);
388     if (error) {
389         RELEASE_LOG_ERROR(MediaStream, "MediaRecorderPrivateWriter CMAudioFormatDescriptionCreate failed with %d", error);
390         return nullptr;
391     }
392     return adoptCF(format);
393 }
394
395 static inline RetainPtr<CMSampleBufferRef> createAudioSampleBuffer(const PlatformAudioData& data, const AudioStreamDescription& description, const WTF::MediaTime& time, size_t sampleCount)
396 {
397     auto format = createAudioFormatDescription(description);
398     if (!format)
399         return nullptr;
400
401     CMSampleBufferRef sampleBuffer = nullptr;
402     auto error = CMAudioSampleBufferCreateWithPacketDescriptions(kCFAllocatorDefault, NULL, false, NULL, NULL, format.get(), sampleCount, toCMTime(time), NULL, &sampleBuffer);
403     if (error) {
404         RELEASE_LOG_ERROR(MediaStream, "MediaRecorderPrivateWriter createAudioSampleBufferWithPacketDescriptions failed with %d", error);
405         return nullptr;
406     }
407     auto buffer = adoptCF(sampleBuffer);
408
409     error = CMSampleBufferSetDataBufferFromAudioBufferList(buffer.get(), kCFAllocatorDefault, kCFAllocatorDefault, 0, downcast<WebAudioBufferList>(data).list());
410     if (error) {
411         RELEASE_LOG_ERROR(MediaStream, "MediaRecorderPrivateWriter CMSampleBufferSetDataBufferFromAudioBufferList failed with %d", error);
412         return nullptr;
413     }
414     return buffer;
415 }
416
417 void MediaRecorderPrivateWriter::appendAudioSampleBuffer(const PlatformAudioData& data, const AudioStreamDescription& description, const WTF::MediaTime& time, size_t sampleCount)
418 {
419     if (auto sampleBuffer = createAudioSampleBuffer(data, description, time, sampleCount))
420         m_audioCompressor->addSampleBuffer(sampleBuffer.get());
421 }
422
423 void MediaRecorderPrivateWriter::stopRecording()
424 {
425     if (m_isStopped)
426         return;
427
428     m_isStopped = true;
429
430     if (m_videoCompressor)
431         m_videoCompressor->finish();
432     if (m_audioCompressor)
433         m_audioCompressor->finish();
434
435     m_isStopping = true;
436     // We hop to the main thread since finishing the video compressor might trigger starting the writer asynchronously.
437     callOnMainThread([this, weakThis = makeWeakPtr(this)]() mutable {
438         auto whenFinished = [this, weakThis] {
439             if (!weakThis)
440                 return;
441
442             m_isStopping = false;
443             m_isStopped = false;
444             m_hasStartedWriting = false;
445
446             if (m_writer)
447                 m_writer.clear();
448             if (m_fetchDataCompletionHandler)
449                 m_fetchDataCompletionHandler(std::exchange(m_data, nullptr));
450         };
451
452         if (!m_hasStartedWriting) {
453             whenFinished();
454             return;
455         }
456
457         ASSERT([m_writer status] == AVAssetWriterStatusWriting);
458         flushCompressedSampleBuffers([this, weakThis = WTFMove(weakThis), whenFinished = WTFMove(whenFinished)]() mutable {
459             if (!weakThis)
460                 return;
461
462             ALLOW_DEPRECATED_DECLARATIONS_BEGIN
463             [m_writer flush];
464             ALLOW_DEPRECATED_DECLARATIONS_END
465
466             [m_writer finishWritingWithCompletionHandler:[whenFinished = WTFMove(whenFinished)]() mutable {
467                 callOnMainThread(WTFMove(whenFinished));
468             }];
469         });
470     });
471 }
472
473 void MediaRecorderPrivateWriter::fetchData(CompletionHandler<void(RefPtr<SharedBuffer>&&)>&& completionHandler)
474 {
475     if (m_isStopping) {
476         m_fetchDataCompletionHandler = WTFMove(completionHandler);
477         return;
478     }
479
480     if (m_hasStartedWriting) {
481         flushCompressedSampleBuffers([this, weakThis = makeWeakPtr(this), completionHandler = WTFMove(completionHandler)]() mutable {
482             if (!weakThis) {
483                 completionHandler(nullptr);
484                 return;
485             }
486
487             ALLOW_DEPRECATED_DECLARATIONS_BEGIN
488             [m_writer flush];
489             ALLOW_DEPRECATED_DECLARATIONS_END
490
491             callOnMainThread([this, weakThis = WTFMove(weakThis), completionHandler = WTFMove(completionHandler)]() mutable {
492                 if (!weakThis) {
493                     completionHandler(nullptr);
494                     return;
495                 }
496
497                 completionHandler(std::exchange(m_data, nullptr));
498             });
499         });
500         return;
501     }
502
503     completionHandler(std::exchange(m_data, nullptr));
504 }
505
506 void MediaRecorderPrivateWriter::appendData(const char* data, size_t size)
507 {
508     if (!m_data) {
509         m_data = SharedBuffer::create(data, size);
510         return;
511     }
512     m_data->append(data, size);
513 }
514
515 void MediaRecorderPrivateWriter::appendData(Ref<SharedBuffer>&& buffer)
516 {
517     if (!m_data) {
518         m_data = WTFMove(buffer);
519         return;
520     }
521     m_data->append(WTFMove(buffer));
522 }
523
524 } // namespace WebCore
525
526 #endif // ENABLE(MEDIA_STREAM) && HAVE(AVASSETWRITERDELEGATE)