Unreviewed, add myself to some watch lists.
[WebKit-https.git] / Source / WebCore / platform / mediarecorder / cocoa / MediaRecorderPrivateWriterCocoa.mm
1 /*
2  * Copyright (C) 2018 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
14  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
15  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
17  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
18  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
19  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
20  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
21  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
22  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
23  * THE POSSIBILITY OF SUCH DAMAGE.
24  */
25
26 #include "config.h"
27 #include "MediaRecorderPrivateWriterCocoa.h"
28
29 #if ENABLE(MEDIA_STREAM) && USE(AVFOUNDATION)
30
31 #include "AudioStreamDescription.h"
32 #include "FileSystem.h"
33 #include "Logging.h"
34 #include "WebAudioBufferList.h"
35 #include <AVFoundation/AVAssetWriter.h>
36 #include <AVFoundation/AVAssetWriterInput.h>
37 #include <pal/cf/CoreMediaSoftLink.h>
38
39 typedef AVAssetWriter AVAssetWriterType;
40 typedef AVAssetWriterInput AVAssetWriterInputType;
41
42 SOFT_LINK_FRAMEWORK_OPTIONAL(AVFoundation)
43
44 SOFT_LINK_CLASS(AVFoundation, AVAssetWriter)
45 SOFT_LINK_CLASS(AVFoundation, AVAssetWriterInput)
46
47 SOFT_LINK_CONSTANT(AVFoundation, AVFileTypeMPEG4, NSString *)
48 SOFT_LINK_CONSTANT(AVFoundation, AVVideoCodecKey, NSString *)
49 SOFT_LINK_CONSTANT(AVFoundation, AVVideoCodecH264, NSString *)
50 SOFT_LINK_CONSTANT(AVFoundation, AVVideoWidthKey, NSString *)
51 SOFT_LINK_CONSTANT(AVFoundation, AVVideoHeightKey, NSString *)
52 SOFT_LINK_CONSTANT(AVFoundation, AVMediaTypeVideo, NSString *)
53 SOFT_LINK_CONSTANT(AVFoundation, AVMediaTypeAudio, NSString *)
54 SOFT_LINK_CONSTANT(AVFoundation, AVEncoderBitRatePerChannelKey, NSString *)
55 SOFT_LINK_CONSTANT(AVFoundation, AVFormatIDKey, NSString *)
56 SOFT_LINK_CONSTANT(AVFoundation, AVNumberOfChannelsKey, NSString *)
57 SOFT_LINK_CONSTANT(AVFoundation, AVSampleRateKey, NSString *)
58
59 #define AVFileTypeMPEG4 getAVFileTypeMPEG4()
60 #define AVMediaTypeAudio getAVMediaTypeAudio()
61 #define AVMediaTypeVideo getAVMediaTypeVideo()
62 #define AVVideoCodecKey getAVVideoCodecKey()
63 #define AVVideoCodecH264 getAVVideoCodecH264()
64 #define AVVideoWidthKey getAVVideoWidthKey()
65 #define AVVideoHeightKey getAVVideoHeightKey()
66 #define AVEncoderBitRatePerChannelKey getAVEncoderBitRatePerChannelKey()
67 #define AVFormatIDKey getAVFormatIDKey()
68 #define AVNumberOfChannelsKey getAVNumberOfChannelsKey()
69 #define AVSampleRateKey getAVSampleRateKey()
70
71 namespace WebCore {
72
73 using namespace PAL;
74
75 bool MediaRecorderPrivateWriter::setupWriter()
76 {
77     ASSERT(!m_writer);
78     
79     NSString *directory = FileSystem::createTemporaryDirectory(@"videos");
80     NSString *filename = [NSString stringWithFormat:@"/%lld.mp4", CMClockGetTime(CMClockGetHostTimeClock()).value];
81     NSString *path = [directory stringByAppendingString:filename];
82
83     NSURL *outputURL = [NSURL fileURLWithPath:path];
84     m_path = [path UTF8String];
85     NSError *error = nil;
86     m_writer = adoptNS([allocAVAssetWriterInstance() initWithURL:outputURL fileType:AVFileTypeMPEG4 error:&error]);
87     if (error) {
88         RELEASE_LOG_ERROR(MediaStream, "create AVAssetWriter instance failed with error code %ld", (long)error.code);
89         m_writer = nullptr;
90         return false;
91     }
92     return true;
93 }
94
95 bool MediaRecorderPrivateWriter::setVideoInput(int width, int height)
96 {
97     ASSERT(!m_videoInput);
98
99     NSDictionary *videoSettings = @{ AVVideoCodecKey: AVVideoCodecH264, AVVideoWidthKey: [NSNumber numberWithInt:width], AVVideoHeightKey: [NSNumber numberWithInt:height] };
100     m_videoInput = adoptNS([allocAVAssetWriterInputInstance() initWithMediaType:AVMediaTypeVideo outputSettings:videoSettings sourceFormatHint:nil]);
101     [m_videoInput setExpectsMediaDataInRealTime:true];
102     
103     if (![m_writer canAddInput:m_videoInput.get()]) {
104         m_videoInput = nullptr;
105         RELEASE_LOG_ERROR(MediaStream, "the video input is not allowed to add to the AVAssetWriter");
106         return false;
107     }
108     [m_writer addInput:m_videoInput.get()];
109     m_videoPullQueue = dispatch_queue_create("WebCoreVideoRecordingPullBufferQueue", DISPATCH_QUEUE_SERIAL);
110     return true;
111 }
112
113 bool MediaRecorderPrivateWriter::setAudioInput()
114 {
115     ASSERT(!m_audioInput);
116
117     NSDictionary *audioSettings = @{ AVEncoderBitRatePerChannelKey : @(28000), AVFormatIDKey : @(kAudioFormatMPEG4AAC), AVNumberOfChannelsKey : @(1), AVSampleRateKey : @(22050) };
118
119     m_audioInput = adoptNS([allocAVAssetWriterInputInstance() initWithMediaType:AVMediaTypeAudio outputSettings:audioSettings sourceFormatHint:nil]);
120     [m_audioInput setExpectsMediaDataInRealTime:true];
121     
122     if (![m_writer canAddInput:m_audioInput.get()]) {
123         m_audioInput = nullptr;
124         RELEASE_LOG_ERROR(MediaStream, "the audio input is not allowed to add to the AVAssetWriter");
125         return false;
126     }
127     [m_writer addInput:m_audioInput.get()];
128     m_audioPullQueue = dispatch_queue_create("WebCoreAudioRecordingPullBufferQueue", DISPATCH_QUEUE_SERIAL);
129     return true;
130 }
131
132 static inline CMSampleBufferRef copySampleBufferWithCurrentTimeStamp(CMSampleBufferRef originalBuffer)
133 {
134     CMTime startTime = CMClockGetTime(CMClockGetHostTimeClock());
135     CMItemCount count = 0;
136     CMSampleBufferGetSampleTimingInfoArray(originalBuffer, 0, nil, &count);
137     
138     Vector<CMSampleTimingInfo> timeInfo(count);
139     CMSampleBufferGetSampleTimingInfoArray(originalBuffer, count, timeInfo.data(), &count);
140     
141     for (CMItemCount i = 0; i < count; i++) {
142         timeInfo[i].decodeTimeStamp = kCMTimeInvalid;
143         timeInfo[i].presentationTimeStamp = startTime;
144     }
145     
146     CMSampleBufferRef newBuffer;
147     CMSampleBufferCreateCopyWithNewTiming(kCFAllocatorDefault, originalBuffer, count, timeInfo.data(), &newBuffer);
148     return newBuffer;
149 }
150
151 void MediaRecorderPrivateWriter::appendVideoSampleBuffer(CMSampleBufferRef sampleBuffer)
152 {
153     ASSERT(m_videoInput);
154     if (m_isStopped)
155         return;
156
157     if (!m_hasStartedWriting) {
158         if (![m_writer startWriting]) {
159             m_isStopped = true;
160             RELEASE_LOG_ERROR(MediaStream, "create AVAssetWriter instance failed with error code %ld", (long)[m_writer error]);
161             return;
162         }
163         [m_writer startSessionAtSourceTime:CMClockGetTime(CMClockGetHostTimeClock())];
164         m_hasStartedWriting = true;
165         RefPtr<MediaRecorderPrivateWriter> protectedThis = this;
166         [m_videoInput requestMediaDataWhenReadyOnQueue:m_videoPullQueue usingBlock:[this, protectedThis] {
167             do {
168                 if (![m_videoInput isReadyForMoreMediaData])
169                     break;
170                 auto locker = holdLock(m_videoLock);
171                 if (m_videoBufferPool.isEmpty())
172                     break;
173                 auto buffer = m_videoBufferPool.takeFirst();
174                 locker.unlockEarly();
175                 if (![m_videoInput appendSampleBuffer:buffer.get()])
176                     break;
177             } while (true);
178             if (m_isStopped && m_videoBufferPool.isEmpty()) {
179                 [m_videoInput markAsFinished];
180                 m_finishWritingVideoSemaphore.signal();
181             }
182         }];
183         return;
184     }
185     CMSampleBufferRef bufferWithCurrentTime = copySampleBufferWithCurrentTimeStamp(sampleBuffer);
186     auto locker = holdLock(m_videoLock);
187     m_videoBufferPool.append(retainPtr(bufferWithCurrentTime));
188 }
189
190 void MediaRecorderPrivateWriter::appendAudioSampleBuffer(const PlatformAudioData& data, const AudioStreamDescription& description, const WTF::MediaTime&, size_t sampleCount)
191 {
192     ASSERT(m_audioInput);
193     if ((!m_hasStartedWriting && m_videoInput) || m_isStopped)
194         return;
195     CMSampleBufferRef sampleBuffer;
196     CMFormatDescriptionRef format;
197     OSStatus error;
198     auto& basicDescription = *WTF::get<const AudioStreamBasicDescription*>(description.platformDescription().description);
199     error = CMAudioFormatDescriptionCreate(kCFAllocatorDefault, &basicDescription, 0, NULL, 0, NULL, NULL, &format);
200     if (m_isFirstAudioSample) {
201         if (!m_videoInput) {
202             // audio-only recording.
203             if (![m_writer startWriting]) {
204                 m_isStopped = true;
205                 return;
206             }
207             [m_writer startSessionAtSourceTime:CMClockGetTime(CMClockGetHostTimeClock())];
208             m_hasStartedWriting = true;
209         }
210         m_isFirstAudioSample = false;
211         RefPtr<MediaRecorderPrivateWriter> protectedThis = this;
212         [m_audioInput requestMediaDataWhenReadyOnQueue:m_audioPullQueue usingBlock:[this, protectedThis] {
213             do {
214                 if (![m_audioInput isReadyForMoreMediaData])
215                     break;
216                 auto locker = holdLock(m_audioLock);
217                 if (m_audioBufferPool.isEmpty())
218                     break;
219                 auto buffer = m_audioBufferPool.takeFirst();
220                 locker.unlockEarly();
221                 [m_audioInput appendSampleBuffer:buffer.get()];
222             } while (true);
223             if (m_isStopped && m_audioBufferPool.isEmpty()) {
224                 [m_audioInput markAsFinished];
225                 m_finishWritingAudioSemaphore.signal();
226             }
227         }];
228     }
229     CMTime startTime = CMClockGetTime(CMClockGetHostTimeClock());
230
231     error = CMAudioSampleBufferCreateWithPacketDescriptions(kCFAllocatorDefault, NULL, false, NULL, NULL, format, sampleCount, startTime, NULL, &sampleBuffer);
232     if (error)
233         return;
234     error = CMSampleBufferSetDataBufferFromAudioBufferList(sampleBuffer, kCFAllocatorDefault, kCFAllocatorDefault, 0, downcast<WebAudioBufferList>(data).list());
235     if (error)
236         return;
237
238     auto locker = holdLock(m_audioLock);
239     m_audioBufferPool.append(retainPtr(sampleBuffer));
240 }
241
242 void MediaRecorderPrivateWriter::stopRecording()
243 {
244     m_isStopped = true;
245     if (!m_hasStartedWriting)
246         return;
247     ASSERT([m_writer status] == AVAssetWriterStatusWriting);
248     if (m_videoInput)
249         m_finishWritingVideoSemaphore.wait();
250
251     if (m_audioInput)
252         m_finishWritingAudioSemaphore.wait();
253
254     [m_writer finishWritingWithCompletionHandler:^{
255         m_isStopped = false;
256         m_hasStartedWriting = false;
257         m_isFirstAudioSample = true;
258         if (m_videoInput) {
259             m_videoInput.clear();
260             m_videoInput = nullptr;
261             dispatch_release(m_videoPullQueue);
262         }
263         if (m_audioInput) {
264             m_audioInput.clear();
265             m_audioInput = nullptr;
266             dispatch_release(m_audioPullQueue);
267         }
268         m_writer.clear();
269         m_writer = nullptr;
270         m_finishWritingSemaphore.signal();
271     }];
272 }
273
274 RefPtr<SharedBuffer> MediaRecorderPrivateWriter::fetchData()
275 {
276     if ((m_path.isEmpty() && !m_isStopped) || !m_hasStartedWriting)
277         return nullptr;
278     
279     m_finishWritingSemaphore.wait();
280     return SharedBuffer::createWithContentsOfFile(m_path);
281 }
282
283 } // namespace WebCore
284
285 #endif // ENABLE(MEDIA_STREAM) && USE(AVFOUNDATION)