d10f3cce9d0ee77df7628b0bba66370268f85db4
[WebKit-https.git] / Source / ThirdParty / libwebrtc / Source / webrtc / sdk / objc / Framework / Classes / VideoToolbox / encoder.mm
1 /*
2  *  Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
3  *
4  *  Use of this source code is governed by a BSD-style license
5  *  that can be found in the LICENSE file in the root of the source
6  *  tree. An additional intellectual property rights grant can be found
7  *  in the file PATENTS.  All contributing project authors may
8  *  be found in the AUTHORS file in the root of the source tree.
9  *
10  */
11
12 #include "webrtc/sdk/objc/Framework/Classes/VideoToolbox/encoder.h"
13
14 #include <memory>
15 #include <string>
16 #include <vector>
17
18 #if defined(WEBRTC_IOS)  && !defined(WEBRTC_WEBKIT_BUILD)
19 #import "Common/RTCUIApplicationStatusObserver.h"
20 #import "WebRTC/UIDevice+RTCDevice.h"
21 #endif
22 #include "libyuv/convert_from.h"
23 #include "webrtc/base/checks.h"
24 #include "webrtc/base/logging.h"
25 #include "webrtc/common_video/h264/profile_level_id.h"
26 #include "webrtc/sdk/objc/Framework/Classes/Video/corevideo_frame_buffer.h"
27 #include "webrtc/sdk/objc/Framework/Classes/VideoToolbox/nalu_rewriter.h"
28 #include "webrtc/system_wrappers/include/clock.h"
29
30 namespace internal {
31
32 // The ratio between kVTCompressionPropertyKey_DataRateLimits and
33 // kVTCompressionPropertyKey_AverageBitRate. The data rate limit is set higher
34 // than the average bit rate to avoid undershooting the target.
35 const float kLimitToAverageBitRateFactor = 1.5f;
36 // These thresholds deviate from the default h264 QP thresholds, as they
37 // have been found to work better on devices that support VideoToolbox
38 const int kLowH264QpThreshold = 28;
39 const int kHighH264QpThreshold = 39;
40
41 // Convenience function for creating a dictionary.
42 inline CFDictionaryRef CreateCFDictionary(CFTypeRef* keys,
43                                           CFTypeRef* values,
44                                           size_t size) {
45   return CFDictionaryCreate(kCFAllocatorDefault, keys, values, size,
46                             &kCFTypeDictionaryKeyCallBacks,
47                             &kCFTypeDictionaryValueCallBacks);
48 }
49
50 // Copies characters from a CFStringRef into a std::string.
51 static std::string CFStringToString(const CFStringRef cf_string) {
52   RTC_DCHECK(cf_string);
53   std::string std_string;
54   // Get the size needed for UTF8 plus terminating character.
55   size_t buffer_size =
56       CFStringGetMaximumSizeForEncoding(CFStringGetLength(cf_string),
57                                         kCFStringEncodingUTF8) +
58       1;
59   std::unique_ptr<char[]> buffer(new char[buffer_size]);
60   if (CFStringGetCString(cf_string, buffer.get(), buffer_size,
61                          kCFStringEncodingUTF8)) {
62     // Copy over the characters.
63     std_string.assign(buffer.get());
64   }
65   return std_string;
66 }
67
68 // Convenience function for setting a VT property.
69 static void SetVTSessionProperty(VTSessionRef session,
70                           CFStringRef key,
71                           int32_t value) {
72   CFNumberRef cfNum =
73       CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &value);
74   OSStatus status = VTSessionSetProperty(session, key, cfNum);
75   CFRelease(cfNum);
76   if (status != noErr) {
77     std::string key_string = CFStringToString(key);
78     LOG(LS_ERROR) << "VTSessionSetProperty failed to set: " << key_string
79                   << " to " << value << ": " << status;
80   }
81 }
82
83 // Convenience function for setting a VT property.
84 static void SetVTSessionProperty(VTSessionRef session,
85                           CFStringRef key,
86                           uint32_t value) {
87   int64_t value_64 = value;
88   CFNumberRef cfNum =
89       CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt64Type, &value_64);
90   OSStatus status = VTSessionSetProperty(session, key, cfNum);
91   CFRelease(cfNum);
92   if (status != noErr) {
93     std::string key_string = CFStringToString(key);
94     LOG(LS_ERROR) << "VTSessionSetProperty failed to set: " << key_string
95                   << " to " << value << ": " << status;
96   }
97 }
98
99 // Convenience function for setting a VT property.
100 static void SetVTSessionProperty(VTSessionRef session, CFStringRef key, bool value) {
101   CFBooleanRef cf_bool = (value) ? kCFBooleanTrue : kCFBooleanFalse;
102   OSStatus status = VTSessionSetProperty(session, key, cf_bool);
103   if (status != noErr) {
104     std::string key_string = CFStringToString(key);
105     LOG(LS_ERROR) << "VTSessionSetProperty failed to set: " << key_string
106                   << " to " << value << ": " << status;
107   }
108 }
109
110 // Convenience function for setting a VT property.
111 static void SetVTSessionProperty(VTSessionRef session,
112                           CFStringRef key,
113                           CFStringRef value) {
114   OSStatus status = VTSessionSetProperty(session, key, value);
115   if (status != noErr) {
116     std::string key_string = CFStringToString(key);
117     std::string val_string = CFStringToString(value);
118     LOG(LS_ERROR) << "VTSessionSetProperty failed to set: " << key_string
119                   << " to " << val_string << ": " << status;
120   }
121 }
122
123 // Struct that we pass to the encoder per frame to encode. We receive it again
124 // in the encoder callback.
125 struct FrameEncodeParams {
126   FrameEncodeParams(webrtc::H264VideoToolboxEncoder* e,
127                     const webrtc::CodecSpecificInfo* csi,
128                     int32_t w,
129                     int32_t h,
130                     int64_t rtms,
131                     uint32_t ts,
132                     webrtc::VideoRotation r)
133       : encoder(e),
134         width(w),
135         height(h),
136         render_time_ms(rtms),
137         timestamp(ts),
138         rotation(r) {
139     if (csi) {
140       codec_specific_info = *csi;
141     } else {
142       codec_specific_info.codecType = webrtc::kVideoCodecH264;
143     }
144   }
145
146   webrtc::H264VideoToolboxEncoder* encoder;
147   webrtc::CodecSpecificInfo codec_specific_info;
148   int32_t width;
149   int32_t height;
150   int64_t render_time_ms;
151   uint32_t timestamp;
152   webrtc::VideoRotation rotation;
153 };
154
155 // We receive I420Frames as input, but we need to feed CVPixelBuffers into the
156 // encoder. This performs the copy and format conversion.
157 // TODO(tkchin): See if encoder will accept i420 frames and compare performance.
158 static bool CopyVideoFrameToPixelBuffer(const rtc::scoped_refptr<webrtc::I420BufferInterface>& frame,
159                                  CVPixelBufferRef pixel_buffer) {
160   RTC_DCHECK(pixel_buffer);
161   RTC_DCHECK_EQ(CVPixelBufferGetPixelFormatType(pixel_buffer),
162                 kCVPixelFormatType_420YpCbCr8BiPlanarFullRange);
163   RTC_DCHECK_EQ(CVPixelBufferGetHeightOfPlane(pixel_buffer, 0),
164                 static_cast<size_t>(frame->height()));
165   RTC_DCHECK_EQ(CVPixelBufferGetWidthOfPlane(pixel_buffer, 0),
166                 static_cast<size_t>(frame->width()));
167
168   CVReturn cvRet = CVPixelBufferLockBaseAddress(pixel_buffer, 0);
169   if (cvRet != kCVReturnSuccess) {
170     LOG(LS_ERROR) << "Failed to lock base address: " << cvRet;
171     return false;
172   }
173   uint8_t* dst_y = reinterpret_cast<uint8_t*>(
174       CVPixelBufferGetBaseAddressOfPlane(pixel_buffer, 0));
175   int dst_stride_y = CVPixelBufferGetBytesPerRowOfPlane(pixel_buffer, 0);
176   uint8_t* dst_uv = reinterpret_cast<uint8_t*>(
177       CVPixelBufferGetBaseAddressOfPlane(pixel_buffer, 1));
178   int dst_stride_uv = CVPixelBufferGetBytesPerRowOfPlane(pixel_buffer, 1);
179   // Convert I420 to NV12.
180   int ret = libyuv::I420ToNV12(
181       frame->DataY(), frame->StrideY(),
182       frame->DataU(), frame->StrideU(),
183       frame->DataV(), frame->StrideV(),
184       dst_y, dst_stride_y, dst_uv, dst_stride_uv,
185       frame->width(), frame->height());
186   CVPixelBufferUnlockBaseAddress(pixel_buffer, 0);
187   if (ret) {
188     LOG(LS_ERROR) << "Error converting I420 VideoFrame to NV12 :" << ret;
189     return false;
190   }
191   return true;
192 }
193
194 static CVPixelBufferRef CreatePixelBuffer(CVPixelBufferPoolRef pixel_buffer_pool) {
195   if (!pixel_buffer_pool) {
196     LOG(LS_ERROR) << "Failed to get pixel buffer pool.";
197     return nullptr;
198   }
199   CVPixelBufferRef pixel_buffer;
200   CVReturn ret = CVPixelBufferPoolCreatePixelBuffer(nullptr, pixel_buffer_pool,
201                                                     &pixel_buffer);
202   if (ret != kCVReturnSuccess) {
203     LOG(LS_ERROR) << "Failed to create pixel buffer: " << ret;
204     // We probably want to drop frames here, since failure probably means
205     // that the pool is empty.
206     return nullptr;
207   }
208   return pixel_buffer;
209 }
210
211 // This is the callback function that VideoToolbox calls when encode is
212 // complete. From inspection this happens on its own queue.
213 static void VTCompressionOutputCallback(void* encoder,
214                                  void* params,
215                                  OSStatus status,
216                                  VTEncodeInfoFlags info_flags,
217                                  CMSampleBufferRef sample_buffer) {
218   std::unique_ptr<FrameEncodeParams> encode_params(
219       reinterpret_cast<FrameEncodeParams*>(params));
220   encode_params->encoder->OnEncodedFrame(
221       status, info_flags, sample_buffer, encode_params->codec_specific_info,
222       encode_params->width, encode_params->height,
223       encode_params->render_time_ms, encode_params->timestamp,
224       encode_params->rotation);
225 }
226
227 // Extract VideoToolbox profile out of the cricket::VideoCodec. If there is no
228 // specific VideoToolbox profile for the specified level, AutoLevel will be
229 // returned. The user must initialize the encoder with a resolution and
230 // framerate conforming to the selected H264 level regardless.
231 static CFStringRef ExtractProfile(const cricket::VideoCodec& codec) {
232   const rtc::Optional<webrtc::H264::ProfileLevelId> profile_level_id =
233       webrtc::H264::ParseSdpProfileLevelId(codec.params);
234   RTC_DCHECK(profile_level_id);
235   switch (profile_level_id->profile) {
236     case webrtc::H264::kProfileConstrainedBaseline:
237     case webrtc::H264::kProfileBaseline:
238       switch (profile_level_id->level) {
239         case webrtc::H264::kLevel3:
240           return kVTProfileLevel_H264_Baseline_3_0;
241         case webrtc::H264::kLevel3_1:
242           return kVTProfileLevel_H264_Baseline_3_1;
243         case webrtc::H264::kLevel3_2:
244           return kVTProfileLevel_H264_Baseline_3_2;
245         case webrtc::H264::kLevel4:
246           return kVTProfileLevel_H264_Baseline_4_0;
247         case webrtc::H264::kLevel4_1:
248           return kVTProfileLevel_H264_Baseline_4_1;
249         case webrtc::H264::kLevel4_2:
250           return kVTProfileLevel_H264_Baseline_4_2;
251         case webrtc::H264::kLevel5:
252           return kVTProfileLevel_H264_Baseline_5_0;
253         case webrtc::H264::kLevel5_1:
254           return kVTProfileLevel_H264_Baseline_5_1;
255         case webrtc::H264::kLevel5_2:
256           return kVTProfileLevel_H264_Baseline_5_2;
257         case webrtc::H264::kLevel1:
258         case webrtc::H264::kLevel1_b:
259         case webrtc::H264::kLevel1_1:
260         case webrtc::H264::kLevel1_2:
261         case webrtc::H264::kLevel1_3:
262         case webrtc::H264::kLevel2:
263         case webrtc::H264::kLevel2_1:
264         case webrtc::H264::kLevel2_2:
265           return kVTProfileLevel_H264_Baseline_AutoLevel;
266       }
267
268     case webrtc::H264::kProfileMain:
269       switch (profile_level_id->level) {
270         case webrtc::H264::kLevel3:
271           return kVTProfileLevel_H264_Main_3_0;
272         case webrtc::H264::kLevel3_1:
273           return kVTProfileLevel_H264_Main_3_1;
274         case webrtc::H264::kLevel3_2:
275           return kVTProfileLevel_H264_Main_3_2;
276         case webrtc::H264::kLevel4:
277           return kVTProfileLevel_H264_Main_4_0;
278         case webrtc::H264::kLevel4_1:
279           return kVTProfileLevel_H264_Main_4_1;
280         case webrtc::H264::kLevel4_2:
281           return kVTProfileLevel_H264_Main_4_2;
282         case webrtc::H264::kLevel5:
283           return kVTProfileLevel_H264_Main_5_0;
284         case webrtc::H264::kLevel5_1:
285           return kVTProfileLevel_H264_Main_5_1;
286         case webrtc::H264::kLevel5_2:
287           return kVTProfileLevel_H264_Main_5_2;
288         case webrtc::H264::kLevel1:
289         case webrtc::H264::kLevel1_b:
290         case webrtc::H264::kLevel1_1:
291         case webrtc::H264::kLevel1_2:
292         case webrtc::H264::kLevel1_3:
293         case webrtc::H264::kLevel2:
294         case webrtc::H264::kLevel2_1:
295         case webrtc::H264::kLevel2_2:
296           return kVTProfileLevel_H264_Main_AutoLevel;
297       }
298
299     case webrtc::H264::kProfileConstrainedHigh:
300     case webrtc::H264::kProfileHigh:
301       switch (profile_level_id->level) {
302         case webrtc::H264::kLevel3:
303           return kVTProfileLevel_H264_High_3_0;
304         case webrtc::H264::kLevel3_1:
305           return kVTProfileLevel_H264_High_3_1;
306         case webrtc::H264::kLevel3_2:
307           return kVTProfileLevel_H264_High_3_2;
308         case webrtc::H264::kLevel4:
309           return kVTProfileLevel_H264_High_4_0;
310         case webrtc::H264::kLevel4_1:
311           return kVTProfileLevel_H264_High_4_1;
312         case webrtc::H264::kLevel4_2:
313           return kVTProfileLevel_H264_High_4_2;
314         case webrtc::H264::kLevel5:
315           return kVTProfileLevel_H264_High_5_0;
316         case webrtc::H264::kLevel5_1:
317           return kVTProfileLevel_H264_High_5_1;
318         case webrtc::H264::kLevel5_2:
319           return kVTProfileLevel_H264_High_5_2;
320         case webrtc::H264::kLevel1:
321         case webrtc::H264::kLevel1_b:
322         case webrtc::H264::kLevel1_1:
323         case webrtc::H264::kLevel1_2:
324         case webrtc::H264::kLevel1_3:
325         case webrtc::H264::kLevel2:
326         case webrtc::H264::kLevel2_1:
327         case webrtc::H264::kLevel2_2:
328           return kVTProfileLevel_H264_High_AutoLevel;
329       }
330   }
331 }
332
333 }  // namespace internal
334
335 namespace webrtc {
336
337 // .5 is set as a mininum to prevent overcompensating for large temporary
338 // overshoots. We don't want to degrade video quality too badly.
339 // .95 is set to prevent oscillations. When a lower bitrate is set on the
340 // encoder than previously set, its output seems to have a brief period of
341 // drastically reduced bitrate, so we want to avoid that. In steady state
342 // conditions, 0.95 seems to give us better overall bitrate over long periods
343 // of time.
344 H264VideoToolboxEncoder::H264VideoToolboxEncoder(const cricket::VideoCodec& codec)
345     : callback_(nullptr),
346       compression_session_(nullptr),
347       bitrate_adjuster_(Clock::GetRealTimeClock(), .5, .95),
348       packetization_mode_(H264PacketizationMode::NonInterleaved),
349       profile_(internal::ExtractProfile(codec)) {
350   LOG(LS_INFO) << "Using profile " << internal::CFStringToString(profile_);
351   RTC_CHECK(cricket::CodecNamesEq(codec.name, cricket::kH264CodecName));
352 }
353
354 H264VideoToolboxEncoder::~H264VideoToolboxEncoder() {
355   DestroyCompressionSession();
356 }
357
358 int H264VideoToolboxEncoder::InitEncode(const VideoCodec* codec_settings,
359                                         int number_of_cores,
360                                         size_t max_payload_size) {
361   RTC_DCHECK(codec_settings);
362   RTC_DCHECK_EQ(codec_settings->codecType, kVideoCodecH264);
363
364   width_ = codec_settings->width;
365   height_ = codec_settings->height;
366   mode_ = codec_settings->mode;
367   // We can only set average bitrate on the HW encoder.
368   target_bitrate_bps_ = codec_settings->startBitrate;
369   bitrate_adjuster_.SetTargetBitrateBps(target_bitrate_bps_);
370
371   // TODO(tkchin): Try setting payload size via
372   // kVTCompressionPropertyKey_MaxH264SliceBytes.
373
374   return ResetCompressionSession();
375 }
376
377 int H264VideoToolboxEncoder::Encode(
378     const VideoFrame& frame,
379     const CodecSpecificInfo* codec_specific_info,
380     const std::vector<FrameType>* frame_types) {
381   // |input_frame| size should always match codec settings.
382   RTC_DCHECK_EQ(frame.width(), width_);
383   RTC_DCHECK_EQ(frame.height(), height_);
384   if (!callback_ || !compression_session_) {
385     return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
386   }
387
388 #if defined(WEBRTC_IOS)
389 #if !defined(WEBRTC_WEBKIT_BUILD)
390     if (![[RTCUIApplicationStatusObserver sharedInstance] isApplicationActive]) {
391 #else
392     if (!is_active_) {
393 #endif
394     // Ignore all encode requests when app isn't active. In this state, the
395     // hardware encoder has been invalidated by the OS.
396     return WEBRTC_VIDEO_CODEC_OK;
397   }
398 #endif
399   bool is_keyframe_required = false;
400
401   // Get a pixel buffer from the pool and copy frame data over.
402   CVPixelBufferPoolRef pixel_buffer_pool =
403       VTCompressionSessionGetPixelBufferPool(compression_session_);
404 #if defined(WEBRTC_IOS)
405   if (!pixel_buffer_pool) {
406     // Kind of a hack. On backgrounding, the compression session seems to get
407     // invalidated, which causes this pool call to fail when the application
408     // is foregrounded and frames are being sent for encoding again.
409     // Resetting the session when this happens fixes the issue.
410     // In addition we request a keyframe so video can recover quickly.
411     ResetCompressionSession();
412     pixel_buffer_pool =
413         VTCompressionSessionGetPixelBufferPool(compression_session_);
414     is_keyframe_required = true;
415     LOG(LS_INFO) << "Resetting compression session due to invalid pool.";
416   }
417 #endif
418
419   CVPixelBufferRef pixel_buffer;
420   if (frame.video_frame_buffer()->type() == VideoFrameBuffer::Type::kNative) {
421     rtc::scoped_refptr<CoreVideoFrameBuffer> core_video_frame_buffer(
422         static_cast<CoreVideoFrameBuffer*>(frame.video_frame_buffer().get()));
423     if (!core_video_frame_buffer->RequiresCropping()) {
424       pixel_buffer = core_video_frame_buffer->pixel_buffer();
425       // This pixel buffer might have a higher resolution than what the
426       // compression session is configured to. The compression session can
427       // handle that and will output encoded frames in the configured
428       // resolution regardless of the input pixel buffer resolution.
429       CVBufferRetain(pixel_buffer);
430     } else {
431       // Cropping required, we need to crop and scale to a new pixel buffer.
432       pixel_buffer = internal::CreatePixelBuffer(pixel_buffer_pool);
433       if (!pixel_buffer) {
434         return WEBRTC_VIDEO_CODEC_ERROR;
435       }
436       if (!core_video_frame_buffer->CropAndScaleTo(&nv12_scale_buffer_,
437                                                    pixel_buffer)) {
438         return WEBRTC_VIDEO_CODEC_ERROR;
439       }
440     }
441   } else {
442     pixel_buffer = internal::CreatePixelBuffer(pixel_buffer_pool);
443     if (!pixel_buffer) {
444       return WEBRTC_VIDEO_CODEC_ERROR;
445     }
446     RTC_DCHECK(pixel_buffer);
447     if (!internal::CopyVideoFrameToPixelBuffer(frame.video_frame_buffer()->ToI420(),
448                                                pixel_buffer)) {
449       LOG(LS_ERROR) << "Failed to copy frame data.";
450       CVBufferRelease(pixel_buffer);
451       return WEBRTC_VIDEO_CODEC_ERROR;
452     }
453   }
454
455   // Check if we need a keyframe.
456   if (!is_keyframe_required && frame_types) {
457     for (auto frame_type : *frame_types) {
458       if (frame_type == kVideoFrameKey) {
459         is_keyframe_required = true;
460         break;
461       }
462     }
463   }
464
465   CMTime presentation_time_stamp =
466       CMTimeMake(frame.render_time_ms(), 1000);
467   CFDictionaryRef frame_properties = nullptr;
468   if (is_keyframe_required) {
469     CFTypeRef keys[] = {kVTEncodeFrameOptionKey_ForceKeyFrame};
470     CFTypeRef values[] = {kCFBooleanTrue};
471     frame_properties = internal::CreateCFDictionary(keys, values, 1);
472   }
473   std::unique_ptr<internal::FrameEncodeParams> encode_params;
474   encode_params.reset(new internal::FrameEncodeParams(
475       this, codec_specific_info, width_, height_, frame.render_time_ms(),
476       frame.timestamp(), frame.rotation()));
477
478   encode_params->codec_specific_info.codecSpecific.H264.packetization_mode =
479       packetization_mode_;
480
481   // Update the bitrate if needed.
482   SetBitrateBps(bitrate_adjuster_.GetAdjustedBitrateBps());
483
484   OSStatus status = VTCompressionSessionEncodeFrame(
485       compression_session_, pixel_buffer, presentation_time_stamp,
486       kCMTimeInvalid, frame_properties, encode_params.release(), nullptr);
487   if (frame_properties) {
488     CFRelease(frame_properties);
489   }
490   if (pixel_buffer) {
491     CVBufferRelease(pixel_buffer);
492   }
493   if (status != noErr) {
494     LOG(LS_ERROR) << "Failed to encode frame with code: " << status;
495     return WEBRTC_VIDEO_CODEC_ERROR;
496   }
497   return WEBRTC_VIDEO_CODEC_OK;
498 }
499
500 int H264VideoToolboxEncoder::RegisterEncodeCompleteCallback(
501     EncodedImageCallback* callback) {
502   callback_ = callback;
503   return WEBRTC_VIDEO_CODEC_OK;
504 }
505
506 int H264VideoToolboxEncoder::SetChannelParameters(uint32_t packet_loss,
507                                                   int64_t rtt) {
508   // Encoder doesn't know anything about packet loss or rtt so just return.
509   return WEBRTC_VIDEO_CODEC_OK;
510 }
511
512 int H264VideoToolboxEncoder::SetRates(uint32_t new_bitrate_kbit,
513                                       uint32_t frame_rate) {
514   target_bitrate_bps_ = 1000 * new_bitrate_kbit;
515   bitrate_adjuster_.SetTargetBitrateBps(target_bitrate_bps_);
516   SetBitrateBps(bitrate_adjuster_.GetAdjustedBitrateBps());
517   return WEBRTC_VIDEO_CODEC_OK;
518 }
519
520 int H264VideoToolboxEncoder::Release() {
521   // Need to reset so that the session is invalidated and won't use the
522   // callback anymore. Do not remove callback until the session is invalidated
523   // since async encoder callbacks can occur until invalidation.
524   int ret = ResetCompressionSession();
525   callback_ = nullptr;
526   return ret;
527 }
528
529 int H264VideoToolboxEncoder::ResetCompressionSession() {
530   DestroyCompressionSession();
531
532   int status = CreateCompressionSession(compression_session_, internal::VTCompressionOutputCallback, width_, height_);
533   if (status != noErr) {
534     LOG(LS_ERROR) << "Failed to create compression session: " << status;
535     return WEBRTC_VIDEO_CODEC_ERROR;
536   }
537   ConfigureCompressionSession();
538   return WEBRTC_VIDEO_CODEC_OK;
539 }
540
541 int H264VideoToolboxEncoder::CreateCompressionSession(VTCompressionSessionRef& compressionSession, VTCompressionOutputCallback outputCallback, int32_t width, int32_t height, bool useHardwareEncoder) {
542
543   // Set source image buffer attributes. These attributes will be present on
544   // buffers retrieved from the encoder's pixel buffer pool.
545   const size_t attributes_size = 3;
546   CFTypeRef keys[attributes_size] = {
547 #if defined(WEBRTC_IOS)
548     kCVPixelBufferOpenGLESCompatibilityKey,
549 #elif defined(WEBRTC_MAC)
550     kCVPixelBufferOpenGLCompatibilityKey,
551 #endif
552     kCVPixelBufferIOSurfacePropertiesKey,
553     kCVPixelBufferPixelFormatTypeKey
554   };
555   CFDictionaryRef io_surface_value =
556       internal::CreateCFDictionary(nullptr, nullptr, 0);
557   int64_t nv12type = kCVPixelFormatType_420YpCbCr8BiPlanarFullRange;
558   CFNumberRef pixel_format =
559       CFNumberCreate(nullptr, kCFNumberLongType, &nv12type);
560   CFTypeRef values[attributes_size] = {kCFBooleanTrue, io_surface_value,
561                                        pixel_format};
562   CFDictionaryRef source_attributes =
563       internal::CreateCFDictionary(keys, values, attributes_size);
564   if (io_surface_value) {
565     CFRelease(io_surface_value);
566     io_surface_value = nullptr;
567   }
568   if (pixel_format) {
569     CFRelease(pixel_format);
570     pixel_format = nullptr;
571   }
572
573 #if defined(WEBRTC_USE_VTB_HARDWARE_ENCODER)
574   CFTypeRef sessionKeys[] = {kVTVideoEncoderSpecification_EnableHardwareAcceleratedVideoEncoder};
575   CFTypeRef sessionValues[] = { useHardwareEncoder ? kCFBooleanTrue : kCFBooleanFalse };
576   CFDictionaryRef encoderSpecification = internal::CreateCFDictionary(sessionKeys, sessionValues, 1);
577 #else
578   CFDictionaryRef encoderSpecification = nullptr;
579 #endif
580
581   OSStatus status = VTCompressionSessionCreate(
582       nullptr,  // use default allocator
583       width, height, kCMVideoCodecType_H264,
584       encoderSpecification,  // use default encoder
585       source_attributes,
586       nullptr,  // use default compressed data allocator
587       outputCallback, this, &compression_session_);
588   if (source_attributes) {
589     CFRelease(source_attributes);
590     source_attributes = nullptr;
591   }
592
593 #if defined(WEBRTC_USE_VTB_HARDWARE_ENCODER)
594   if (encoderSpecification) {
595     CFRelease(encoderSpecification);
596     encoderSpecification = nullptr;
597   }
598 #endif
599   return status;
600 }
601
602 void H264VideoToolboxEncoder::ConfigureCompressionSession() {
603   RTC_DCHECK(compression_session_);
604   internal::SetVTSessionProperty(compression_session_,
605                                  kVTCompressionPropertyKey_RealTime, true);
606   internal::SetVTSessionProperty(compression_session_,
607                                  kVTCompressionPropertyKey_ProfileLevel,
608                                  profile_);
609   internal::SetVTSessionProperty(compression_session_,
610                                  kVTCompressionPropertyKey_AllowFrameReordering,
611                                  false);
612   SetEncoderBitrateBps(target_bitrate_bps_);
613   // TODO(tkchin): Look at entropy mode and colorspace matrices.
614   // TODO(tkchin): Investigate to see if there's any way to make this work.
615   // May need it to interop with Android. Currently this call just fails.
616   // On inspecting encoder output on iOS8, this value is set to 6.
617   // internal::SetVTSessionProperty(compression_session_,
618   //     kVTCompressionPropertyKey_MaxFrameDelayCount,
619   //     1);
620
621   // Set a relatively large value for keyframe emission (7200 frames or
622   // 4 minutes).
623   internal::SetVTSessionProperty(
624       compression_session_,
625       kVTCompressionPropertyKey_MaxKeyFrameInterval, 7200);
626   internal::SetVTSessionProperty(
627       compression_session_,
628       kVTCompressionPropertyKey_MaxKeyFrameIntervalDuration, 240);
629 }
630
631 void H264VideoToolboxEncoder::DestroyCompressionSession() {
632   if (compression_session_) {
633     VTCompressionSessionInvalidate(compression_session_);
634     CFRelease(compression_session_);
635     compression_session_ = nullptr;
636   }
637 }
638
639 const char* H264VideoToolboxEncoder::ImplementationName() const {
640   return "VideoToolbox";
641 }
642
643 bool H264VideoToolboxEncoder::SupportsNativeHandle() const {
644   return true;
645 }
646
647 void H264VideoToolboxEncoder::SetBitrateBps(uint32_t bitrate_bps) {
648   if (encoder_bitrate_bps_ != bitrate_bps) {
649     SetEncoderBitrateBps(bitrate_bps);
650   }
651 }
652
653 void H264VideoToolboxEncoder::SetEncoderBitrateBps(uint32_t bitrate_bps) {
654   if (compression_session_) {
655     internal::SetVTSessionProperty(compression_session_,
656                                    kVTCompressionPropertyKey_AverageBitRate,
657                                    bitrate_bps);
658
659     // TODO(tkchin): Add a helper method to set array value.
660     int64_t data_limit_bytes_per_second_value = static_cast<int64_t>(
661         bitrate_bps * internal::kLimitToAverageBitRateFactor / 8);
662     CFNumberRef bytes_per_second =
663         CFNumberCreate(kCFAllocatorDefault,
664                        kCFNumberSInt64Type,
665                        &data_limit_bytes_per_second_value);
666     int64_t one_second_value = 1;
667     CFNumberRef one_second =
668         CFNumberCreate(kCFAllocatorDefault,
669                        kCFNumberSInt64Type,
670                        &one_second_value);
671     const void* nums[2] = { bytes_per_second, one_second };
672     CFArrayRef data_rate_limits =
673         CFArrayCreate(nullptr, nums, 2, &kCFTypeArrayCallBacks);
674     OSStatus status =
675         VTSessionSetProperty(compression_session_,
676                              kVTCompressionPropertyKey_DataRateLimits,
677                              data_rate_limits);
678     if (bytes_per_second) {
679       CFRelease(bytes_per_second);
680     }
681     if (one_second) {
682       CFRelease(one_second);
683     }
684     if (data_rate_limits) {
685       CFRelease(data_rate_limits);
686     }
687     if (status != noErr) {
688       LOG(LS_ERROR) << "Failed to set data rate limit";
689     }
690
691     encoder_bitrate_bps_ = bitrate_bps;
692   }
693 }
694
695 void H264VideoToolboxEncoder::OnEncodedFrame(
696     OSStatus status,
697     VTEncodeInfoFlags info_flags,
698     CMSampleBufferRef sample_buffer,
699     CodecSpecificInfo codec_specific_info,
700     int32_t width,
701     int32_t height,
702     int64_t render_time_ms,
703     uint32_t timestamp,
704     VideoRotation rotation) {
705   if (status != noErr) {
706     LOG(LS_ERROR) << "H264 encode failed.";
707     return;
708   }
709   if (info_flags & kVTEncodeInfo_FrameDropped) {
710     LOG(LS_INFO) << "H264 encode dropped frame.";
711     return;
712   }
713
714   bool is_keyframe = false;
715   CFArrayRef attachments =
716       CMSampleBufferGetSampleAttachmentsArray(sample_buffer, 0);
717   if (attachments != nullptr && CFArrayGetCount(attachments)) {
718     CFDictionaryRef attachment =
719         static_cast<CFDictionaryRef>(CFArrayGetValueAtIndex(attachments, 0));
720     is_keyframe =
721         !CFDictionaryContainsKey(attachment, kCMSampleAttachmentKey_NotSync);
722   }
723
724   if (is_keyframe) {
725     LOG(LS_INFO) << "Generated keyframe";
726   }
727
728   // Convert the sample buffer into a buffer suitable for RTP packetization.
729   // TODO(tkchin): Allocate buffers through a pool.
730   std::unique_ptr<rtc::Buffer> buffer(new rtc::Buffer());
731   std::unique_ptr<webrtc::RTPFragmentationHeader> header;
732   {
733     webrtc::RTPFragmentationHeader* header_raw;
734     bool result = H264CMSampleBufferToAnnexBBuffer(sample_buffer, is_keyframe,
735                                                    buffer.get(), &header_raw);
736     header.reset(header_raw);
737     if (!result) {
738       return;
739     }
740   }
741   webrtc::EncodedImage frame(buffer->data(), buffer->size(), buffer->size());
742   frame._encodedWidth = width;
743   frame._encodedHeight = height;
744   frame._completeFrame = true;
745   frame._frameType =
746       is_keyframe ? webrtc::kVideoFrameKey : webrtc::kVideoFrameDelta;
747   frame.capture_time_ms_ = render_time_ms;
748   frame._timeStamp = timestamp;
749   frame.rotation_ = rotation;
750   frame.content_type_ =
751       (mode_ == kScreensharing) ? VideoContentType::SCREENSHARE : VideoContentType::UNSPECIFIED;
752   frame.timing_.is_timing_frame = false;
753
754   h264_bitstream_parser_.ParseBitstream(buffer->data(), buffer->size());
755   h264_bitstream_parser_.GetLastSliceQp(&frame.qp_);
756
757   EncodedImageCallback::Result res =
758       callback_->OnEncodedImage(frame, &codec_specific_info, header.get());
759   if (res.error != EncodedImageCallback::Result::OK) {
760     LOG(LS_ERROR) << "Encode callback failed: " << res.error;
761     return;
762   }
763   bitrate_adjuster_.Update(frame._length);
764 }
765
766 VideoEncoder::ScalingSettings H264VideoToolboxEncoder::GetScalingSettings()
767     const {
768   return VideoEncoder::ScalingSettings(true, internal::kLowH264QpThreshold,
769                                        internal::kHighH264QpThreshold);
770 }
771 }  // namespace webrtc