[WebRTC][GStreamer] Build and use the openh264 based encoder if present on the system
[WebKit-https.git] / Source / WebCore / platform / mediastream / libwebrtc / GStreamerVideoEncoderFactory.cpp
1 /*
2  * Copyright (C) 2018 Metrological Group B.V.
3  * Copyright (C) 2018 Igalia S.L. All rights reserved.
4  *
5  * This library is free software; you can redistribute it and/or
6  * modify it under the terms of the GNU Library General Public
7  * License as published by the Free Software Foundation; either
8  * version 2 of the License, or (at your option) any later version.
9  *
10  * This library is distributed in the hope that it will be useful,
11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
13  * Library General Public License for more details.
14  *
15  * You should have received a copy of the GNU Library General Public License
16  * aint with this library; see the file COPYING.LIB.  If not, write to
17  * the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
18  * Boston, MA 02110-1301, USA.
19  */
20
21 #include "config.h"
22
23 #if ENABLE(VIDEO) && ENABLE(MEDIA_STREAM) && USE(LIBWEBRTC) && USE(GSTREAMER)
24 #include "GStreamerVideoEncoderFactory.h"
25
26 #include "GStreamerVideoCommon.h"
27 #include "GStreamerVideoEncoder.h"
28 #include "GStreamerVideoFrameLibWebRTC.h"
29 #include "LibWebRTCWebKitMacros.h"
30 #include "webrtc/common_video/h264/h264_common.h"
31 #include "webrtc/modules/video_coding/codecs/h264/include/h264.h"
32 #include "webrtc/modules/video_coding/codecs/vp8/include/vp8.h"
33 #include "webrtc/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.h"
34 #include "webrtc/modules/video_coding/include/video_codec_interface.h"
35 #include "webrtc/modules/video_coding/utility/simulcast_utility.h"
36
37 #include <gst/app/gstappsink.h>
38 #include <gst/app/gstappsrc.h>
39
40 #define GST_USE_UNSTABLE_API
41 #include <gst/codecparsers/gsth264parser.h>
42 #undef GST_USE_UNSTABLE_API
43
44 #include <gst/pbutils/encoding-profile.h>
45 #include <gst/video/video.h>
46 #include <wtf/Atomics.h>
47 #include <wtf/HashMap.h>
48 #include <wtf/Lock.h>
49 #include <wtf/StdMap.h>
50 #include <wtf/text/StringConcatenateNumbers.h>
51
52 GST_DEBUG_CATEGORY(webkit_webrtcenc_debug);
53 #define GST_CAT_DEFAULT webkit_webrtcenc_debug
54
55 namespace WebCore {
56
57 class GStreamerEncodedImageBuffer : public webrtc::EncodedImageBufferInterface {
58     WTF_MAKE_FAST_ALLOCATED;
59
60 public:
61     static rtc::scoped_refptr<GStreamerEncodedImageBuffer> create(GRefPtr<GstSample>&& sample)
62     {
63         return new rtc::RefCountedObject<GStreamerEncodedImageBuffer>(WTFMove(sample));
64     }
65
66     const uint8_t* data() const final { return m_mappedBuffer->data(); }
67     uint8_t* data() final { return m_mappedBuffer->data(); }
68     size_t size() const final { return m_mappedBuffer->size(); }
69
70     const GstBuffer* getBuffer() const { return gst_sample_get_buffer(m_sample.get()); }
71     Optional<FloatSize> getVideoResolution() const { return getVideoResolutionFromCaps(gst_sample_get_caps(m_sample.get())); }
72
73 protected:
74     GStreamerEncodedImageBuffer() = default;
75     ~GStreamerEncodedImageBuffer() = default;
76     GStreamerEncodedImageBuffer(GRefPtr<GstSample>&& sample)
77         : m_sample(sample)
78     {
79         m_mappedBuffer = GstMappedOwnedBuffer::create(gst_sample_get_buffer(m_sample.get()));
80     }
81
82     GRefPtr<GstSample> m_sample;
83     RefPtr<GstMappedOwnedBuffer> m_mappedBuffer;
84 };
85
86 class GStreamerVideoEncoder : public webrtc::VideoEncoder {
87     WTF_MAKE_FAST_ALLOCATED;
88 public:
89     GStreamerVideoEncoder(const webrtc::SdpVideoFormat&)
90         : m_firstFramePts(GST_CLOCK_TIME_NONE)
91         , m_restrictionCaps(adoptGRef(gst_caps_new_empty_simple("video/x-raw")))
92     {
93     }
94     GStreamerVideoEncoder()
95         : m_firstFramePts(GST_CLOCK_TIME_NONE)
96         , m_restrictionCaps(adoptGRef(gst_caps_new_empty_simple("video/x-raw")))
97     {
98     }
99
100     void SetRates(const webrtc::VideoEncoder::RateControlParameters& parameters) override
101     {
102         GST_INFO_OBJECT(m_pipeline.get(), "New bitrate: %d - framerate is %f",
103             parameters.bitrate.get_sum_bps(), parameters.framerate_fps);
104
105         auto caps = adoptGRef(gst_caps_copy(m_restrictionCaps.get()));
106
107         SetRestrictionCaps(WTFMove(caps));
108
109         if (m_encoder)
110             g_object_set(m_encoder, "bitrate", parameters.bitrate.get_sum_bps(), nullptr);
111     }
112
113     GstElement* pipeline()
114     {
115         return m_pipeline.get();
116     }
117
118     GstElement* makeElement(const gchar* factoryName)
119     {
120         static Atomic<uint32_t> elementId;
121         auto name = makeString(Name(), "-enc-", factoryName, "-", elementId.exchangeAdd(1));
122         auto elem = gst_element_factory_make(factoryName, name.utf8().data());
123
124         return elem;
125     }
126
127     int32_t InitEncode(const webrtc::VideoCodec* codecSettings, int32_t, size_t) override
128     {
129         g_return_val_if_fail(codecSettings, WEBRTC_VIDEO_CODEC_ERR_PARAMETER);
130         g_return_val_if_fail(codecSettings->codecType == CodecType(), WEBRTC_VIDEO_CODEC_ERR_PARAMETER);
131
132         if (webrtc::SimulcastUtility::NumberOfSimulcastStreams(*codecSettings) > 1) {
133             GST_ERROR("Simulcast not supported.");
134
135             return WEBRTC_VIDEO_CODEC_ERR_SIMULCAST_PARAMETERS_NOT_SUPPORTED;
136         }
137
138         m_pipeline = makeElement("pipeline");
139
140         connectSimpleBusMessageCallback(m_pipeline.get());
141         auto encoder = createEncoder();
142         ASSERT(encoder);
143         m_encoder = encoder.get();
144
145         g_object_set(m_encoder, "keyframe-interval", KeyframeInterval(codecSettings), nullptr);
146
147         m_src = makeElement("appsrc");
148         g_object_set(m_src, "is-live", true, "format", GST_FORMAT_TIME, nullptr);
149
150         auto videoconvert = makeElement("videoconvert");
151         auto videoscale = makeElement("videoscale");
152         m_sink = makeElement("appsink");
153         g_object_set(m_sink, "sync", FALSE, nullptr);
154
155         m_capsFilter = makeElement("capsfilter");
156         if (m_restrictionCaps)
157             g_object_set(m_capsFilter, "caps", m_restrictionCaps.get(), nullptr);
158
159         gst_bin_add_many(GST_BIN(m_pipeline.get()), m_src, videoconvert, videoscale, m_capsFilter, encoder.leakRef(), m_sink, nullptr);
160         if (!gst_element_link_many(m_src, videoconvert, videoscale, m_capsFilter, m_encoder, m_sink, nullptr)) {
161             GST_DEBUG_BIN_TO_DOT_FILE_WITH_TS(GST_BIN(m_pipeline.get()), GST_DEBUG_GRAPH_SHOW_VERBOSE, "webkit-webrtc-encoder.error");
162
163             ASSERT_NOT_REACHED();
164         }
165
166         gst_element_set_state(m_pipeline.get(), GST_STATE_PLAYING);
167
168         return WEBRTC_VIDEO_CODEC_OK;
169     }
170
171     int32_t RegisterEncodeCompleteCallback(webrtc::EncodedImageCallback* callback) final
172     {
173         m_imageReadyCb = callback;
174
175         return WEBRTC_VIDEO_CODEC_OK;
176     }
177
178     int32_t Release() final
179     {
180         m_encodedFrame.ClearEncodedData();
181         if (m_pipeline) {
182             disconnectSimpleBusMessageCallback(m_pipeline.get());
183             gst_element_set_state(m_pipeline.get(), GST_STATE_NULL);
184             m_src = nullptr;
185             m_encoder = nullptr;
186             m_capsFilter = nullptr;
187             m_sink = nullptr;
188             m_pipeline = nullptr;
189         }
190
191         return WEBRTC_VIDEO_CODEC_OK;
192     }
193
194     int32_t returnFromFlowReturn(GstFlowReturn flow)
195     {
196         switch (flow) {
197         case GST_FLOW_OK:
198             return WEBRTC_VIDEO_CODEC_OK;
199         case GST_FLOW_FLUSHING:
200             return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
201         default:
202             return WEBRTC_VIDEO_CODEC_ERROR;
203         }
204     }
205
206     VideoEncoder::EncoderInfo GetEncoderInfo() const override
207     {
208         EncoderInfo info;
209         info.supports_native_handle = false;
210         info.implementation_name = "GStreamer";
211         info.has_trusted_rate_controller = true;
212         info.is_hardware_accelerated = true;
213         info.has_internal_source = false;
214         return info;
215     }
216
217     int32_t Encode(const webrtc::VideoFrame& frame,
218         const std::vector<webrtc::VideoFrameType>* frameTypes) final
219     {
220         int32_t res;
221
222         if (!m_imageReadyCb) {
223             GST_INFO_OBJECT(m_pipeline.get(), "No encoded callback set yet!");
224
225             return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
226         }
227
228         if (!m_src) {
229             GST_INFO_OBJECT(m_pipeline.get(), "No source set yet!");
230
231             return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
232         }
233
234         auto sample = GStreamerSampleFromLibWebRTCVideoFrame(frame);
235         auto buffer = gst_sample_get_buffer(sample.get());
236
237         if (!GST_CLOCK_TIME_IS_VALID(m_firstFramePts)) {
238             m_firstFramePts = GST_BUFFER_PTS(buffer);
239             auto pad = adoptGRef(gst_element_get_static_pad(m_src, "src"));
240             gst_pad_set_offset(pad.get(), -m_firstFramePts);
241         }
242
243         for (auto frame_type : *frameTypes) {
244             if (frame_type == webrtc::VideoFrameType::kVideoFrameKey) {
245                 auto pad = adoptGRef(gst_element_get_static_pad(m_src, "src"));
246                 auto forceKeyUnit = gst_video_event_new_downstream_force_key_unit(GST_CLOCK_TIME_NONE,
247                     GST_CLOCK_TIME_NONE, GST_CLOCK_TIME_NONE, FALSE, 1);
248                 GST_INFO_OBJECT(m_pipeline.get(), "Requesting KEYFRAME!");
249
250                 if (!gst_pad_push_event(pad.get(), forceKeyUnit))
251                     GST_WARNING_OBJECT(pipeline(), "Could not send ForceKeyUnit event");
252
253                 break;
254             }
255         }
256
257         res = returnFromFlowReturn(gst_app_src_push_sample(GST_APP_SRC(m_src), sample.get()));
258         if (res != WEBRTC_VIDEO_CODEC_OK)
259             return res;
260
261         auto encodedSample = adoptGRef(gst_app_sink_try_pull_sample(GST_APP_SINK(m_sink), 5 * GST_SECOND));
262         if (!encodedSample) {
263             GST_ERROR("Didn't get any encodedSample");
264             return WEBRTC_VIDEO_CODEC_ERROR;
265         }
266
267         auto encodedData = GStreamerEncodedImageBuffer::create(WTFMove(encodedSample));
268         const auto* encodedBuffer = encodedData->getBuffer();
269         auto resolution = encodedData->getVideoResolution();
270         m_encodedFrame.SetEncodedData(encodedData);
271         if (!m_encodedFrame.size())
272             return WEBRTC_VIDEO_CODEC_OK;
273
274         ASSERT(resolution);
275         m_encodedFrame._encodedWidth = resolution->width();
276         m_encodedFrame._encodedHeight = resolution->height();
277         m_encodedFrame._frameType = GST_BUFFER_FLAG_IS_SET(encodedBuffer, GST_BUFFER_FLAG_DELTA_UNIT) ? webrtc::VideoFrameType::kVideoFrameDelta : webrtc::VideoFrameType::kVideoFrameKey;
278         m_encodedFrame._completeFrame = true;
279         m_encodedFrame.capture_time_ms_ = frame.render_time_ms();
280         m_encodedFrame.SetTimestamp(frame.timestamp());
281
282         GST_LOG_OBJECT(m_pipeline.get(), "Got buffer capture_time_ms: %" G_GINT64_FORMAT  " _timestamp: %u",
283             m_encodedFrame.capture_time_ms_, m_encodedFrame.Timestamp());
284
285         webrtc::CodecSpecificInfo codecInfo;
286         PopulateCodecSpecific(&codecInfo, encodedBuffer);
287         webrtc::EncodedImageCallback::Result result = m_imageReadyCb->OnEncodedImage(m_encodedFrame, &codecInfo);
288         if (result.error != webrtc::EncodedImageCallback::Result::OK)
289             GST_ERROR_OBJECT(m_pipeline.get(), "Encode callback failed: %d", result.error);
290
291         return WEBRTC_VIDEO_CODEC_OK;
292     }
293
294     GRefPtr<GstElement> createEncoder(void)
295     {
296         GRefPtr<GstElement> encoder = nullptr;
297         GstElement* webrtcencoder = GST_ELEMENT(g_object_ref_sink(gst_element_factory_make("webrtcvideoencoder", NULL)));
298
299         g_object_set(webrtcencoder, "format", adoptGRef(gst_caps_from_string(Caps())).get(), NULL);
300         g_object_get(webrtcencoder, "encoder", &encoder.outPtr(), NULL);
301
302         if (!encoder) {
303             GST_INFO("No encoder found for %s", Caps());
304
305             return nullptr;
306         }
307
308         return webrtcencoder;
309     }
310
311     void AddCodecIfSupported(std::vector<webrtc::SdpVideoFormat>& supportedFormats)
312     {
313         if (auto encoder = createEncoder()) {
314             auto formats = ConfigureSupportedCodec();
315             supportedFormats.insert(supportedFormats.end(), formats.begin(), formats.end());
316         }
317     }
318
319     virtual const gchar* Caps()
320     {
321         return nullptr;
322     }
323
324     virtual std::vector<webrtc::SdpVideoFormat> ConfigureSupportedCodec()
325     {
326         return { webrtc::SdpVideoFormat(Name()) };
327     }
328
329     virtual webrtc::VideoCodecType CodecType() = 0;
330     virtual void PopulateCodecSpecific(webrtc::CodecSpecificInfo*, const GstBuffer*) = 0;
331     virtual const gchar* Name() = 0;
332     virtual int KeyframeInterval(const webrtc::VideoCodec* codecSettings) = 0;
333
334     void SetRestrictionCaps(GRefPtr<GstCaps> caps)
335     {
336         if (m_restrictionCaps)
337             g_object_set(m_capsFilter, "caps", m_restrictionCaps.get(), nullptr);
338
339         m_restrictionCaps = caps;
340     }
341
342 private:
343     GRefPtr<GstElement> m_pipeline;
344     GstElement* m_src;
345     GstElement* m_encoder;
346     GstElement* m_capsFilter;
347
348     webrtc::EncodedImageCallback* m_imageReadyCb;
349     GstClockTime m_firstFramePts;
350     GRefPtr<GstCaps> m_restrictionCaps;
351     webrtc::EncodedImage m_encodedFrame;
352
353     Lock m_bufferMapLock;
354     GstElement* m_sink;
355 };
356
357 class GStreamerH264Encoder : public GStreamerVideoEncoder {
358 public:
359     GStreamerH264Encoder() { }
360
361     GStreamerH264Encoder(const webrtc::SdpVideoFormat& format)
362         : m_parser(gst_h264_nal_parser_new())
363         , packetizationMode(webrtc::H264PacketizationMode::NonInterleaved)
364     {
365         auto it = format.parameters.find(cricket::kH264FmtpPacketizationMode);
366
367         if (it != format.parameters.end() && it->second == "1")
368             packetizationMode = webrtc::H264PacketizationMode::NonInterleaved;
369     }
370
371     int KeyframeInterval(const webrtc::VideoCodec* codecSettings) final
372     {
373         return codecSettings->H264().keyFrameInterval;
374     }
375
376     std::vector<webrtc::SdpVideoFormat> ConfigureSupportedCodec() final
377     {
378         return supportedH264Formats();
379     }
380
381     const gchar* Caps() final { return "video/x-h264"; }
382     const gchar* Name() final { return cricket::kH264CodecName; }
383     GstH264NalParser* m_parser;
384     webrtc::VideoCodecType CodecType() final { return webrtc::kVideoCodecH264; }
385
386     void PopulateCodecSpecific(webrtc::CodecSpecificInfo* codecSpecificInfos, const GstBuffer*) final
387     {
388         codecSpecificInfos->codecType = CodecType();
389         webrtc::CodecSpecificInfoH264* h264Info = &(codecSpecificInfos->codecSpecific.H264);
390         h264Info->packetization_mode = packetizationMode;
391     }
392
393     webrtc::H264PacketizationMode packetizationMode;
394 };
395
396 class GStreamerVP8Encoder : public GStreamerVideoEncoder {
397 public:
398     GStreamerVP8Encoder() { }
399     GStreamerVP8Encoder(const webrtc::SdpVideoFormat&) { }
400     const gchar* Caps() final { return "video/x-vp8"; }
401     const gchar* Name() final { return cricket::kVp8CodecName; }
402     webrtc::VideoCodecType CodecType() final { return webrtc::kVideoCodecVP8; }
403
404     int KeyframeInterval(const webrtc::VideoCodec* codecSettings) final
405     {
406         return codecSettings->VP8().keyFrameInterval;
407     }
408
409     void PopulateCodecSpecific(webrtc::CodecSpecificInfo* codecSpecificInfos, const GstBuffer* buffer) final
410     {
411         codecSpecificInfos->codecType = webrtc::kVideoCodecVP8;
412         webrtc::CodecSpecificInfoVP8* vp8Info = &(codecSpecificInfos->codecSpecific.VP8);
413         vp8Info->temporalIdx = 0;
414
415         vp8Info->keyIdx = webrtc::kNoKeyIdx;
416         vp8Info->nonReference = GST_BUFFER_FLAG_IS_SET(buffer, GST_BUFFER_FLAG_DELTA_UNIT);
417     }
418 };
419
420 std::unique_ptr<webrtc::VideoEncoder> GStreamerVideoEncoderFactory::CreateVideoEncoder(const webrtc::SdpVideoFormat& format)
421 {
422     if (format.name == cricket::kVp8CodecName) {
423         GRefPtr<GstElement> webrtcencoder = adoptGRef(GST_ELEMENT(g_object_ref_sink(gst_element_factory_make("webrtcvideoencoder", NULL))));
424         GRefPtr<GstElement> encoder = nullptr;
425
426         g_object_set(webrtcencoder.get(), "format", adoptGRef(gst_caps_from_string("video/x-vp8")).get(), NULL);
427         g_object_get(webrtcencoder.get(), "encoder", &encoder.outPtr(), NULL);
428
429         if (encoder)
430             return makeUnique<GStreamerVP8Encoder>(format);
431
432         GST_INFO("Using VP8 Encoder from LibWebRTC.");
433         return makeUniqueWithoutFastMallocCheck<webrtc::LibvpxVp8Encoder>(webrtc::LibvpxInterface::CreateEncoder(), webrtc::VP8Encoder::Settings());
434     }
435
436     if (format.name == cricket::kH264CodecName) {
437 #if WEBKIT_LIBWEBRTC_OPENH264_ENCODER
438         GST_INFO("Using OpenH264 libwebrtc encoder.");
439         return webrtc::H264Encoder::Create(cricket::VideoCodec(format));
440 #else
441         GST_INFO("Using H264 GStreamer encoder.");
442         return makeUnique<GStreamerH264Encoder>(format);
443 #endif
444     }
445
446     return nullptr;
447 }
448
449 GStreamerVideoEncoderFactory::GStreamerVideoEncoderFactory()
450 {
451     ensureGStreamerInitialized();
452
453     static std::once_flag debugRegisteredFlag;
454     std::call_once(debugRegisteredFlag, [] {
455         GST_DEBUG_CATEGORY_INIT(webkit_webrtcenc_debug, "webkitlibwebrtcvideoencoder", 0, "WebKit WebRTC video encoder");
456         auto factory = adoptGRef(gst_element_factory_find("webrtcvideoencoder"));
457         if (!factory)
458             gst_element_register(nullptr, "webrtcvideoencoder", GST_RANK_NONE, WEBKIT_TYPE_WEBRTC_VIDEO_ENCODER);
459     });
460 }
461
462 std::vector<webrtc::SdpVideoFormat> GStreamerVideoEncoderFactory::GetSupportedFormats() const
463 {
464     std::vector<webrtc::SdpVideoFormat> supportedCodecs;
465
466     supportedCodecs.push_back(webrtc::SdpVideoFormat(cricket::kVp8CodecName));
467
468     // If OpenH264 is present, prefer it over the GStreamer encoders (x264enc, usually).
469 #if WEBKIT_LIBWEBRTC_OPENH264_ENCODER
470     auto formats = supportedH264Formats();
471     supportedCodecs.insert(supportedCodecs.end(), formats.begin(), formats.end());
472 #else
473     GStreamerH264Encoder().AddCodecIfSupported(supportedCodecs);
474 #endif
475
476     return supportedCodecs;
477 }
478
479 } // namespace WebCore
480 #endif