66224ed1dd38170f7726deeb435a854c2f0e807f
[WebKit-https.git] / Source / WebCore / platform / mediastream / libwebrtc / GStreamerVideoEncoderFactory.cpp
1 /*
2  * Copyright (C) 2018 Metrological Group B.V.
3  * Copyright (C) 2018 Igalia S.L. All rights reserved.
4  *
5  * This library is free software; you can redistribute it and/or
6  * modify it under the terms of the GNU Library General Public
7  * License as published by the Free Software Foundation; either
8  * version 2 of the License, or (at your option) any later version.
9  *
10  * This library is distributed in the hope that it will be useful,
11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
13  * Library General Public License for more details.
14  *
15  * You should have received a copy of the GNU Library General Public License
16  * aint with this library; see the file COPYING.LIB.  If not, write to
17  * the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
18  * Boston, MA 02110-1301, USA.
19  */
20
21 #include "config.h"
22
23 #if ENABLE(VIDEO) && ENABLE(MEDIA_STREAM) && USE(LIBWEBRTC) && USE(GSTREAMER)
24 #include "GStreamerVideoEncoderFactory.h"
25
26 #include "GStreamerVideoCommon.h"
27 #include "GStreamerVideoEncoder.h"
28 #include "GStreamerVideoFrameLibWebRTC.h"
29 #include "webrtc/common_video/h264/h264_common.h"
30 #include "webrtc/modules/video_coding/codecs/h264/include/h264.h"
31 #include "webrtc/modules/video_coding/codecs/vp8/include/vp8.h"
32 #include "webrtc/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.h"
33 #include "webrtc/modules/video_coding/include/video_codec_interface.h"
34 #include "webrtc/modules/video_coding/utility/simulcast_utility.h"
35
36 #include <gst/app/gstappsink.h>
37 #include <gst/app/gstappsrc.h>
38
39 #define GST_USE_UNSTABLE_API
40 #include <gst/codecparsers/gsth264parser.h>
41 #undef GST_USE_UNSTABLE_API
42
43 #include <gst/pbutils/encoding-profile.h>
44 #include <gst/video/video.h>
45 #include <wtf/Atomics.h>
46 #include <wtf/HashMap.h>
47 #include <wtf/Lock.h>
48 #include <wtf/StdMap.h>
49 #include <wtf/text/StringConcatenateNumbers.h>
50
51 GST_DEBUG_CATEGORY(webkit_webrtcenc_debug);
52 #define GST_CAT_DEFAULT webkit_webrtcenc_debug
53
54 namespace WebCore {
55
56 class GStreamerEncodedImageBuffer : public webrtc::EncodedImageBufferInterface {
57     WTF_MAKE_FAST_ALLOCATED;
58
59 public:
60     static rtc::scoped_refptr<GStreamerEncodedImageBuffer> create(GRefPtr<GstSample>&& sample)
61     {
62         return new rtc::RefCountedObject<GStreamerEncodedImageBuffer>(WTFMove(sample));
63     }
64
65     const uint8_t* data() const final { return m_mappedBuffer->data(); }
66     uint8_t* data() final { return m_mappedBuffer->data(); }
67     size_t size() const final { return m_mappedBuffer->size(); }
68
69     const GstBuffer* getBuffer() const { return gst_sample_get_buffer(m_sample.get()); }
70     Optional<FloatSize> getVideoResolution() const { return getVideoResolutionFromCaps(gst_sample_get_caps(m_sample.get())); }
71
72 protected:
73     GStreamerEncodedImageBuffer() = default;
74     ~GStreamerEncodedImageBuffer() = default;
75     GStreamerEncodedImageBuffer(GRefPtr<GstSample>&& sample)
76         : m_sample(sample)
77     {
78         m_mappedBuffer = GstMappedOwnedBuffer::create(gst_sample_get_buffer(m_sample.get()));
79     }
80
81     GRefPtr<GstSample> m_sample;
82     RefPtr<GstMappedOwnedBuffer> m_mappedBuffer;
83 };
84
85 class GStreamerVideoEncoder : public webrtc::VideoEncoder {
86     WTF_MAKE_FAST_ALLOCATED;
87 public:
88     GStreamerVideoEncoder(const webrtc::SdpVideoFormat&)
89         : m_firstFramePts(GST_CLOCK_TIME_NONE)
90         , m_restrictionCaps(adoptGRef(gst_caps_new_empty_simple("video/x-raw")))
91     {
92     }
93     GStreamerVideoEncoder()
94         : m_firstFramePts(GST_CLOCK_TIME_NONE)
95         , m_restrictionCaps(adoptGRef(gst_caps_new_empty_simple("video/x-raw")))
96     {
97     }
98
99     void SetRates(const webrtc::VideoEncoder::RateControlParameters& parameters) override
100     {
101         GST_INFO_OBJECT(m_pipeline.get(), "New bitrate: %d - framerate is %f",
102             parameters.bitrate.get_sum_bps(), parameters.framerate_fps);
103
104         auto caps = adoptGRef(gst_caps_copy(m_restrictionCaps.get()));
105
106         SetRestrictionCaps(WTFMove(caps));
107
108         if (m_encoder)
109             g_object_set(m_encoder, "bitrate", parameters.bitrate.get_sum_bps(), nullptr);
110     }
111
112     GstElement* pipeline()
113     {
114         return m_pipeline.get();
115     }
116
117     GstElement* makeElement(const gchar* factoryName)
118     {
119         static Atomic<uint32_t> elementId;
120         auto name = makeString(Name(), "-enc-", factoryName, "-", elementId.exchangeAdd(1));
121         auto elem = gst_element_factory_make(factoryName, name.utf8().data());
122
123         return elem;
124     }
125
126     int32_t InitEncode(const webrtc::VideoCodec* codecSettings, int32_t, size_t) override
127     {
128         g_return_val_if_fail(codecSettings, WEBRTC_VIDEO_CODEC_ERR_PARAMETER);
129         g_return_val_if_fail(codecSettings->codecType == CodecType(), WEBRTC_VIDEO_CODEC_ERR_PARAMETER);
130
131         if (webrtc::SimulcastUtility::NumberOfSimulcastStreams(*codecSettings) > 1) {
132             GST_ERROR("Simulcast not supported.");
133
134             return WEBRTC_VIDEO_CODEC_ERR_SIMULCAST_PARAMETERS_NOT_SUPPORTED;
135         }
136
137         m_pipeline = makeElement("pipeline");
138
139         connectSimpleBusMessageCallback(m_pipeline.get());
140         auto encoder = createEncoder();
141         ASSERT(encoder);
142         m_encoder = encoder.get();
143
144         g_object_set(m_encoder, "keyframe-interval", KeyframeInterval(codecSettings), nullptr);
145
146         m_src = makeElement("appsrc");
147         g_object_set(m_src, "is-live", true, "format", GST_FORMAT_TIME, nullptr);
148
149         auto videoconvert = makeElement("videoconvert");
150         auto videoscale = makeElement("videoscale");
151         m_sink = makeElement("appsink");
152         g_object_set(m_sink, "sync", FALSE, nullptr);
153
154         m_capsFilter = makeElement("capsfilter");
155         if (m_restrictionCaps)
156             g_object_set(m_capsFilter, "caps", m_restrictionCaps.get(), nullptr);
157
158         gst_bin_add_many(GST_BIN(m_pipeline.get()), m_src, videoconvert, videoscale, m_capsFilter, encoder.leakRef(), m_sink, nullptr);
159         if (!gst_element_link_many(m_src, videoconvert, videoscale, m_capsFilter, m_encoder, m_sink, nullptr)) {
160             GST_DEBUG_BIN_TO_DOT_FILE_WITH_TS(GST_BIN(m_pipeline.get()), GST_DEBUG_GRAPH_SHOW_VERBOSE, "webkit-webrtc-encoder.error");
161
162             ASSERT_NOT_REACHED();
163         }
164
165         gst_element_set_state(m_pipeline.get(), GST_STATE_PLAYING);
166
167         return WEBRTC_VIDEO_CODEC_OK;
168     }
169
170     int32_t RegisterEncodeCompleteCallback(webrtc::EncodedImageCallback* callback) final
171     {
172         m_imageReadyCb = callback;
173
174         return WEBRTC_VIDEO_CODEC_OK;
175     }
176
177     int32_t Release() final
178     {
179         m_encodedFrame.ClearEncodedData();
180         if (m_pipeline) {
181             disconnectSimpleBusMessageCallback(m_pipeline.get());
182             gst_element_set_state(m_pipeline.get(), GST_STATE_NULL);
183             m_src = nullptr;
184             m_encoder = nullptr;
185             m_capsFilter = nullptr;
186             m_sink = nullptr;
187             m_pipeline = nullptr;
188         }
189
190         return WEBRTC_VIDEO_CODEC_OK;
191     }
192
193     int32_t returnFromFlowReturn(GstFlowReturn flow)
194     {
195         switch (flow) {
196         case GST_FLOW_OK:
197             return WEBRTC_VIDEO_CODEC_OK;
198         case GST_FLOW_FLUSHING:
199             return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
200         default:
201             return WEBRTC_VIDEO_CODEC_ERROR;
202         }
203     }
204
205     VideoEncoder::EncoderInfo GetEncoderInfo() const override
206     {
207         EncoderInfo info;
208         info.supports_native_handle = false;
209         info.implementation_name = "GStreamer";
210         info.has_trusted_rate_controller = true;
211         info.is_hardware_accelerated = true;
212         info.has_internal_source = false;
213         return info;
214     }
215
216     int32_t Encode(const webrtc::VideoFrame& frame,
217         const std::vector<webrtc::VideoFrameType>* frameTypes) final
218     {
219         int32_t res;
220
221         if (!m_imageReadyCb) {
222             GST_INFO_OBJECT(m_pipeline.get(), "No encoded callback set yet!");
223
224             return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
225         }
226
227         if (!m_src) {
228             GST_INFO_OBJECT(m_pipeline.get(), "No source set yet!");
229
230             return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
231         }
232
233         auto sample = GStreamerSampleFromLibWebRTCVideoFrame(frame);
234         auto buffer = gst_sample_get_buffer(sample.get());
235
236         if (!GST_CLOCK_TIME_IS_VALID(m_firstFramePts)) {
237             m_firstFramePts = GST_BUFFER_PTS(buffer);
238             auto pad = adoptGRef(gst_element_get_static_pad(m_src, "src"));
239             gst_pad_set_offset(pad.get(), -m_firstFramePts);
240         }
241
242         for (auto frame_type : *frameTypes) {
243             if (frame_type == webrtc::VideoFrameType::kVideoFrameKey) {
244                 auto pad = adoptGRef(gst_element_get_static_pad(m_src, "src"));
245                 auto forceKeyUnit = gst_video_event_new_downstream_force_key_unit(GST_CLOCK_TIME_NONE,
246                     GST_CLOCK_TIME_NONE, GST_CLOCK_TIME_NONE, FALSE, 1);
247                 GST_INFO_OBJECT(m_pipeline.get(), "Requesting KEYFRAME!");
248
249                 if (!gst_pad_push_event(pad.get(), forceKeyUnit))
250                     GST_WARNING_OBJECT(pipeline(), "Could not send ForceKeyUnit event");
251
252                 break;
253             }
254         }
255
256         res = returnFromFlowReturn(gst_app_src_push_sample(GST_APP_SRC(m_src), sample.get()));
257         if (res != WEBRTC_VIDEO_CODEC_OK)
258             return res;
259
260         auto encodedSample = adoptGRef(gst_app_sink_try_pull_sample(GST_APP_SINK(m_sink), 5 * GST_SECOND));
261         if (!encodedSample) {
262             GST_ERROR("Didn't get any encodedSample");
263             return WEBRTC_VIDEO_CODEC_ERROR;
264         }
265
266         auto encodedData = GStreamerEncodedImageBuffer::create(WTFMove(encodedSample));
267         const auto* encodedBuffer = encodedData->getBuffer();
268         auto resolution = encodedData->getVideoResolution();
269         m_encodedFrame.SetEncodedData(encodedData);
270         if (!m_encodedFrame.size())
271             return WEBRTC_VIDEO_CODEC_OK;
272
273         ASSERT(resolution);
274         m_encodedFrame._encodedWidth = resolution->width();
275         m_encodedFrame._encodedHeight = resolution->height();
276         m_encodedFrame._frameType = GST_BUFFER_FLAG_IS_SET(encodedBuffer, GST_BUFFER_FLAG_DELTA_UNIT) ? webrtc::VideoFrameType::kVideoFrameDelta : webrtc::VideoFrameType::kVideoFrameKey;
277         m_encodedFrame._completeFrame = true;
278         m_encodedFrame.capture_time_ms_ = frame.render_time_ms();
279         m_encodedFrame.SetTimestamp(frame.timestamp());
280
281         GST_LOG_OBJECT(m_pipeline.get(), "Got buffer capture_time_ms: %" G_GINT64_FORMAT  " _timestamp: %u",
282             m_encodedFrame.capture_time_ms_, m_encodedFrame.Timestamp());
283
284         webrtc::CodecSpecificInfo codecInfo;
285         PopulateCodecSpecific(&codecInfo, encodedBuffer);
286         webrtc::EncodedImageCallback::Result result = m_imageReadyCb->OnEncodedImage(m_encodedFrame, &codecInfo);
287         if (result.error != webrtc::EncodedImageCallback::Result::OK)
288             GST_ERROR_OBJECT(m_pipeline.get(), "Encode callback failed: %d", result.error);
289
290         return WEBRTC_VIDEO_CODEC_OK;
291     }
292
293     GRefPtr<GstElement> createEncoder(void)
294     {
295         GRefPtr<GstElement> encoder = nullptr;
296         GstElement* webrtcencoder = GST_ELEMENT(g_object_ref_sink(gst_element_factory_make("webrtcvideoencoder", NULL)));
297
298         g_object_set(webrtcencoder, "format", adoptGRef(gst_caps_from_string(Caps())).get(), NULL);
299         g_object_get(webrtcencoder, "encoder", &encoder.outPtr(), NULL);
300
301         if (!encoder) {
302             GST_INFO("No encoder found for %s", Caps());
303
304             return nullptr;
305         }
306
307         return webrtcencoder;
308     }
309
310     void AddCodecIfSupported(std::vector<webrtc::SdpVideoFormat>& supportedFormats)
311     {
312         if (auto encoder = createEncoder()) {
313             auto formats = ConfigureSupportedCodec();
314             supportedFormats.insert(supportedFormats.end(), formats.begin(), formats.end());
315         }
316     }
317
318     virtual const gchar* Caps()
319     {
320         return nullptr;
321     }
322
323     virtual std::vector<webrtc::SdpVideoFormat> ConfigureSupportedCodec()
324     {
325         return { webrtc::SdpVideoFormat(Name()) };
326     }
327
328     virtual webrtc::VideoCodecType CodecType() = 0;
329     virtual void PopulateCodecSpecific(webrtc::CodecSpecificInfo*, const GstBuffer*) = 0;
330     virtual const gchar* Name() = 0;
331     virtual int KeyframeInterval(const webrtc::VideoCodec* codecSettings) = 0;
332
333     void SetRestrictionCaps(GRefPtr<GstCaps> caps)
334     {
335         if (m_restrictionCaps)
336             g_object_set(m_capsFilter, "caps", m_restrictionCaps.get(), nullptr);
337
338         m_restrictionCaps = caps;
339     }
340
341 private:
342     GRefPtr<GstElement> m_pipeline;
343     GstElement* m_src;
344     GstElement* m_encoder;
345     GstElement* m_capsFilter;
346
347     webrtc::EncodedImageCallback* m_imageReadyCb;
348     GstClockTime m_firstFramePts;
349     GRefPtr<GstCaps> m_restrictionCaps;
350     webrtc::EncodedImage m_encodedFrame;
351
352     Lock m_bufferMapLock;
353     GstElement* m_sink;
354 };
355
356 class GStreamerH264Encoder : public GStreamerVideoEncoder {
357 public:
358     GStreamerH264Encoder() { }
359
360     GStreamerH264Encoder(const webrtc::SdpVideoFormat& format)
361         : m_parser(gst_h264_nal_parser_new())
362         , packetizationMode(webrtc::H264PacketizationMode::NonInterleaved)
363     {
364         auto it = format.parameters.find(cricket::kH264FmtpPacketizationMode);
365
366         if (it != format.parameters.end() && it->second == "1")
367             packetizationMode = webrtc::H264PacketizationMode::NonInterleaved;
368     }
369
370     int KeyframeInterval(const webrtc::VideoCodec* codecSettings) final
371     {
372         return codecSettings->H264().keyFrameInterval;
373     }
374
375     std::vector<webrtc::SdpVideoFormat> ConfigureSupportedCodec() final
376     {
377         return gstreamerSupportedH264Codecs();
378     }
379
380     const gchar* Caps() final { return "video/x-h264"; }
381     const gchar* Name() final { return cricket::kH264CodecName; }
382     GstH264NalParser* m_parser;
383     webrtc::VideoCodecType CodecType() final { return webrtc::kVideoCodecH264; }
384
385     void PopulateCodecSpecific(webrtc::CodecSpecificInfo* codecSpecificInfos, const GstBuffer*) final
386     {
387         codecSpecificInfos->codecType = CodecType();
388         webrtc::CodecSpecificInfoH264* h264Info = &(codecSpecificInfos->codecSpecific.H264);
389         h264Info->packetization_mode = packetizationMode;
390     }
391
392     webrtc::H264PacketizationMode packetizationMode;
393 };
394
395 class GStreamerVP8Encoder : public GStreamerVideoEncoder {
396 public:
397     GStreamerVP8Encoder() { }
398     GStreamerVP8Encoder(const webrtc::SdpVideoFormat&) { }
399     const gchar* Caps() final { return "video/x-vp8"; }
400     const gchar* Name() final { return cricket::kVp8CodecName; }
401     webrtc::VideoCodecType CodecType() final { return webrtc::kVideoCodecVP8; }
402
403     int KeyframeInterval(const webrtc::VideoCodec* codecSettings) final
404     {
405         return codecSettings->VP8().keyFrameInterval;
406     }
407
408     void PopulateCodecSpecific(webrtc::CodecSpecificInfo* codecSpecificInfos, const GstBuffer* buffer) final
409     {
410         codecSpecificInfos->codecType = webrtc::kVideoCodecVP8;
411         webrtc::CodecSpecificInfoVP8* vp8Info = &(codecSpecificInfos->codecSpecific.VP8);
412         vp8Info->temporalIdx = 0;
413
414         vp8Info->keyIdx = webrtc::kNoKeyIdx;
415         vp8Info->nonReference = GST_BUFFER_FLAG_IS_SET(buffer, GST_BUFFER_FLAG_DELTA_UNIT);
416     }
417 };
418
419 std::unique_ptr<webrtc::VideoEncoder> GStreamerVideoEncoderFactory::CreateVideoEncoder(const webrtc::SdpVideoFormat& format)
420 {
421     if (format.name == cricket::kVp8CodecName) {
422         GRefPtr<GstElement> webrtcencoder = adoptGRef(GST_ELEMENT(g_object_ref_sink(gst_element_factory_make("webrtcvideoencoder", NULL))));
423         GRefPtr<GstElement> encoder = nullptr;
424
425         g_object_set(webrtcencoder.get(), "format", adoptGRef(gst_caps_from_string("video/x-vp8")).get(), NULL);
426         g_object_get(webrtcencoder.get(), "encoder", &encoder.outPtr(), NULL);
427
428         if (encoder)
429             return makeUnique<GStreamerVP8Encoder>(format);
430
431         GST_INFO("Using VP8 Encoder from LibWebRTC.");
432         return makeUniqueWithoutFastMallocCheck<webrtc::LibvpxVp8Encoder>(webrtc::LibvpxInterface::CreateEncoder(), webrtc::VP8Encoder::Settings());
433     }
434
435     if (format.name == cricket::kH264CodecName)
436         return makeUnique<GStreamerH264Encoder>(format);
437
438     return nullptr;
439 }
440
441 GStreamerVideoEncoderFactory::GStreamerVideoEncoderFactory()
442 {
443     ensureGStreamerInitialized();
444
445     static std::once_flag debugRegisteredFlag;
446     std::call_once(debugRegisteredFlag, [] {
447         GST_DEBUG_CATEGORY_INIT(webkit_webrtcenc_debug, "webkitlibwebrtcvideoencoder", 0, "WebKit WebRTC video encoder");
448         auto factory = adoptGRef(gst_element_factory_find("webrtcvideoencoder"));
449         if (!factory)
450             gst_element_register(nullptr, "webrtcvideoencoder", GST_RANK_NONE, WEBKIT_TYPE_WEBRTC_VIDEO_ENCODER);
451     });
452 }
453
454 std::vector<webrtc::SdpVideoFormat> GStreamerVideoEncoderFactory::GetSupportedFormats() const
455 {
456     std::vector<webrtc::SdpVideoFormat> supportedCodecs;
457
458     supportedCodecs.push_back(webrtc::SdpVideoFormat(cricket::kVp8CodecName));
459     GStreamerH264Encoder().AddCodecIfSupported(supportedCodecs);
460
461     return supportedCodecs;
462 }
463
464 } // namespace WebCore
465 #endif