d6d23264a3919bbc129f671e80b6ce2c4d6aea8a
[WebKit-https.git] / Source / WebCore / platform / graphics / gstreamer / VideoSinkGStreamer.cpp
1 /*
2  *  Copyright (C) 2007 OpenedHand
3  *  Copyright (C) 2007 Alp Toker <alp@atoker.com>
4  *  Copyright (C) 2009, 2010, 2011, 2012 Igalia S.L
5  *
6  *  This library is free software; you can redistribute it and/or
7  *  modify it under the terms of the GNU Lesser General Public
8  *  License as published by the Free Software Foundation; either
9  *  version 2 of the License, or (at your option) any later version.
10  *
11  *  This library is distributed in the hope that it will be useful,
12  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
13  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14  *  Lesser General Public License for more details.
15  *
16  *  You should have received a copy of the GNU Lesser General Public
17  *  License along with this library; if not, write to the Free Software
18  *  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
19  */
20
21 /*
22  *
23  * WebKitVideoSink is a GStreamer sink element that triggers
24  * repaints in the WebKit GStreamer media player for the
25  * current video buffer.
26  */
27
28 #include "config.h"
29 #include "VideoSinkGStreamer.h"
30
31 #if ENABLE(VIDEO) && USE(GSTREAMER)
32 #include "GRefPtrGStreamer.h"
33 #include "GStreamerUtilities.h"
34 #include "IntSize.h"
35 #include <glib.h>
36 #include <gst/gst.h>
37 #include <gst/video/gstvideometa.h>
38 #include <wtf/Condition.h>
39 #include <wtf/RunLoop.h>
40
41 using namespace WebCore;
42
43 // CAIRO_FORMAT_RGB24 used to render the video buffers is little/big endian dependant.
44 #if G_BYTE_ORDER == G_LITTLE_ENDIAN
45 #define GST_CAPS_FORMAT "{ BGRx, BGRA }"
46 #else
47 #define GST_CAPS_FORMAT "{ xRGB, ARGB }"
48 #endif
49 #if GST_CHECK_VERSION(1, 1, 0)
50 #define GST_FEATURED_CAPS GST_VIDEO_CAPS_MAKE_WITH_FEATURES(GST_CAPS_FEATURE_META_GST_VIDEO_GL_TEXTURE_UPLOAD_META, GST_CAPS_FORMAT) ";"
51 #else
52 #define GST_FEATURED_CAPS
53 #endif
54
55 #define WEBKIT_VIDEO_SINK_PAD_CAPS GST_FEATURED_CAPS GST_VIDEO_CAPS_MAKE(GST_CAPS_FORMAT)
56
57 static GstStaticPadTemplate s_sinkTemplate = GST_STATIC_PAD_TEMPLATE("sink", GST_PAD_SINK, GST_PAD_ALWAYS, GST_STATIC_CAPS(WEBKIT_VIDEO_SINK_PAD_CAPS));
58
59
60 GST_DEBUG_CATEGORY_STATIC(webkitVideoSinkDebug);
61 #define GST_CAT_DEFAULT webkitVideoSinkDebug
62
63 enum {
64     REPAINT_REQUESTED,
65     LAST_SIGNAL
66 };
67
68 static guint webkitVideoSinkSignals[LAST_SIGNAL] = { 0, };
69
70 static void webkitVideoSinkRepaintRequested(WebKitVideoSink*, GstSample*);
71 static GRefPtr<GstSample> webkitVideoSinkRequestRender(WebKitVideoSink*, GstBuffer*);
72
73 class VideoRenderRequestScheduler {
74 public:
75     VideoRenderRequestScheduler()
76         : m_timer(RunLoop::main(), this, &VideoRenderRequestScheduler::render)
77     {
78 #if PLATFORM(GTK)
79         // Use a higher priority than WebCore timers (G_PRIORITY_HIGH_IDLE + 20).
80         m_timer.setPriority(G_PRIORITY_HIGH_IDLE + 19);
81 #endif
82     }
83
84     void start()
85     {
86         LockHolder locker(m_sampleMutex);
87         m_unlocked = false;
88     }
89
90     void stop()
91     {
92         LockHolder locker(m_sampleMutex);
93         m_timer.stop();
94         m_sample = nullptr;
95         m_unlocked = true;
96         m_dataCondition.notifyOne();
97     }
98
99     bool requestRender(WebKitVideoSink* sink, GstBuffer* buffer)
100     {
101         LockHolder locker(m_sampleMutex);
102         if (m_unlocked)
103             return true;
104
105         m_sample = webkitVideoSinkRequestRender(sink, buffer);
106         if (!m_sample)
107             return false;
108
109         m_sink = sink;
110         m_timer.startOneShot(0);
111         m_dataCondition.wait(m_sampleMutex);
112         return true;
113     }
114
115 private:
116
117     void render()
118     {
119         LockHolder locker(m_sampleMutex);
120         GRefPtr<GstSample> sample = WTF::move(m_sample);
121         GRefPtr<WebKitVideoSink> sink = WTF::move(m_sink);
122         if (sample && !m_unlocked && LIKELY(GST_IS_SAMPLE(sample.get())))
123             webkitVideoSinkRepaintRequested(sink.get(), sample.get());
124         m_dataCondition.notifyOne();
125     }
126
127     RunLoop::Timer<VideoRenderRequestScheduler> m_timer;
128     Lock m_sampleMutex;
129     Condition m_dataCondition;
130     GRefPtr<GstSample> m_sample;
131     GRefPtr<WebKitVideoSink> m_sink;
132
133     // If this is true all processing should finish ASAP
134     // This is necessary because there could be a race between
135     // unlock() and render(), where unlock() wins, signals the
136     // Condition, then render() tries to render a frame although
137     // everything else isn't running anymore. This will lead
138     // to deadlocks because render() holds the stream lock.
139     //
140     // Protected by the sample mutex
141     bool m_unlocked { false };
142 };
143
144 struct _WebKitVideoSinkPrivate {
145     _WebKitVideoSinkPrivate()
146     {
147         gst_video_info_init(&info);
148     }
149
150     ~_WebKitVideoSinkPrivate()
151     {
152         if (currentCaps)
153             gst_caps_unref(currentCaps);
154     }
155
156     VideoRenderRequestScheduler scheduler;
157     GstVideoInfo info;
158     GstCaps* currentCaps;
159 };
160
161 #define webkit_video_sink_parent_class parent_class
162 G_DEFINE_TYPE_WITH_CODE(WebKitVideoSink, webkit_video_sink, GST_TYPE_VIDEO_SINK, GST_DEBUG_CATEGORY_INIT(webkitVideoSinkDebug, "webkitsink", 0, "webkit video sink"));
163
164
165 static void webkit_video_sink_init(WebKitVideoSink* sink)
166 {
167     sink->priv = G_TYPE_INSTANCE_GET_PRIVATE(sink, WEBKIT_TYPE_VIDEO_SINK, WebKitVideoSinkPrivate);
168     g_object_set(GST_BASE_SINK(sink), "enable-last-sample", FALSE, NULL);
169     new (sink->priv) WebKitVideoSinkPrivate();
170 }
171
172 static void webkitVideoSinkRepaintRequested(WebKitVideoSink* sink, GstSample* sample)
173 {
174     g_signal_emit(sink, webkitVideoSinkSignals[REPAINT_REQUESTED], 0, sample);
175 }
176
177 static GRefPtr<GstSample> webkitVideoSinkRequestRender(WebKitVideoSink* sink, GstBuffer* buffer)
178 {
179     WebKitVideoSinkPrivate* priv = sink->priv;
180     GRefPtr<GstSample> sample = adoptGRef(gst_sample_new(buffer, priv->currentCaps, nullptr, nullptr));
181
182     // The video info structure is valid only if the sink handled an allocation query.
183     GstVideoFormat format = GST_VIDEO_INFO_FORMAT(&priv->info);
184     if (format == GST_VIDEO_FORMAT_UNKNOWN)
185         return nullptr;
186
187 #if !(USE(TEXTURE_MAPPER_GL) && !USE(COORDINATED_GRAPHICS))
188     // Cairo's ARGB has pre-multiplied alpha while GStreamer's doesn't.
189     // Here we convert to Cairo's ARGB.
190     if (format == GST_VIDEO_FORMAT_ARGB || format == GST_VIDEO_FORMAT_BGRA) {
191         // Because GstBaseSink::render() only owns the buffer reference in the
192         // method scope we can't use gst_buffer_make_writable() here. Also
193         // The buffer content should not be changed here because the same buffer
194         // could be passed multiple times to this method (in theory).
195
196         GstBuffer* newBuffer = WebCore::createGstBuffer(buffer);
197
198         // Check if allocation failed.
199         if (UNLIKELY(!newBuffer)) {
200             gst_buffer_unref(buffer);
201             return nullptr;
202         }
203
204         // We don't use Color::premultipliedARGBFromColor() here because
205         // one function call per video pixel is just too expensive:
206         // For 720p/PAL for example this means 1280*720*25=23040000
207         // function calls per second!
208         GstVideoFrame sourceFrame;
209         GstVideoFrame destinationFrame;
210
211         if (!gst_video_frame_map(&sourceFrame, &priv->info, buffer, GST_MAP_READ)) {
212             gst_buffer_unref(newBuffer);
213             return nullptr;
214         }
215         if (!gst_video_frame_map(&destinationFrame, &priv->info, newBuffer, GST_MAP_WRITE)) {
216             gst_video_frame_unmap(&sourceFrame);
217             gst_buffer_unref(newBuffer);
218             return nullptr;
219         }
220
221         const guint8* source = static_cast<guint8*>(GST_VIDEO_FRAME_PLANE_DATA(&sourceFrame, 0));
222         guint8* destination = static_cast<guint8*>(GST_VIDEO_FRAME_PLANE_DATA(&destinationFrame, 0));
223
224         for (int x = 0; x < GST_VIDEO_FRAME_HEIGHT(&sourceFrame); x++) {
225             for (int y = 0; y < GST_VIDEO_FRAME_WIDTH(&sourceFrame); y++) {
226 #if G_BYTE_ORDER == G_LITTLE_ENDIAN
227                 unsigned short alpha = source[3];
228                 destination[0] = (source[0] * alpha + 128) / 255;
229                 destination[1] = (source[1] * alpha + 128) / 255;
230                 destination[2] = (source[2] * alpha + 128) / 255;
231                 destination[3] = alpha;
232 #else
233                 unsigned short alpha = source[0];
234                 destination[0] = alpha;
235                 destination[1] = (source[1] * alpha + 128) / 255;
236                 destination[2] = (source[2] * alpha + 128) / 255;
237                 destination[3] = (source[3] * alpha + 128) / 255;
238 #endif
239                 source += 4;
240                 destination += 4;
241             }
242         }
243
244         gst_video_frame_unmap(&sourceFrame);
245         gst_video_frame_unmap(&destinationFrame);
246         sample = adoptGRef(gst_sample_new(newBuffer, priv->currentCaps, nullptr, nullptr));
247     }
248 #endif
249
250     return sample;
251 }
252
253 static GstFlowReturn webkitVideoSinkRender(GstBaseSink* baseSink, GstBuffer* buffer)
254 {
255     WebKitVideoSink* sink = WEBKIT_VIDEO_SINK(baseSink);
256     return sink->priv->scheduler.requestRender(sink, buffer) ? GST_FLOW_OK : GST_FLOW_ERROR;
257 }
258
259 static void webkitVideoSinkFinalize(GObject* object)
260 {
261     WEBKIT_VIDEO_SINK(object)->priv->~WebKitVideoSinkPrivate();
262     G_OBJECT_CLASS(parent_class)->finalize(object);
263 }
264
265 static gboolean webkitVideoSinkUnlock(GstBaseSink* baseSink)
266 {
267     WebKitVideoSinkPrivate* priv = WEBKIT_VIDEO_SINK(baseSink)->priv;
268
269     priv->scheduler.stop();
270
271     return GST_CALL_PARENT_WITH_DEFAULT(GST_BASE_SINK_CLASS, unlock, (baseSink), TRUE);
272 }
273
274 static gboolean webkitVideoSinkUnlockStop(GstBaseSink* baseSink)
275 {
276     WebKitVideoSinkPrivate* priv = WEBKIT_VIDEO_SINK(baseSink)->priv;
277
278     priv->scheduler.start();
279
280     return GST_CALL_PARENT_WITH_DEFAULT(GST_BASE_SINK_CLASS, unlock_stop, (baseSink), TRUE);
281 }
282
283 static gboolean webkitVideoSinkStop(GstBaseSink* baseSink)
284 {
285     WebKitVideoSinkPrivate* priv = WEBKIT_VIDEO_SINK(baseSink)->priv;
286
287     priv->scheduler.stop();
288     if (priv->currentCaps) {
289         gst_caps_unref(priv->currentCaps);
290         priv->currentCaps = nullptr;
291     }
292
293     return TRUE;
294 }
295
296 static gboolean webkitVideoSinkStart(GstBaseSink* baseSink)
297 {
298     WebKitVideoSinkPrivate* priv = WEBKIT_VIDEO_SINK(baseSink)->priv;
299
300     priv->scheduler.start();
301
302     return TRUE;
303 }
304
305 static gboolean webkitVideoSinkSetCaps(GstBaseSink* baseSink, GstCaps* caps)
306 {
307     WebKitVideoSink* sink = WEBKIT_VIDEO_SINK(baseSink);
308     WebKitVideoSinkPrivate* priv = sink->priv;
309
310     GST_DEBUG_OBJECT(sink, "Current caps %" GST_PTR_FORMAT ", setting caps %" GST_PTR_FORMAT, priv->currentCaps, caps);
311
312     GstVideoInfo videoInfo;
313     gst_video_info_init(&videoInfo);
314     if (!gst_video_info_from_caps(&videoInfo, caps)) {
315         GST_ERROR_OBJECT(sink, "Invalid caps %" GST_PTR_FORMAT, caps);
316         return FALSE;
317     }
318
319     priv->info = videoInfo;
320     gst_caps_replace(&priv->currentCaps, caps);
321     return TRUE;
322 }
323
324 static gboolean webkitVideoSinkProposeAllocation(GstBaseSink* baseSink, GstQuery* query)
325 {
326     GstCaps* caps;
327     gst_query_parse_allocation(query, &caps, 0);
328     if (!caps)
329         return FALSE;
330
331     WebKitVideoSink* sink = WEBKIT_VIDEO_SINK(baseSink);
332     if (!gst_video_info_from_caps(&sink->priv->info, caps))
333         return FALSE;
334
335     gst_query_add_allocation_meta(query, GST_VIDEO_META_API_TYPE, 0);
336     gst_query_add_allocation_meta(query, GST_VIDEO_CROP_META_API_TYPE, 0);
337 #if GST_CHECK_VERSION(1, 1, 0)
338     gst_query_add_allocation_meta(query, GST_VIDEO_GL_TEXTURE_UPLOAD_META_API_TYPE, 0);
339 #endif
340     return TRUE;
341 }
342
343 static void webkit_video_sink_class_init(WebKitVideoSinkClass* klass)
344 {
345     GObjectClass* gobjectClass = G_OBJECT_CLASS(klass);
346     GstBaseSinkClass* baseSinkClass = GST_BASE_SINK_CLASS(klass);
347     GstElementClass* elementClass = GST_ELEMENT_CLASS(klass);
348
349     gst_element_class_add_pad_template(elementClass, gst_static_pad_template_get(&s_sinkTemplate));
350     gst_element_class_set_metadata(elementClass, "WebKit video sink", "Sink/Video", "Sends video data from a GStreamer pipeline to WebKit", "Igalia, Alp Toker <alp@atoker.com>");
351
352     g_type_class_add_private(klass, sizeof(WebKitVideoSinkPrivate));
353
354     gobjectClass->finalize = webkitVideoSinkFinalize;
355
356     baseSinkClass->unlock = webkitVideoSinkUnlock;
357     baseSinkClass->unlock_stop = webkitVideoSinkUnlockStop;
358     baseSinkClass->render = webkitVideoSinkRender;
359     baseSinkClass->preroll = webkitVideoSinkRender;
360     baseSinkClass->stop = webkitVideoSinkStop;
361     baseSinkClass->start = webkitVideoSinkStart;
362     baseSinkClass->set_caps = webkitVideoSinkSetCaps;
363     baseSinkClass->propose_allocation = webkitVideoSinkProposeAllocation;
364
365     webkitVideoSinkSignals[REPAINT_REQUESTED] = g_signal_new("repaint-requested",
366             G_TYPE_FROM_CLASS(klass),
367             static_cast<GSignalFlags>(G_SIGNAL_RUN_LAST | G_SIGNAL_ACTION),
368             0, // Class offset
369             0, // Accumulator
370             0, // Accumulator data
371             g_cclosure_marshal_generic,
372             G_TYPE_NONE, // Return type
373             1, // Only one parameter
374             GST_TYPE_SAMPLE);
375 }
376
377
378 GstElement* webkitVideoSinkNew()
379 {
380     return GST_ELEMENT(g_object_new(WEBKIT_TYPE_VIDEO_SINK, 0));
381 }
382
383 #endif // ENABLE(VIDEO) && USE(GSTREAMER)