81604692138d7821c15c4a383218045c34eda6b5
[WebKit-https.git] / Source / WebCore / platform / graphics / gstreamer / VideoSinkGStreamer.cpp
1 /*
2  *  Copyright (C) 2007 OpenedHand
3  *  Copyright (C) 2007 Alp Toker <alp@atoker.com>
4  *  Copyright (C) 2009, 2010, 2011, 2012 Igalia S.L
5  *
6  *  This library is free software; you can redistribute it and/or
7  *  modify it under the terms of the GNU Lesser General Public
8  *  License as published by the Free Software Foundation; either
9  *  version 2 of the License, or (at your option) any later version.
10  *
11  *  This library is distributed in the hope that it will be useful,
12  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
13  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14  *  Lesser General Public License for more details.
15  *
16  *  You should have received a copy of the GNU Lesser General Public
17  *  License along with this library; if not, write to the Free Software
18  *  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
19  */
20
21 /*
22  *
23  * WebKitVideoSink is a GStreamer sink element that triggers
24  * repaints in the WebKit GStreamer media player for the
25  * current video buffer.
26  */
27
28 #include "config.h"
29 #include "VideoSinkGStreamer.h"
30
31 #if ENABLE(VIDEO) && USE(GSTREAMER)
32 #include "GRefPtrGStreamer.h"
33 #include "GStreamerVersioning.h"
34 #include "IntSize.h"
35 #include <glib.h>
36 #include <gst/gst.h>
37 #ifdef GST_API_VERSION_1
38 #include <gst/video/gstvideometa.h>
39 #include <gst/video/gstvideopool.h>
40 #endif
41 #include <wtf/OwnPtr.h>
42
43 // CAIRO_FORMAT_RGB24 used to render the video buffers is little/big endian dependant.
44 #ifdef GST_API_VERSION_1
45 #if G_BYTE_ORDER == G_LITTLE_ENDIAN
46 #define GST_CAPS_FORMAT "{ BGRx, BGRA }"
47 #else
48 #define GST_CAPS_FORMAT "{ xRGB, ARGB }"
49 #endif
50 #if GST_CHECK_VERSION(1, 1, 0)
51 #define GST_FEATURED_CAPS GST_VIDEO_CAPS_MAKE_WITH_FEATURES(GST_CAPS_FEATURE_META_GST_VIDEO_GL_TEXTURE_UPLOAD_META, GST_CAPS_FORMAT) ";"
52 #else
53 #define GST_FEATURED_CAPS
54 #endif
55 #endif // GST_API_VERSION_1
56
57 #ifdef GST_API_VERSION_1
58 #define WEBKIT_VIDEO_SINK_PAD_CAPS GST_FEATURED_CAPS GST_VIDEO_CAPS_MAKE(GST_CAPS_FORMAT)
59 #else
60 #if G_BYTE_ORDER == G_LITTLE_ENDIAN
61 #define WEBKIT_VIDEO_SINK_PAD_CAPS GST_VIDEO_CAPS_BGRx ";" GST_VIDEO_CAPS_BGRA
62 #else
63 #define WEBKIT_VIDEO_SINK_PAD_CAPS GST_VIDEO_CAPS_xRGB ";" GST_VIDEO_CAPS_ARGB
64 #endif
65 #endif // GST_API_VERSION_1
66
67 static GstStaticPadTemplate s_sinkTemplate = GST_STATIC_PAD_TEMPLATE("sink", GST_PAD_SINK, GST_PAD_ALWAYS, GST_STATIC_CAPS(WEBKIT_VIDEO_SINK_PAD_CAPS));
68
69
70 GST_DEBUG_CATEGORY_STATIC(webkitVideoSinkDebug);
71 #define GST_CAT_DEFAULT webkitVideoSinkDebug
72
73 enum {
74     REPAINT_REQUESTED,
75     LAST_SIGNAL
76 };
77
78 enum {
79     PROP_0,
80     PROP_CAPS
81 };
82
83 static guint webkitVideoSinkSignals[LAST_SIGNAL] = { 0, };
84
85 struct _WebKitVideoSinkPrivate {
86     GstBuffer* buffer;
87     guint timeoutId;
88     GMutex* bufferMutex;
89     GCond* dataCondition;
90
91 #ifdef GST_API_VERSION_1
92     GstVideoInfo info;
93 #endif
94
95 #if USE(NATIVE_FULLSCREEN_VIDEO)
96     WebCore::GStreamerGWorld* gstGWorld;
97 #endif
98
99     GstCaps* currentCaps;
100
101     // If this is TRUE all processing should finish ASAP
102     // This is necessary because there could be a race between
103     // unlock() and render(), where unlock() wins, signals the
104     // GCond, then render() tries to render a frame although
105     // everything else isn't running anymore. This will lead
106     // to deadlocks because render() holds the stream lock.
107     //
108     // Protected by the buffer mutex
109     bool unlocked;
110 };
111
112 #define webkit_video_sink_parent_class parent_class
113 G_DEFINE_TYPE_WITH_CODE(WebKitVideoSink, webkit_video_sink, GST_TYPE_VIDEO_SINK, GST_DEBUG_CATEGORY_INIT(webkitVideoSinkDebug, "webkitsink", 0, "webkit video sink"));
114
115
116 static void webkit_video_sink_init(WebKitVideoSink* sink)
117 {
118     sink->priv = G_TYPE_INSTANCE_GET_PRIVATE(sink, WEBKIT_TYPE_VIDEO_SINK, WebKitVideoSinkPrivate);
119 #if GLIB_CHECK_VERSION(2, 31, 0)
120     sink->priv->dataCondition = new GCond;
121     g_cond_init(sink->priv->dataCondition);
122     sink->priv->bufferMutex = new GMutex;
123     g_mutex_init(sink->priv->bufferMutex);
124 #else
125     sink->priv->dataCondition = g_cond_new();
126     sink->priv->bufferMutex = g_mutex_new();
127 #endif
128
129 #ifdef GST_API_VERSION_1
130     gst_video_info_init(&sink->priv->info);
131 #endif
132 }
133
134 static gboolean webkitVideoSinkTimeoutCallback(gpointer data)
135 {
136     WebKitVideoSink* sink = reinterpret_cast<WebKitVideoSink*>(data);
137     WebKitVideoSinkPrivate* priv = sink->priv;
138
139     g_mutex_lock(priv->bufferMutex);
140     GstBuffer* buffer = priv->buffer;
141     priv->buffer = 0;
142     priv->timeoutId = 0;
143
144     if (!buffer || priv->unlocked || UNLIKELY(!GST_IS_BUFFER(buffer))) {
145         g_cond_signal(priv->dataCondition);
146         g_mutex_unlock(priv->bufferMutex);
147         return FALSE;
148     }
149
150     g_signal_emit(sink, webkitVideoSinkSignals[REPAINT_REQUESTED], 0, buffer);
151     gst_buffer_unref(buffer);
152     g_cond_signal(priv->dataCondition);
153     g_mutex_unlock(priv->bufferMutex);
154
155     return FALSE;
156 }
157
158 static GstFlowReturn webkitVideoSinkRender(GstBaseSink* baseSink, GstBuffer* buffer)
159 {
160     WebKitVideoSink* sink = WEBKIT_VIDEO_SINK(baseSink);
161     WebKitVideoSinkPrivate* priv = sink->priv;
162
163     g_mutex_lock(priv->bufferMutex);
164
165     if (priv->unlocked) {
166         g_mutex_unlock(priv->bufferMutex);
167         return GST_FLOW_OK;
168     }
169
170 #if USE(NATIVE_FULLSCREEN_VIDEO)
171     // Ignore buffers if the video is already in fullscreen using
172     // another sink.
173     if (priv->gstGWorld->isFullscreen()) {
174         g_mutex_unlock(priv->bufferMutex);
175         return GST_FLOW_OK;
176     }
177 #endif
178
179     priv->buffer = gst_buffer_ref(buffer);
180
181 #ifndef GST_API_VERSION_1
182     // For the unlikely case where the buffer has no caps, the caps
183     // are implicitely the caps of the pad. This shouldn't happen.
184     if (UNLIKELY(!GST_BUFFER_CAPS(buffer))) {
185         buffer = priv->buffer = gst_buffer_make_metadata_writable(priv->buffer);
186         gst_buffer_set_caps(priv->buffer, GST_PAD_CAPS(GST_BASE_SINK_PAD(baseSink)));
187     }
188
189     GRefPtr<GstCaps> caps = GST_BUFFER_CAPS(buffer);
190 #else
191     GRefPtr<GstCaps> caps;
192     // The video info structure is valid only if the sink handled an allocation query.
193     if (GST_VIDEO_INFO_FORMAT(&priv->info) != GST_VIDEO_FORMAT_UNKNOWN)
194         caps = adoptGRef(gst_video_info_to_caps(&priv->info));
195     else
196         caps = priv->currentCaps;
197 #endif
198
199     GstVideoFormat format;
200     WebCore::IntSize size;
201     int pixelAspectRatioNumerator, pixelAspectRatioDenominator, stride;
202     if (!getVideoSizeAndFormatFromCaps(caps.get(), size, format, pixelAspectRatioNumerator, pixelAspectRatioDenominator, stride)) {
203         gst_buffer_unref(buffer);
204         g_mutex_unlock(priv->bufferMutex);
205         return GST_FLOW_ERROR;
206     }
207
208     // Cairo's ARGB has pre-multiplied alpha while GStreamer's doesn't.
209     // Here we convert to Cairo's ARGB.
210     if (format == GST_VIDEO_FORMAT_ARGB || format == GST_VIDEO_FORMAT_BGRA) {
211         // Because GstBaseSink::render() only owns the buffer reference in the
212         // method scope we can't use gst_buffer_make_writable() here. Also
213         // The buffer content should not be changed here because the same buffer
214         // could be passed multiple times to this method (in theory).
215
216         GstBuffer* newBuffer = createGstBuffer(buffer);
217
218         // Check if allocation failed.
219         if (UNLIKELY(!newBuffer)) {
220             g_mutex_unlock(priv->bufferMutex);
221             return GST_FLOW_ERROR;
222         }
223
224         // We don't use Color::premultipliedARGBFromColor() here because
225         // one function call per video pixel is just too expensive:
226         // For 720p/PAL for example this means 1280*720*25=23040000
227         // function calls per second!
228 #ifndef GST_API_VERSION_1
229         const guint8* source = GST_BUFFER_DATA(buffer);
230         guint8* destination = GST_BUFFER_DATA(newBuffer);
231 #else
232         GstMapInfo sourceInfo;
233         GstMapInfo destinationInfo;
234         gst_buffer_map(buffer, &sourceInfo, GST_MAP_READ);
235         const guint8* source = const_cast<guint8*>(sourceInfo.data);
236         gst_buffer_map(newBuffer, &destinationInfo, GST_MAP_WRITE);
237         guint8* destination = static_cast<guint8*>(destinationInfo.data);
238 #endif
239
240         for (int x = 0; x < size.height(); x++) {
241             for (int y = 0; y < size.width(); y++) {
242 #if G_BYTE_ORDER == G_LITTLE_ENDIAN
243                 unsigned short alpha = source[3];
244                 destination[0] = (source[0] * alpha + 128) / 255;
245                 destination[1] = (source[1] * alpha + 128) / 255;
246                 destination[2] = (source[2] * alpha + 128) / 255;
247                 destination[3] = alpha;
248 #else
249                 unsigned short alpha = source[0];
250                 destination[0] = alpha;
251                 destination[1] = (source[1] * alpha + 128) / 255;
252                 destination[2] = (source[2] * alpha + 128) / 255;
253                 destination[3] = (source[3] * alpha + 128) / 255;
254 #endif
255                 source += 4;
256                 destination += 4;
257             }
258         }
259
260 #ifdef GST_API_VERSION_1
261         gst_buffer_unmap(buffer, &sourceInfo);
262         gst_buffer_unmap(newBuffer, &destinationInfo);
263 #endif
264         gst_buffer_unref(buffer);
265         buffer = priv->buffer = newBuffer;
266     }
267
268     // This should likely use a lower priority, but glib currently starves
269     // lower priority sources.
270     // See: https://bugzilla.gnome.org/show_bug.cgi?id=610830.
271     priv->timeoutId = g_timeout_add_full(G_PRIORITY_DEFAULT, 0, webkitVideoSinkTimeoutCallback,
272                                           gst_object_ref(sink), reinterpret_cast<GDestroyNotify>(gst_object_unref));
273
274     g_cond_wait(priv->dataCondition, priv->bufferMutex);
275     g_mutex_unlock(priv->bufferMutex);
276     return GST_FLOW_OK;
277 }
278
279 static void webkitVideoSinkDispose(GObject* object)
280 {
281     WebKitVideoSink* sink = WEBKIT_VIDEO_SINK(object);
282     WebKitVideoSinkPrivate* priv = sink->priv;
283
284     if (priv->dataCondition) {
285 #if GLIB_CHECK_VERSION(2, 31, 0)
286         g_cond_clear(priv->dataCondition);
287         delete priv->dataCondition;
288 #else
289         g_cond_free(priv->dataCondition);
290 #endif
291         priv->dataCondition = 0;
292     }
293
294     if (priv->bufferMutex) {
295 #if GLIB_CHECK_VERSION(2, 31, 0)
296         g_mutex_clear(priv->bufferMutex);
297         delete priv->bufferMutex;
298 #else
299         g_mutex_free(priv->bufferMutex);
300 #endif
301         priv->bufferMutex = 0;
302     }
303
304     G_OBJECT_CLASS(parent_class)->dispose(object);
305 }
306
307 static void webkitVideoSinkGetProperty(GObject* object, guint propertyId, GValue* value, GParamSpec* parameterSpec)
308 {
309     WebKitVideoSink* sink = WEBKIT_VIDEO_SINK(object);
310     WebKitVideoSinkPrivate* priv = sink->priv;
311
312     switch (propertyId) {
313     case PROP_CAPS: {
314         GstCaps* caps = priv->currentCaps;
315         if (caps)
316             gst_caps_ref(caps);
317         g_value_take_boxed(value, caps);
318         break;
319     }
320     default:
321         G_OBJECT_WARN_INVALID_PROPERTY_ID(object, propertyId, parameterSpec);
322     }
323 }
324
325 static void unlockBufferMutex(WebKitVideoSinkPrivate* priv)
326 {
327     g_mutex_lock(priv->bufferMutex);
328
329     if (priv->buffer) {
330         gst_buffer_unref(priv->buffer);
331         priv->buffer = 0;
332     }
333
334     priv->unlocked = true;
335
336     g_cond_signal(priv->dataCondition);
337     g_mutex_unlock(priv->bufferMutex);
338 }
339
340 static gboolean webkitVideoSinkUnlock(GstBaseSink* baseSink)
341 {
342     WebKitVideoSink* sink = WEBKIT_VIDEO_SINK(baseSink);
343
344     unlockBufferMutex(sink->priv);
345
346     return GST_CALL_PARENT_WITH_DEFAULT(GST_BASE_SINK_CLASS, unlock, (baseSink), TRUE);
347 }
348
349 static gboolean webkitVideoSinkUnlockStop(GstBaseSink* baseSink)
350 {
351     WebKitVideoSinkPrivate* priv = WEBKIT_VIDEO_SINK(baseSink)->priv;
352
353     g_mutex_lock(priv->bufferMutex);
354     priv->unlocked = false;
355     g_mutex_unlock(priv->bufferMutex);
356
357     return GST_CALL_PARENT_WITH_DEFAULT(GST_BASE_SINK_CLASS, unlock_stop, (baseSink), TRUE);
358 }
359
360 static gboolean webkitVideoSinkStop(GstBaseSink* baseSink)
361 {
362     WebKitVideoSinkPrivate* priv = WEBKIT_VIDEO_SINK(baseSink)->priv;
363
364     unlockBufferMutex(priv);
365
366     if (priv->currentCaps) {
367         gst_caps_unref(priv->currentCaps);
368         priv->currentCaps = 0;
369     }
370
371     return TRUE;
372 }
373
374 static gboolean webkitVideoSinkStart(GstBaseSink* baseSink)
375 {
376     WebKitVideoSinkPrivate* priv = WEBKIT_VIDEO_SINK(baseSink)->priv;
377
378     g_mutex_lock(priv->bufferMutex);
379     priv->unlocked = false;
380     g_mutex_unlock(priv->bufferMutex);
381     return TRUE;
382 }
383
384 static gboolean webkitVideoSinkSetCaps(GstBaseSink* baseSink, GstCaps* caps)
385 {
386     WebKitVideoSink* sink = WEBKIT_VIDEO_SINK(baseSink);
387     WebKitVideoSinkPrivate* priv = sink->priv;
388
389     GST_DEBUG_OBJECT(sink, "Current caps %" GST_PTR_FORMAT ", setting caps %" GST_PTR_FORMAT, priv->currentCaps, caps);
390
391 #ifdef GST_API_VERSION_1
392     GstVideoInfo info;
393     if (!gst_video_info_from_caps(&info, caps)) {
394         GST_ERROR_OBJECT(sink, "Invalid caps %" GST_PTR_FORMAT, caps);
395         return FALSE;
396     }
397 #endif
398
399     gst_caps_replace(&priv->currentCaps, caps);
400     return TRUE;
401 }
402
403 #ifdef GST_API_VERSION_1
404 static gboolean webkitVideoSinkProposeAllocation(GstBaseSink* baseSink, GstQuery* query)
405 {
406     GstCaps* caps;
407     gst_query_parse_allocation(query, &caps, 0);
408     if (!caps)
409         return FALSE;
410
411     WebKitVideoSink* sink = WEBKIT_VIDEO_SINK(baseSink);
412     if (!gst_video_info_from_caps(&sink->priv->info, caps))
413         return FALSE;
414
415     gst_query_add_allocation_meta(query, GST_VIDEO_META_API_TYPE, 0);
416     gst_query_add_allocation_meta(query, GST_VIDEO_CROP_META_API_TYPE, 0);
417 #if GST_CHECK_VERSION(1, 1, 0)
418     gst_query_add_allocation_meta(query, GST_VIDEO_GL_TEXTURE_UPLOAD_META_API_TYPE, 0);
419 #endif
420     return TRUE;
421 }
422 #endif
423
424 #ifndef GST_API_VERSION_1
425 static void webkitVideoSinkMarshalVoidAndMiniObject(GClosure* closure, GValue*, guint parametersNumber, const GValue* parameterValues, gpointer, gpointer marshalData)
426 {
427     typedef void (*marshalfunc_VOID__MINIOBJECT) (gpointer obj, gpointer arg1, gpointer data2);
428     marshalfunc_VOID__MINIOBJECT callback;
429     GCClosure* cclosure = reinterpret_cast<GCClosure*>(closure);
430     gpointer data1, data2;
431
432     g_return_if_fail(parametersNumber == 2);
433
434     if (G_CCLOSURE_SWAP_DATA(closure)) {
435         data1 = closure->data;
436         data2 = g_value_peek_pointer(parameterValues + 0);
437     } else {
438         data1 = g_value_peek_pointer(parameterValues + 0);
439         data2 = closure->data;
440     }
441
442     callback = (marshalfunc_VOID__MINIOBJECT) (marshalData ? marshalData : cclosure->callback);
443     callback(data1, gst_value_get_mini_object(parameterValues + 1), data2);
444 }
445 #endif
446
447 static void webkit_video_sink_class_init(WebKitVideoSinkClass* klass)
448 {
449     GObjectClass* gobjectClass = G_OBJECT_CLASS(klass);
450     GstBaseSinkClass* baseSinkClass = GST_BASE_SINK_CLASS(klass);
451     GstElementClass* elementClass = GST_ELEMENT_CLASS(klass);
452
453     gst_element_class_add_pad_template(elementClass, gst_static_pad_template_get(&s_sinkTemplate));
454     setGstElementClassMetadata(elementClass, "WebKit video sink", "Sink/Video", "Sends video data from a GStreamer pipeline to a Cairo surface", "Alp Toker <alp@atoker.com>");
455
456     g_type_class_add_private(klass, sizeof(WebKitVideoSinkPrivate));
457
458     gobjectClass->dispose = webkitVideoSinkDispose;
459     gobjectClass->get_property = webkitVideoSinkGetProperty;
460
461     baseSinkClass->unlock = webkitVideoSinkUnlock;
462     baseSinkClass->unlock_stop = webkitVideoSinkUnlockStop;
463     baseSinkClass->render = webkitVideoSinkRender;
464     baseSinkClass->preroll = webkitVideoSinkRender;
465     baseSinkClass->stop = webkitVideoSinkStop;
466     baseSinkClass->start = webkitVideoSinkStart;
467     baseSinkClass->set_caps = webkitVideoSinkSetCaps;
468 #ifdef GST_API_VERSION_1
469     baseSinkClass->propose_allocation = webkitVideoSinkProposeAllocation;
470 #endif
471
472     g_object_class_install_property(gobjectClass, PROP_CAPS,
473         g_param_spec_boxed("current-caps", "Current-Caps", "Current caps", GST_TYPE_CAPS, G_PARAM_READABLE));
474
475     webkitVideoSinkSignals[REPAINT_REQUESTED] = g_signal_new("repaint-requested",
476             G_TYPE_FROM_CLASS(klass),
477             static_cast<GSignalFlags>(G_SIGNAL_RUN_LAST | G_SIGNAL_ACTION),
478             0, // Class offset
479             0, // Accumulator
480             0, // Accumulator data
481 #ifndef GST_API_VERSION_1
482             webkitVideoSinkMarshalVoidAndMiniObject,
483 #else
484             g_cclosure_marshal_generic,
485 #endif
486             G_TYPE_NONE, // Return type
487             1, // Only one parameter
488             GST_TYPE_BUFFER);
489 }
490
491
492 #if USE(NATIVE_FULLSCREEN_VIDEO)
493 GstElement* webkitVideoSinkNew(WebCore::GStreamerGWorld* gstGWorld)
494 {
495     GstElement* element = GST_ELEMENT(g_object_new(WEBKIT_TYPE_VIDEO_SINK, 0));
496     WEBKIT_VIDEO_SINK(element)->priv->gstGWorld = gstGWorld;
497     return element;
498 }
499 #else
500 GstElement* webkitVideoSinkNew()
501 {
502     return GST_ELEMENT(g_object_new(WEBKIT_TYPE_VIDEO_SINK, 0));
503 }
504 #endif
505
506 #endif // ENABLE(VIDEO) && USE(GSTREAMER)