[GTK] Copying video textures to webgl should not depend on cairo-gl
authorcommit-queue@webkit.org <commit-queue@webkit.org@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Thu, 6 Oct 2016 10:53:28 +0000 (10:53 +0000)
committercommit-queue@webkit.org <commit-queue@webkit.org@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Thu, 6 Oct 2016 10:53:28 +0000 (10:53 +0000)
https://bugs.webkit.org/show_bug.cgi?id=162904

Patch by Miguel Gomez <magomez@igalia.com> on 2016-10-06
Reviewed by Žan Doberšek.

Perform the texture copy without using cairo-gl.

Covered by existent tests.

* platform/GStreamer.cmake:
Add the new VideoTextureCopierGStreamer class to the build.
* platform/graphics/gstreamer/MediaPlayerPrivateGStreamerBase.cpp:
(WebCore::MediaPlayerPrivateGStreamerBase::copyVideoTextureToPlatformTexture):
Use VideoTextureCopierGStreamer to perform the copy.
* platform/graphics/gstreamer/MediaPlayerPrivateGStreamerBase.h:
Add a VideoTextureCopierGStreamer as a class attribute.
* platform/graphics/gstreamer/VideoTextureCopierGStreamer.cpp: Added.
(WebCore::VideoTextureCopierGStreamer::VideoTextureCopierGStreamer):
(WebCore::VideoTextureCopierGStreamer::~VideoTextureCopierGStreamer):
(WebCore::VideoTextureCopierGStreamer::updateTextureSpaceMatrix):
Recalculates the matrix used as the texture coordinates transformation.
(WebCore::VideoTextureCopierGStreamer::updateTransformationMatrix):
Recalculates the matrices used as the vertices coordinates transformation.
(WebCore::VideoTextureCopierGStreamer::copyVideoTextureToPlatformTexture):
Performs the texture copy by using a shader that applies the rotation needed to follow
the video orientation.
* platform/graphics/gstreamer/VideoTextureCopierGStreamer.h: Added.

git-svn-id: https://svn.webkit.org/repository/webkit/trunk@206862 268f45cc-cd09-0410-ab3c-d52691b4dbfc

Source/WebCore/ChangeLog
Source/WebCore/platform/GStreamer.cmake
Source/WebCore/platform/graphics/gstreamer/MediaPlayerPrivateGStreamerBase.cpp
Source/WebCore/platform/graphics/gstreamer/MediaPlayerPrivateGStreamerBase.h
Source/WebCore/platform/graphics/gstreamer/VideoTextureCopierGStreamer.cpp [new file with mode: 0644]
Source/WebCore/platform/graphics/gstreamer/VideoTextureCopierGStreamer.h [new file with mode: 0644]

index fbe4bd7..5923850 100644 (file)
@@ -1,3 +1,33 @@
+2016-10-06  Miguel Gomez  <magomez@igalia.com>
+
+        [GTK] Copying video textures to webgl should not depend on cairo-gl
+        https://bugs.webkit.org/show_bug.cgi?id=162904
+
+        Reviewed by Žan Doberšek.
+
+        Perform the texture copy without using cairo-gl.
+
+        Covered by existent tests.
+
+        * platform/GStreamer.cmake:
+        Add the new VideoTextureCopierGStreamer class to the build.
+        * platform/graphics/gstreamer/MediaPlayerPrivateGStreamerBase.cpp:
+        (WebCore::MediaPlayerPrivateGStreamerBase::copyVideoTextureToPlatformTexture):
+        Use VideoTextureCopierGStreamer to perform the copy.
+        * platform/graphics/gstreamer/MediaPlayerPrivateGStreamerBase.h:
+        Add a VideoTextureCopierGStreamer as a class attribute.
+        * platform/graphics/gstreamer/VideoTextureCopierGStreamer.cpp: Added.
+        (WebCore::VideoTextureCopierGStreamer::VideoTextureCopierGStreamer):
+        (WebCore::VideoTextureCopierGStreamer::~VideoTextureCopierGStreamer):
+        (WebCore::VideoTextureCopierGStreamer::updateTextureSpaceMatrix):
+        Recalculates the matrix used as the texture coordinates transformation.
+        (WebCore::VideoTextureCopierGStreamer::updateTransformationMatrix):
+        Recalculates the matrices used as the vertices coordinates transformation.
+        (WebCore::VideoTextureCopierGStreamer::copyVideoTextureToPlatformTexture):
+        Performs the texture copy by using a shader that applies the rotation needed to follow
+        the video orientation.
+        * platform/graphics/gstreamer/VideoTextureCopierGStreamer.h: Added.
+
 2016-10-06  Gwang Yoon Hwang  <yoon@igalia.com>
 
         [GTK] Build fix for X11 and GStreamerGL after r183731
index caf5373..031c9f1 100644 (file)
@@ -88,6 +88,9 @@ if (ENABLE_VIDEO)
         list(APPEND WebCore_LIBRARIES
             ${GSTREAMER_GL_LIBRARIES}
         )
+        list(APPEND WebCore_SOURCES
+            platform/graphics/gstreamer/VideoTextureCopierGStreamer.cpp
+        )
     endif ()
 endif ()
 
index 34735f6..cc385f6 100644 (file)
@@ -74,6 +74,7 @@
 #if PLATFORM(X11) && GST_GL_HAVE_PLATFORM_EGL
 #undef None
 #endif // PLATFORM(X11) && GST_GL_HAVE_PLATFORM_EGL
+#include "VideoTextureCopierGStreamer.h"
 #endif // USE(GSTREAMER_GL)
 
 #if USE(TEXTURE_MAPPER_GL)
@@ -781,9 +782,8 @@ bool MediaPlayerPrivateGStreamerBase::paintToCairoSurface(cairo_surface_t* outpu
 
 bool MediaPlayerPrivateGStreamerBase::copyVideoTextureToPlatformTexture(GraphicsContext3D* context, Platform3DObject outputTexture, GC3Denum outputTarget, GC3Dint level, GC3Denum internalFormat, GC3Denum format, GC3Denum type, bool premultiplyAlpha, bool flipY)
 {
-#if !USE(CAIRO)
-    return false;
-#endif
+#if USE(GSTREAMER_GL)
+    UNUSED_PARAM(context);
 
     if (m_usingFallbackVideoSink)
         return false;
@@ -791,31 +791,33 @@ bool MediaPlayerPrivateGStreamerBase::copyVideoTextureToPlatformTexture(Graphics
     if (premultiplyAlpha)
         return false;
 
-    GstVideoInfo videoInfo;
-    IntSize size, rotatedSize;
     WTF::GMutexLocker<GMutex> lock(m_sampleMutex);
-    GLContext* glContext = prepareContextForCairoPaint(videoInfo, size, rotatedSize);
-    if (!glContext)
+
+    GstVideoInfo videoInfo;
+    if (!getSampleVideoInfo(m_sample.get(), videoInfo))
+        return false;
+
+    GstBuffer* buffer = gst_sample_get_buffer(m_sample.get());
+    GstVideoFrame videoFrame;
+    if (!gst_video_frame_map(&videoFrame, &videoInfo, buffer, static_cast<GstMapFlags>(GST_MAP_READ | GST_MAP_GL)))
         return false;
 
-    // Allocate uninitialized memory for the output texture.
-    context->bindTexture(outputTarget, outputTexture);
-    context->texImage2DDirect(outputTarget, level, internalFormat, rotatedSize.width(), rotatedSize.height(), 0, format, type, nullptr);
+    IntSize size(GST_VIDEO_INFO_WIDTH(&videoInfo), GST_VIDEO_INFO_HEIGHT(&videoInfo));
+    if (m_videoSourceOrientation.usesWidthAsHeight())
+        size = size.transposedSize();
+    unsigned textureID = *reinterpret_cast<unsigned*>(videoFrame.data[0]);
 
-    // cairo_gl_surface_create_for_texture sets these parameters to GL_NEAREST, so we need to backup them.
-    GC3Dint minFilter, magFilter;
-    context->getTexParameteriv(outputTarget, GL_TEXTURE_MIN_FILTER, &minFilter);
-    context->getTexParameteriv(outputTarget, GL_TEXTURE_MAG_FILTER, &magFilter);
+    if (!m_videoTextureCopier)
+        m_videoTextureCopier = std::make_unique<VideoTextureCopierGStreamer>();
 
-    RefPtr<cairo_surface_t> outputSurface = adoptRef(cairo_gl_surface_create_for_texture(glContext->cairoDevice(), CAIRO_CONTENT_COLOR_ALPHA, outputTexture, rotatedSize.width(), rotatedSize.height()));
-    if (!paintToCairoSurface(outputSurface.get(), glContext->cairoDevice(), videoInfo, size, rotatedSize, flipY))
-        return false;
+    bool copied = m_videoTextureCopier->copyVideoTextureToPlatformTexture(textureID, size, outputTexture, outputTarget, level, internalFormat, format, type, flipY, m_videoSourceOrientation);
 
-    context->bindTexture(outputTarget, outputTexture);
-    context->texParameteri(outputTarget, GraphicsContext3D::TEXTURE_MIN_FILTER, minFilter);
-    context->texParameteri(outputTarget, GraphicsContext3D::TEXTURE_MAG_FILTER, magFilter);
+    gst_video_frame_unmap(&videoFrame);
 
-    return true;
+    return copied;
+#else
+    return false;
+#endif
 }
 
 NativeImagePtr MediaPlayerPrivateGStreamerBase::nativeImageForCurrentTime()
index 36bd8a5..2b4e198 100644 (file)
@@ -52,6 +52,7 @@ class GraphicsContext;
 class GraphicsContext3D;
 class IntSize;
 class IntRect;
+class VideoTextureCopierGStreamer;
 
 class MediaPlayerPrivateGStreamerBase : public MediaPlayerPrivateInterface
 #if USE(COORDINATED_GRAPHICS_THREADED) || (USE(TEXTURE_MAPPER_GL) && !USE(COORDINATED_GRAPHICS))
@@ -205,6 +206,10 @@ protected:
 #endif
 
     ImageOrientation m_videoSourceOrientation;
+
+#if USE(GSTREAMER_GL)
+    std::unique_ptr<VideoTextureCopierGStreamer> m_videoTextureCopier;
+#endif
 };
 }
 
diff --git a/Source/WebCore/platform/graphics/gstreamer/VideoTextureCopierGStreamer.cpp b/Source/WebCore/platform/graphics/gstreamer/VideoTextureCopierGStreamer.cpp
new file mode 100644 (file)
index 0000000..abb8b9b
--- /dev/null
@@ -0,0 +1,190 @@
+/*
+ Copyright (C) 2016 Igalia S.L.
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Library General Public
+ License as published by the Free Software Foundation; either
+ version 2 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ Library General Public License for more details.
+
+ You should have received a copy of the GNU Library General Public License
+ along with this library; see the file COPYING.LIB.  If not, write to
+ the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ Boston, MA 02110-1301, USA.
+ */
+
+
+#include "config.h"
+#include "VideoTextureCopierGStreamer.h"
+
+#if USE(GSTREAMER_GL)
+
+#include "GLContext.h"
+#include "ImageOrientation.h"
+#include "TextureMapperShaderProgram.h"
+
+namespace WebCore {
+
+VideoTextureCopierGStreamer::VideoTextureCopierGStreamer()
+{
+    GLContext* previousContext = GLContext::current();
+    ASSERT(previousContext);
+    PlatformDisplay::sharedDisplayForCompositing().sharingGLContext()->makeContextCurrent();
+
+    m_context3D = GraphicsContext3D::createForCurrentGLContext();
+
+    m_shaderProgram = TextureMapperShaderProgram::create(*m_context3D, TextureMapperShaderProgram::Texture);
+
+    m_framebuffer = m_context3D->createFramebuffer();
+
+    static const GLfloat vertices[] = { 0, 0, 1, 0, 1, 1, 0, 1 };
+    m_vbo = m_context3D->createBuffer();
+    m_context3D->bindBuffer(GraphicsContext3D::ARRAY_BUFFER, m_vbo);
+    m_context3D->bufferData(GraphicsContext3D::ARRAY_BUFFER, sizeof(GC3Dfloat) * 8, vertices, GraphicsContext3D::STATIC_DRAW);
+
+    updateTextureSpaceMatrix();
+
+    previousContext->makeContextCurrent();
+}
+
+VideoTextureCopierGStreamer::~VideoTextureCopierGStreamer()
+{
+    GLContext* previousContext = GLContext::current();
+    ASSERT(previousContext);
+    PlatformDisplay::sharedDisplayForCompositing().sharingGLContext()->makeContextCurrent();
+
+    m_context3D->deleteFramebuffer(m_framebuffer);
+    m_context3D->deleteBuffer(m_vbo);
+    m_shaderProgram = nullptr;
+    m_context3D = nullptr;
+
+    previousContext->makeContextCurrent();
+}
+
+void VideoTextureCopierGStreamer::updateTextureSpaceMatrix()
+{
+    m_textureSpaceMatrix.makeIdentity();
+
+    switch (m_orientation) {
+    case OriginRightTop:
+        m_textureSpaceMatrix.rotate(-90);
+        m_textureSpaceMatrix.translate(-1, 0);
+        break;
+    case OriginBottomRight:
+        m_textureSpaceMatrix.rotate(180);
+        m_textureSpaceMatrix.translate(-1, -1);
+        break;
+    case OriginLeftBottom:
+        m_textureSpaceMatrix.rotate(-270);
+        m_textureSpaceMatrix.translate(0, -1);
+        break;
+    default:
+        ASSERT_NOT_REACHED();
+    }
+
+    if (!m_flipY) {
+        m_textureSpaceMatrix.flipY();
+        m_textureSpaceMatrix.translate(0, -1);
+    }
+}
+
+void VideoTextureCopierGStreamer::updateTransformationMatrix()
+{
+    FloatRect targetRect = FloatRect(FloatPoint(), m_size);
+    TransformationMatrix identityMatrix;
+    m_modelViewMatrix = TransformationMatrix(identityMatrix).multiply(TransformationMatrix::rectToRect(FloatRect(0, 0, 1, 1), targetRect));
+
+    // Taken from TextureMapperGL.
+    const float nearValue = 9999999;
+    const float farValue = -99999;
+
+    m_projectionMatrix = TransformationMatrix(2.0 / float(m_size.width()), 0, 0, 0,
+        0, (-2.0) / float(m_size.height()), 0, 0,
+        0, 0, -2.f / (farValue - nearValue), 0,
+        -1, 1, -(farValue + nearValue) / (farValue - nearValue), 1);
+}
+
+bool VideoTextureCopierGStreamer::copyVideoTextureToPlatformTexture(Platform3DObject inputTexture, IntSize& frameSize, Platform3DObject outputTexture, GC3Denum outputTarget, GC3Dint level, GC3Denum internalFormat, GC3Denum format, GC3Denum type, bool flipY, ImageOrientation& sourceOrientation)
+{
+    if (!m_shaderProgram || !m_framebuffer || !m_vbo || frameSize.isEmpty())
+        return false;
+
+    if (m_size != frameSize) {
+        m_size = frameSize;
+        updateTransformationMatrix();
+    }
+
+    if (m_flipY != flipY || m_orientation != sourceOrientation) {
+        m_flipY = flipY;
+        m_orientation = sourceOrientation;
+        updateTextureSpaceMatrix();
+    }
+
+    // Save previous context and activate the sharing one.
+    GLContext* previousContext = GLContext::current();
+    ASSERT(previousContext);
+    PlatformDisplay::sharedDisplayForCompositing().sharingGLContext()->makeContextCurrent();
+
+    // Save previous bound framebuffer, texture and viewport.
+    GC3Dint boundFramebuffer = 0;
+    GC3Dint boundTexture = 0;
+    GC3Dint previousViewport[4] = { 0, 0, 0, 0};
+    m_context3D->getIntegerv(GraphicsContext3D::FRAMEBUFFER_BINDING, &boundFramebuffer);
+    m_context3D->getIntegerv(GraphicsContext3D::TEXTURE_BINDING_2D, &boundTexture);
+    m_context3D->getIntegerv(GraphicsContext3D::VIEWPORT, previousViewport);
+
+    // Set proper parameters to the output texture and allocate uninitialized memory for it.
+    m_context3D->bindTexture(outputTarget, outputTexture);
+    m_context3D->texParameteri(GraphicsContext3D::TEXTURE_2D, GraphicsContext3D::TEXTURE_MIN_FILTER, GraphicsContext3D::LINEAR);
+    m_context3D->texParameterf(GraphicsContext3D::TEXTURE_2D, GraphicsContext3D::TEXTURE_WRAP_S, GraphicsContext3D::CLAMP_TO_EDGE);
+    m_context3D->texParameterf(GraphicsContext3D::TEXTURE_2D, GraphicsContext3D::TEXTURE_WRAP_T, GraphicsContext3D::CLAMP_TO_EDGE);
+    m_context3D->texImage2DDirect(outputTarget, level, internalFormat, m_size.width(), m_size.height(), 0, format, type, nullptr);
+
+    // Bind framebuffer to paint and attach the destination texture to it.
+    m_context3D->bindFramebuffer(GraphicsContext3D::FRAMEBUFFER, m_framebuffer);
+    m_context3D->framebufferTexture2D(GraphicsContext3D::FRAMEBUFFER, GraphicsContext3D::COLOR_ATTACHMENT0, GL_TEXTURE_2D, outputTexture, 0);
+
+    // Set proper wrap parameter to the source texture.
+    m_context3D->bindTexture(GL_TEXTURE_2D, inputTexture);
+    m_context3D->texParameteri(GraphicsContext3D::TEXTURE_2D, GraphicsContext3D::TEXTURE_MIN_FILTER, GraphicsContext3D::LINEAR);
+    m_context3D->texParameterf(GraphicsContext3D::TEXTURE_2D, GraphicsContext3D::TEXTURE_WRAP_S, GraphicsContext3D::CLAMP_TO_EDGE);
+    m_context3D->texParameterf(GraphicsContext3D::TEXTURE_2D, GraphicsContext3D::TEXTURE_WRAP_T, GraphicsContext3D::CLAMP_TO_EDGE);
+
+    // Set the viewport.
+    m_context3D->viewport(0, 0, m_size.width(), m_size.height());
+
+    // Set program parameters.
+    m_context3D->useProgram(m_shaderProgram->programID());
+    m_context3D->uniform1i(m_shaderProgram->samplerLocation(), 0);
+    m_shaderProgram->setMatrix(m_shaderProgram->modelViewMatrixLocation(), m_modelViewMatrix);
+    m_shaderProgram->setMatrix(m_shaderProgram->projectionMatrixLocation(), m_projectionMatrix);
+    m_shaderProgram->setMatrix(m_shaderProgram->textureSpaceMatrixLocation(), m_textureSpaceMatrix);
+
+    // Perform the copy.
+    m_context3D->enableVertexAttribArray(m_shaderProgram->vertexLocation());
+    m_context3D->bindBuffer(GraphicsContext3D::ARRAY_BUFFER, m_vbo);
+    m_context3D->vertexAttribPointer(m_shaderProgram->vertexLocation(), 2, GraphicsContext3D::FLOAT, false, 0, 0);
+    m_context3D->drawArrays(GraphicsContext3D::TRIANGLE_FAN, 0, 4);
+    m_context3D->bindBuffer(GraphicsContext3D::ARRAY_BUFFER, 0);
+    m_context3D->disableVertexAttribArray(m_shaderProgram->vertexLocation());
+    m_context3D->useProgram(0);
+
+    // Restore previous bindings and viewport.
+    m_context3D->bindFramebuffer(GraphicsContext3D::FRAMEBUFFER, boundFramebuffer);
+    m_context3D->bindTexture(outputTarget, boundTexture);
+    m_context3D->viewport(previousViewport[0], previousViewport[1], previousViewport[2], previousViewport[3]);
+
+    bool ok = (m_context3D->getError() == GraphicsContext3D::NO_ERROR);
+
+    // Restore previous context.
+    previousContext->makeContextCurrent();
+    return ok;
+}
+
+} // namespace WebCore
+
+#endif // USE(GSTREAMER_GL)
diff --git a/Source/WebCore/platform/graphics/gstreamer/VideoTextureCopierGStreamer.h b/Source/WebCore/platform/graphics/gstreamer/VideoTextureCopierGStreamer.h
new file mode 100644 (file)
index 0000000..945a7b3
--- /dev/null
@@ -0,0 +1,59 @@
+/*
+ Copyright (C) 2016 Igalia S.L.
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Library General Public
+ License as published by the Free Software Foundation; either
+ version 2 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ Library General Public License for more details.
+
+ You should have received a copy of the GNU Library General Public License
+ along with this library; see the file COPYING.LIB.  If not, write to
+ the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ Boston, MA 02110-1301, USA.
+ */
+
+#ifndef VideoTextureCopierGStreamer_h
+#define VideoTextureCopierGStreamer_h
+
+#if USE(GSTREAMER_GL)
+
+#include "GraphicsContext3D.h"
+#include "TransformationMatrix.h"
+
+namespace WebCore {
+
+class TextureMapperShaderProgram;
+class ImageOrientation;
+
+class VideoTextureCopierGStreamer {
+public:
+    VideoTextureCopierGStreamer();
+    ~VideoTextureCopierGStreamer();
+
+    bool copyVideoTextureToPlatformTexture(Platform3DObject inputTexture, IntSize& frameSize, Platform3DObject outputTexture, GC3Denum outputTarget, GC3Dint level, GC3Denum internalFormat, GC3Denum format, GC3Denum type, bool flipY, ImageOrientation& sourceOrientation);
+    void updateTextureSpaceMatrix();
+    void updateTransformationMatrix();
+
+private:
+    RefPtr<GraphicsContext3D> m_context3D;
+    RefPtr<TextureMapperShaderProgram> m_shaderProgram;
+    Platform3DObject m_framebuffer { 0 };
+    Platform3DObject m_vbo { 0 };
+    bool m_flipY { false };
+    ImageOrientation m_orientation;
+    IntSize m_size;
+    TransformationMatrix m_modelViewMatrix;
+    TransformationMatrix m_projectionMatrix;
+    TransformationMatrix m_textureSpaceMatrix;
+};
+
+} // namespace WebCore
+
+#endif // USE(GSTREAMER_GL)
+
+#endif // VideoTextureCopierGStreamer_h