Implement accelerated video-to-texture upload path for ANGLE backend for WebGL on...
authordino@apple.com <dino@apple.com@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Thu, 21 Nov 2019 18:53:59 +0000 (18:53 +0000)
committerdino@apple.com <dino@apple.com@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Thu, 21 Nov 2019 18:53:59 +0000 (18:53 +0000)
https://bugs.webkit.org/show_bug.cgi?id=200904

Support GPU-accelerated video uploads to WebGL textures with ANGLE.

Uses the IOSurface path, as the CVOpenGLTextureCache seems to have
been disabled at the OS level, even on top-of-tree WebKit without
using ANGLE.

Uses the EGL_ANGLE_iosurface_client_buffer extension to import
IOSurfaces; reuses all of the existing shader and OpenGL code.

Necessary other fixes to GraphicsContext3D's ANGLE backend: supporting
context sharing, and querying the selected EGLConfig.

Covered by WebGL conformance tests. Verified speedup with
"Animating textures in WebGL" MDN article.

Patch by Kenneth Russell <kbr@chromium.org> on 2019-11-21
Reviewed by Dean Jackson.

* platform/graphics/GraphicsContext3D.h:
(WebCore::GraphicsContext3D::platformDisplay const):
(WebCore::GraphicsContext3D::platformConfig const):
* platform/graphics/cocoa/GraphicsContext3DCocoa.mm:
(WebCore::GraphicsContext3D::GraphicsContext3D):
* platform/graphics/cv/VideoTextureCopierCV.cpp:
(WebCore::VideoTextureCopierCV::initializeContextObjects):
(WebCore::VideoTextureCopierCV::initializeUVContextObjects):
(WebCore::VideoTextureCopierCV::attachIOSurfaceToTexture):
(WebCore::VideoTextureCopierCV::detachIOSurfaceFromTexture):
(WebCore::VideoTextureCopierCV::copyImageToPlatformTexture):
(WebCore::VideoTextureCopierCV::copyVideoTextureToPlatformTexture):
* platform/graphics/cv/VideoTextureCopierCV.h:

git-svn-id: https://svn.webkit.org/repository/webkit/trunk@252741 268f45cc-cd09-0410-ab3c-d52691b4dbfc

Source/WebCore/ChangeLog
Source/WebCore/platform/graphics/GraphicsContext3D.h
Source/WebCore/platform/graphics/cocoa/GraphicsContext3DCocoa.mm
Source/WebCore/platform/graphics/cv/VideoTextureCopierCV.cpp
Source/WebCore/platform/graphics/cv/VideoTextureCopierCV.h

index 3489044..4d4b684 100644 (file)
@@ -1,3 +1,39 @@
+2019-11-21  Kenneth Russell  <kbr@chromium.org>
+
+        Implement accelerated video-to-texture upload path for ANGLE backend for WebGL on desktop
+        https://bugs.webkit.org/show_bug.cgi?id=200904
+
+        Support GPU-accelerated video uploads to WebGL textures with ANGLE.
+
+        Uses the IOSurface path, as the CVOpenGLTextureCache seems to have
+        been disabled at the OS level, even on top-of-tree WebKit without
+        using ANGLE.
+
+        Uses the EGL_ANGLE_iosurface_client_buffer extension to import
+        IOSurfaces; reuses all of the existing shader and OpenGL code.
+
+        Necessary other fixes to GraphicsContext3D's ANGLE backend: supporting
+        context sharing, and querying the selected EGLConfig.
+
+        Covered by WebGL conformance tests. Verified speedup with
+        "Animating textures in WebGL" MDN article.
+
+        Reviewed by Dean Jackson.
+
+        * platform/graphics/GraphicsContext3D.h:
+        (WebCore::GraphicsContext3D::platformDisplay const):
+        (WebCore::GraphicsContext3D::platformConfig const):
+        * platform/graphics/cocoa/GraphicsContext3DCocoa.mm:
+        (WebCore::GraphicsContext3D::GraphicsContext3D):
+        * platform/graphics/cv/VideoTextureCopierCV.cpp:
+        (WebCore::VideoTextureCopierCV::initializeContextObjects):
+        (WebCore::VideoTextureCopierCV::initializeUVContextObjects):
+        (WebCore::VideoTextureCopierCV::attachIOSurfaceToTexture):
+        (WebCore::VideoTextureCopierCV::detachIOSurfaceFromTexture):
+        (WebCore::VideoTextureCopierCV::copyImageToPlatformTexture):
+        (WebCore::VideoTextureCopierCV::copyVideoTextureToPlatformTexture):
+        * platform/graphics/cv/VideoTextureCopierCV.h:
+
 2019-11-21  Youenn Fablet  <youenn@apple.com>
 
         Regression (r252660): Layout Test platform/ios/mediastream/audio-muted-in-background-tab.html is failing
index 619fb4a..6e6cb4d 100644 (file)
@@ -742,8 +742,11 @@ public:
         PRIMITIVE_RESTART_FIXED_INDEX = 0x8D69,
         PRIMITIVE_RESTART = 0x8F9D,
 
-        // OpenGL ES 3 constants
-        MAP_READ_BIT = 0x0001
+        // OpenGL ES 3 constants.
+        MAP_READ_BIT = 0x0001,
+
+        // Necessary desktop OpenGL constants.
+        TEXTURE_RECTANGLE_ARB = 0x84F5
     };
 
     enum RenderStyle {
@@ -777,6 +780,10 @@ public:
     PlatformGraphicsContext3D platformGraphicsContext3D() const { return m_contextObj; }
     Platform3DObject platformTexture() const { return m_texture; }
     CALayer* platformLayer() const { return reinterpret_cast<CALayer*>(m_webGLLayer.get()); }
+#if USE(ANGLE)
+    PlatformGraphicsContext3DDisplay platformDisplay() const { return m_displayObj; }
+    PlatformGraphicsContext3DConfig platformConfig() const { return m_configObj; }
+#endif // USE(ANGLE)
 #else
     PlatformGraphicsContext3D platformGraphicsContext3D();
     Platform3DObject platformTexture() const;
@@ -1380,6 +1387,7 @@ private:
     PlatformGraphicsContext3D m_contextObj { nullptr };
 #if USE(ANGLE)
     PlatformGraphicsContext3DDisplay m_displayObj { nullptr };
+    PlatformGraphicsContext3DConfig m_configObj { nullptr };
 #endif // USE(ANGLE)
 #endif // PLATFORM(COCOA)
 
index d1af803..0d5837c 100644 (file)
@@ -290,7 +290,6 @@ GraphicsContext3D::GraphicsContext3D(GraphicsContext3DAttributes attrs, HostWind
 #elif USE(ANGLE)
 
     UNUSED_PARAM(hostWindow);
-    UNUSED_PARAM(sharedContext);
 
     m_displayObj = EGL_GetDisplay(EGL_DEFAULT_DISPLAY);
     if (m_displayObj == EGL_NO_DISPLAY)
@@ -304,7 +303,6 @@ GraphicsContext3D::GraphicsContext3D(GraphicsContext3DAttributes attrs, HostWind
     const char *displayExtensions = EGL_QueryString(m_displayObj, EGL_EXTENSIONS);
     LOG(WebGL, "Extensions: %s", displayExtensions);
 
-    EGLConfig config;
     EGLint configAttributes[] = {
         EGL_RENDERABLE_TYPE, EGL_OPENGL_ES2_BIT,
         EGL_RED_SIZE, 8,
@@ -314,7 +312,7 @@ GraphicsContext3D::GraphicsContext3D(GraphicsContext3DAttributes attrs, HostWind
         EGL_NONE
     };
     EGLint numberConfigsReturned = 0;
-    EGL_ChooseConfig(m_displayObj, configAttributes, &config, 1, &numberConfigsReturned);
+    EGL_ChooseConfig(m_displayObj, configAttributes, &m_configObj, 1, &numberConfigsReturned);
     if (numberConfigsReturned != 1) {
         LOG(WebGL, "EGLConfig Initialization failed.");
         return;
@@ -349,7 +347,7 @@ GraphicsContext3D::GraphicsContext3D(GraphicsContext3DAttributes attrs, HostWind
     }
     contextAttributes.push_back(EGL_NONE);
 
-    m_contextObj = EGL_CreateContext(m_displayObj, config, EGL_NO_CONTEXT, contextAttributes.data());
+    m_contextObj = EGL_CreateContext(m_displayObj, m_configObj, sharedContext ? static_cast<EGLContext>(sharedContext->m_contextObj) : EGL_NO_CONTEXT, contextAttributes.data());
     if (m_contextObj == EGL_NO_CONTEXT) {
         LOG(WebGL, "EGLContext Initialization failed.");
         return;
@@ -387,7 +385,7 @@ GraphicsContext3D::GraphicsContext3D(GraphicsContext3DAttributes attrs, HostWind
         [m_webGLLayer setName:@"WebGL Layer"];
 #endif
 #if USE(ANGLE)
-        [m_webGLLayer setEGLDisplay:m_displayObj config:config];
+        [m_webGLLayer setEGLDisplay:m_displayObj config:m_configObj];
 #endif
     END_BLOCK_OBJC_EXCEPTIONS
 
index d7a6ba1..120724d 100644 (file)
 #include <OpenGLES/ES3/glext.h>
 #endif
 
+#if USE(ANGLE)
+#define EGL_EGL_PROTOTYPES 0
+#include <ANGLE/egl.h>
+#include <ANGLE/eglext.h>
+#include <ANGLE/eglext_angle.h>
+#include <ANGLE/entry_points_egl.h>
+#include <ANGLE/entry_points_gles_2_0_autogen.h>
+// Skip the inclusion of ANGLE's explicit context entry points for now.
+#define GL_ANGLE_explicit_context
+#include <ANGLE/gl2ext.h>
+#include <ANGLE/gl2ext_angle.h>
+#endif
+
 #include "CoreVideoSoftLink.h"
 
 namespace WebCore {
@@ -515,14 +528,13 @@ bool VideoTextureCopierCV::initializeContextObjects()
 
     StringBuilder fragmentShaderSource;
 
-#if USE(OPENGL_ES)
+#if USE(OPENGL_ES) || USE(ANGLE)
     fragmentShaderSource.appendLiteral("precision mediump float;\n");
+#endif
+#if USE(OPENGL_ES) || (USE(ANGLE) && PLATFORM(IOS_FAMILY))
     fragmentShaderSource.appendLiteral("uniform sampler2D u_texture;\n");
-#elif USE(OPENGL)
+#elif USE(OPENGL) || (USE(ANGLE) && !PLATFORM(IOS_FAMILY))
     fragmentShaderSource.appendLiteral("uniform sampler2DRect u_texture;\n");
-#elif USE(ANGLE)
-    // FIXME: determine how to access rectangular textures via ANGLE.
-    ASSERT_NOT_REACHED();
 #else
 #error Unsupported configuration
 #endif
@@ -532,13 +544,10 @@ bool VideoTextureCopierCV::initializeContextObjects()
     fragmentShaderSource.appendLiteral("uniform int u_swapColorChannels;\n");
     fragmentShaderSource.appendLiteral("void main() {\n");
     fragmentShaderSource.appendLiteral("    vec2 texPos = vec2(v_texturePosition.x * u_textureDimensions.x, v_texturePosition.y * u_textureDimensions.y);\n");
-#if USE(OPENGL_ES)
+#if USE(OPENGL_ES) || (USE(ANGLE) && PLATFORM(IOS_FAMILY))
     fragmentShaderSource.appendLiteral("    vec4 color = texture2D(u_texture, texPos);\n");
-#elif USE(OPENGL)
+#elif USE(OPENGL) || (USE(ANGLE) && !PLATFORM(IOS_FAMILY))
     fragmentShaderSource.appendLiteral("    vec4 color = texture2DRect(u_texture, texPos);\n");
-#elif USE(ANGLE)
-    // FIXME: determine how to access rectangular textures via ANGLE.
-    ASSERT_NOT_REACHED();
 #else
 #error Unsupported configuration
 #endif
@@ -618,14 +627,12 @@ bool VideoTextureCopierCV::initializeUVContextObjects()
         "   if (u_flipY == 1) {\n"
         "       normalizedPosition.y = 1.0 - normalizedPosition.y;\n"
         "   }\n"
-#if USE(OPENGL_ES)
+#if USE(OPENGL_ES) || (USE(ANGLE) && PLATFORM(IOS_FAMILY))
         "   v_yTextureCoordinate = normalizedPosition;\n"
         "   v_uvTextureCoordinate = normalizedPosition;\n"
-#elif USE(OPENGL)
+#elif USE(OPENGL) || (USE(ANGLE) && !PLATFORM(IOS_FAMILY))
         "   v_yTextureCoordinate = normalizedPosition * u_yTextureSize;\n"
         "   v_uvTextureCoordinate = normalizedPosition * u_uvTextureSize;\n"
-#elif USE(ANGLE)
-        // FIXME: determine how to access rectangular textures via ANGLE.
 #else
 #error Unsupported configuration
 #endif
@@ -645,15 +652,15 @@ bool VideoTextureCopierCV::initializeUVContextObjects()
     }
 
     String fragmentShaderSource {
-#if USE(OPENGL_ES)
+#if USE(OPENGL_ES) || USE(ANGLE)
         "precision mediump float;\n"
+#endif
+#if USE(OPENGL_ES) || (USE(ANGLE) && PLATFORM(IOS_FAMILY))
         "#define SAMPLERTYPE sampler2D\n"
         "#define TEXTUREFUNC texture2D\n"
-#elif USE(OPENGL)
+#elif USE(OPENGL) || (USE(ANGLE) && !PLATFORM(IOS_FAMILY))
         "#define SAMPLERTYPE sampler2DRect\n"
         "#define TEXTUREFUNC texture2DRect\n"
-#elif USE(ANGLE)
-        // FIXME: determine how to access rectangular textures via ANGLE.
 #else
 #error Unsupported configuration
 #endif
@@ -722,8 +729,57 @@ bool VideoTextureCopierCV::initializeUVContextObjects()
     return true;
 }
 
+#if USE(ANGLE)
+void* VideoTextureCopierCV::attachIOSurfaceToTexture(GC3Denum target, GC3Denum internalFormat, GC3Dsizei width, GC3Dsizei height, GC3Denum type, IOSurfaceRef surface, GC3Duint plane)
+{
+    auto display = m_context->platformDisplay();
+    EGLint eglTextureTarget = 0;
+
+    if (target == GraphicsContext3D::TEXTURE_RECTANGLE_ARB)
+        eglTextureTarget = EGL_TEXTURE_RECTANGLE_ANGLE;
+    else if (target == GraphicsContext3D::TEXTURE_2D)
+        eglTextureTarget = EGL_TEXTURE_2D;
+    else {
+        LOG(WebGL, "Unknown texture target %d.", static_cast<int>(target));
+        return nullptr;
+    }
+
+    const EGLint surfaceAttributes[] = {
+        EGL_WIDTH, width,
+        EGL_HEIGHT, height,
+        EGL_IOSURFACE_PLANE_ANGLE, static_cast<EGLint>(plane),
+        EGL_TEXTURE_TARGET, static_cast<EGLint>(eglTextureTarget),
+        EGL_TEXTURE_INTERNAL_FORMAT_ANGLE, static_cast<EGLint>(internalFormat),
+        EGL_TEXTURE_FORMAT, EGL_TEXTURE_RGBA,
+        EGL_TEXTURE_TYPE_ANGLE, static_cast<EGLint>(type),
+        EGL_NONE, EGL_NONE
+    };
+    EGLSurface pbuffer = EGL_CreatePbufferFromClientBuffer(display, EGL_IOSURFACE_ANGLE, surface, m_context->platformConfig(), surfaceAttributes);
+    if (!pbuffer)
+        return nullptr;
+    if (!EGL_BindTexImage(display, pbuffer, EGL_BACK_BUFFER)) {
+        EGL_DestroySurface(display, pbuffer);
+        return nullptr;
+    }
+    return pbuffer;
+}
+
+void VideoTextureCopierCV::detachIOSurfaceFromTexture(void* handle)
+{
+    auto display = m_context->platformDisplay();
+    EGL_ReleaseTexImage(display, handle, EGL_BACK_BUFFER);
+    EGL_DestroySurface(display, handle);
+}
+#endif
+
 bool VideoTextureCopierCV::copyImageToPlatformTexture(CVPixelBufferRef image, size_t width, size_t height, Platform3DObject outputTexture, GC3Denum outputTarget, GC3Dint level, GC3Denum internalFormat, GC3Denum format, GC3Denum type, bool premultiplyAlpha, bool flipY)
 {
+    // CVOpenGLTextureCache seems to be disabled since the deprecation of
+    // OpenGL. To avoid porting unused code to the ANGLE code paths, remove it.
+#if USE(ANGLE)
+    UNUSED_PARAM(outputTarget);
+    UNUSED_PARAM(premultiplyAlpha);
+#else
     if (!m_textureCache) {
         m_textureCache = TextureCacheCV::create(m_context);
         if (!m_textureCache)
@@ -738,6 +794,7 @@ bool VideoTextureCopierCV::copyImageToPlatformTexture(CVPixelBufferRef image, si
 #endif
         return copyVideoTextureToPlatformTexture(texture.get(), width, height, outputTexture, outputTarget, level, internalFormat, format, type, premultiplyAlpha, flipY, swapColorChannels);
     }
+#endif // USE(ANGLE)
 
 #if HAVE(IOSURFACE)
     // FIXME: This currently only supports '420v' and '420f' pixel formats. Investigate supporting more pixel formats.
@@ -794,13 +851,10 @@ bool VideoTextureCopierCV::copyImageToPlatformTexture(CVPixelBufferRef image, si
     auto uvPlaneWidth = IOSurfaceGetWidthOfPlane(surface, 1);
     auto uvPlaneHeight = IOSurfaceGetHeightOfPlane(surface, 1);
 
-#if USE(OPENGL_ES)
-    GC3Denum videoTextureTarget = GraphicsContext3D::TEXTURE_2D;
-#elif USE(OPENGL)
-    GC3Denum videoTextureTarget = GL_TEXTURE_RECTANGLE_ARB;
-#elif USE(ANGLE)
-    // FIXME: determine how to access rectangular textures via ANGLE.
+#if USE(OPENGL_ES) || (USE(ANGLE) && PLATFORM(IOS_FAMILY))
     GC3Denum videoTextureTarget = GraphicsContext3D::TEXTURE_2D;
+#elif USE(OPENGL) || (USE(ANGLE) && !PLATFORM(IOS_FAMILY))
+    GC3Denum videoTextureTarget = GraphicsContext3D::TEXTURE_RECTANGLE_ARB;
 #else
 #error Unsupported configuration
 #endif
@@ -811,10 +865,18 @@ bool VideoTextureCopierCV::copyImageToPlatformTexture(CVPixelBufferRef image, si
     m_context->texParameteri(videoTextureTarget, GraphicsContext3D::TEXTURE_MIN_FILTER, GraphicsContext3D::LINEAR);
     m_context->texParameteri(videoTextureTarget, GraphicsContext3D::TEXTURE_WRAP_S, GraphicsContext3D::CLAMP_TO_EDGE);
     m_context->texParameteri(videoTextureTarget, GraphicsContext3D::TEXTURE_WRAP_T, GraphicsContext3D::CLAMP_TO_EDGE);
+#if USE(ANGLE)
+    auto uvHandle = attachIOSurfaceToTexture(videoTextureTarget, GraphicsContext3D::RG, uvPlaneWidth, uvPlaneHeight, GraphicsContext3D::UNSIGNED_BYTE, surface, 1);
+    if (!uvHandle) {
+        m_context->deleteTexture(uvTexture);
+        return false;
+    }
+#else
     if (!m_context->texImageIOSurface2D(videoTextureTarget, GraphicsContext3D::RG, uvPlaneWidth, uvPlaneHeight, GraphicsContext3D::RG, GraphicsContext3D::UNSIGNED_BYTE, surface, 1)) {
         m_context->deleteTexture(uvTexture);
         return false;
     }
+#endif // USE(ANGLE)
 
     auto yTexture = m_context->createTexture();
     m_context->activeTexture(GraphicsContext3D::TEXTURE0);
@@ -823,11 +885,20 @@ bool VideoTextureCopierCV::copyImageToPlatformTexture(CVPixelBufferRef image, si
     m_context->texParameteri(videoTextureTarget, GraphicsContext3D::TEXTURE_MIN_FILTER, GraphicsContext3D::LINEAR);
     m_context->texParameteri(videoTextureTarget, GraphicsContext3D::TEXTURE_WRAP_S, GraphicsContext3D::CLAMP_TO_EDGE);
     m_context->texParameteri(videoTextureTarget, GraphicsContext3D::TEXTURE_WRAP_T, GraphicsContext3D::CLAMP_TO_EDGE);
+#if USE(ANGLE)
+    auto yHandle = attachIOSurfaceToTexture(videoTextureTarget, GraphicsContext3D::RED, yPlaneWidth, yPlaneHeight, GraphicsContext3D::UNSIGNED_BYTE, surface, 0);
+    if (!yHandle) {
+        m_context->deleteTexture(yTexture);
+        m_context->deleteTexture(uvTexture);
+        return false;
+    }
+#else
     if (!m_context->texImageIOSurface2D(videoTextureTarget, GraphicsContext3D::LUMINANCE, yPlaneWidth, yPlaneHeight, GraphicsContext3D::LUMINANCE, GraphicsContext3D::UNSIGNED_BYTE, surface, 0)) {
         m_context->deleteTexture(yTexture);
         m_context->deleteTexture(uvTexture);
         return false;
     }
+#endif // USE(ANGLE)
 
     // Configure the drawing parameters.
     m_context->uniform1i(m_yTextureUniformLocation, 0);
@@ -844,7 +915,7 @@ bool VideoTextureCopierCV::copyImageToPlatformTexture(CVPixelBufferRef image, si
     // Do the actual drawing.
     m_context->drawArrays(GraphicsContext3D::TRIANGLES, 0, 6);
 
-#if USE(OPENGL_ES)
+#if USE(OPENGL_ES) || (USE(ANGLE) && PLATFORM(IOS_FAMILY))
     // flush() must be called here in order to re-synchronize the output texture's contents across the
     // two EAGL contexts.
     m_context->flush();
@@ -853,6 +924,10 @@ bool VideoTextureCopierCV::copyImageToPlatformTexture(CVPixelBufferRef image, si
     // Clean-up.
     m_context->deleteTexture(yTexture);
     m_context->deleteTexture(uvTexture);
+#if USE(ANGLE)
+    detachIOSurfaceFromTexture(yHandle);
+    detachIOSurfaceFromTexture(uvHandle);
+#endif
 
     m_lastSurface = surface;
     m_lastSurfaceSeed = newSurfaceSeed;
@@ -886,6 +961,10 @@ bool VideoTextureCopierCV::copyVideoTextureToPlatformTexture(TextureType inputVi
     videoTextureTarget = CVOpenGLTextureGetTarget(inputVideoTexture);
     CVOpenGLTextureGetCleanTexCoords(inputVideoTexture, lowerLeft, lowerRight, upperRight, upperLeft);
 #elif USE(ANGLE)
+    // CVOpenGLTextureCacheCreateTextureFromImage seems to always return
+    // kCVReturnPixelBufferNotOpenGLCompatible on desktop macOS now, so this
+    // entire code path seems to be unused. Assume the IOSurface path will be
+    // taken when using ANGLE.
     UNUSED_PARAM(lowerLeft);
     UNUSED_PARAM(lowerRight);
     UNUSED_PARAM(upperLeft);
index 9d392ea..36bda91 100644 (file)
@@ -70,9 +70,22 @@ private:
     }
 #endif
 
+#if USE(ANGLE)
+#if !HAVE(IOSURFACE)
+#error USE(ANGLE) requires HAVE(IOSURFACE)
+#endif // !HAVE(IOSURFACE)
+
+    // Returns a handle which, if non-null, must be released via the
+    // detach call below.
+    void* attachIOSurfaceToTexture(GC3Denum target, GC3Denum internalFormat, GC3Dsizei width, GC3Dsizei height, GC3Denum type, IOSurfaceRef, GC3Duint plane);
+    void detachIOSurfaceFromTexture(void* handle);
+#endif
+
     Ref<GraphicsContext3D> m_sharedContext;
     Ref<GraphicsContext3D> m_context;
+#if !USE(ANGLE)
     std::unique_ptr<TextureCacheCV> m_textureCache;
+#endif
     Platform3DObject m_framebuffer { 0 };
     Platform3DObject m_program { 0 };
     Platform3DObject m_vertexBuffer { 0 };