Enable GPU-GPU texture copy in texImage2D() for HTMLVideoElement if hardware accelera...
authorcommit-queue@webkit.org <commit-queue@webkit.org@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Wed, 13 Mar 2013 01:47:39 +0000 (01:47 +0000)
committercommit-queue@webkit.org <commit-queue@webkit.org@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Wed, 13 Mar 2013 01:47:39 +0000 (01:47 +0000)
https://bugs.webkit.org/show_bug.cgi?id=111126

Patch by Jun Jiang <jun.a.jiang@intel.com> on 2013-03-12
Reviewed by Kenneth Russell.

Source/WebCore:

In texImage2D() for HTMLVideoElement in WebGL, it is possible to do a GPU-GPU texture copy instead of CPU readback
and upload when videoFrame is in the form of textures. Each platform port needs to implement the interface
copyVideoTextureToPlatformTexture() defined in MediaPlayer.h to make it work.

Already covered by current tests.

* html/HTMLVideoElement.cpp:
(WebCore::HTMLVideoElement::copyVideoTextureToPlatformTexture):
* html/HTMLVideoElement.h:
(HTMLVideoElement):
* html/canvas/WebGLRenderingContext.cpp:
(WebCore::WebGLRenderingContext::videoFrameToImage): Move some security check to upper level.
(WebCore::WebGLRenderingContext::texImage2D): Add the fast GPU-GPU textures copy path for HTMLVideoElement and some security check.
(WebCore::WebGLRenderingContext::texSubImage2D): Add some security check.
* html/canvas/WebGLTexture.cpp:
(WebCore::WebGLTexture::isValid): Added to check if the texture is defined at the given level.
* html/canvas/WebGLTexture.h:
(WebGLTexture):
* platform/graphics/MediaPlayer.cpp:
(WebCore::MediaPlayer::copyVideoTextureToPlatformTexture): Added to do the GPU-GPU textures copy.
* platform/graphics/MediaPlayer.h:
(MediaPlayer):
* platform/graphics/MediaPlayerPrivate.h:
(WebCore::MediaPlayerPrivateInterface::copyVideoTextureToPlatformTexture):

Source/WebKit/chromium:

* public/WebMediaPlayer.h:
(WebKit::WebMediaPlayer::copyVideoTextureToPlatformTexture):
* src/WebMediaPlayerClientImpl.cpp:
(WebKit::WebMediaPlayerClientImpl::copyVideoTextureToPlatformTexture): do a GPU-GPU textures copy if possible.
* src/WebMediaPlayerClientImpl.h:
(WebMediaPlayerClientImpl):

git-svn-id: https://svn.webkit.org/repository/webkit/trunk@145669 268f45cc-cd09-0410-ab3c-d52691b4dbfc

13 files changed:
Source/WebCore/ChangeLog
Source/WebCore/html/HTMLVideoElement.cpp
Source/WebCore/html/HTMLVideoElement.h
Source/WebCore/html/canvas/WebGLRenderingContext.cpp
Source/WebCore/html/canvas/WebGLTexture.cpp
Source/WebCore/html/canvas/WebGLTexture.h
Source/WebCore/platform/graphics/MediaPlayer.cpp
Source/WebCore/platform/graphics/MediaPlayer.h
Source/WebCore/platform/graphics/MediaPlayerPrivate.h
Source/WebKit/chromium/ChangeLog
Source/WebKit/chromium/public/WebMediaPlayer.h
Source/WebKit/chromium/src/WebMediaPlayerClientImpl.cpp
Source/WebKit/chromium/src/WebMediaPlayerClientImpl.h

index 544064b..85d3b24 100644 (file)
@@ -1,3 +1,35 @@
+2013-03-12  Jun Jiang  <jun.a.jiang@intel.com>
+
+        Enable GPU-GPU texture copy in texImage2D() for HTMLVideoElement if hardware accelerated video decode is in use
+        https://bugs.webkit.org/show_bug.cgi?id=111126
+
+        Reviewed by Kenneth Russell.
+
+        In texImage2D() for HTMLVideoElement in WebGL, it is possible to do a GPU-GPU texture copy instead of CPU readback
+        and upload when videoFrame is in the form of textures. Each platform port needs to implement the interface
+        copyVideoTextureToPlatformTexture() defined in MediaPlayer.h to make it work.
+        Already covered by current tests.
+
+        * html/HTMLVideoElement.cpp:
+        (WebCore::HTMLVideoElement::copyVideoTextureToPlatformTexture):
+        * html/HTMLVideoElement.h:
+        (HTMLVideoElement):
+        * html/canvas/WebGLRenderingContext.cpp:
+        (WebCore::WebGLRenderingContext::videoFrameToImage): Move some security check to upper level.
+        (WebCore::WebGLRenderingContext::texImage2D): Add the fast GPU-GPU textures copy path for HTMLVideoElement and some security check.
+        (WebCore::WebGLRenderingContext::texSubImage2D): Add some security check.
+        * html/canvas/WebGLTexture.cpp:
+        (WebCore::WebGLTexture::isValid): Added to check if the texture is defined at the given level.
+        * html/canvas/WebGLTexture.h:
+        (WebGLTexture):
+        * platform/graphics/MediaPlayer.cpp:
+        (WebCore::MediaPlayer::copyVideoTextureToPlatformTexture): Added to do the GPU-GPU textures copy.
+        * platform/graphics/MediaPlayer.h:
+        (MediaPlayer):
+        * platform/graphics/MediaPlayerPrivate.h:
+        (WebCore::MediaPlayerPrivateInterface::copyVideoTextureToPlatformTexture):
+
 2013-03-12  Sheriff Bot  <webkit.review.bot@gmail.com>
 
         Unreviewed, rolling out r145280.
index cfe9571..a377b56 100644 (file)
@@ -237,6 +237,13 @@ void HTMLVideoElement::paintCurrentFrameInContext(GraphicsContext* context, cons
     player->paintCurrentFrameInContext(context, destRect);
 }
 
+bool HTMLVideoElement::copyVideoTextureToPlatformTexture(GraphicsContext3D* context, Platform3DObject texture, GC3Dint level, GC3Denum type, GC3Denum internalFormat, bool premultiplyAlpha, bool flipY)
+{
+    if (!player())
+        return false;
+    return player()->copyVideoTextureToPlatformTexture(context, texture, level, type, internalFormat, premultiplyAlpha, flipY);
+}
+
 bool HTMLVideoElement::hasAvailableVideoFrame() const
 {
     if (!player())
index cf4feac..0c3fd68 100644 (file)
@@ -63,6 +63,10 @@ public:
     // Used by canvas to gain raw pixel access
     void paintCurrentFrameInContext(GraphicsContext*, const IntRect&);
 
+    // Used by WebGL to do GPU-GPU textures copy if possible.
+    // See more details at MediaPlayer::copyVideoTextureToPlatformTexture() defined in Source/WebCore/platform/graphics/MediaPlayer.h.
+    bool copyVideoTextureToPlatformTexture(GraphicsContext3D*, Platform3DObject texture, GC3Dint level, GC3Denum type, GC3Denum internalFormat, bool premultiplyAlpha, bool flipY);
+
     bool shouldDisplayPosterImage() const { return displayMode() == Poster || displayMode() == PosterWaitingForVideo; }
 
 private:
index 5252856..820cf13 100644 (file)
@@ -3891,22 +3891,14 @@ void WebGLRenderingContext::texImage2D(GC3Denum target, GC3Dint level, GC3Denum
 }
 
 #if ENABLE(VIDEO)
-PassRefPtr<Image> WebGLRenderingContext::videoFrameToImage(HTMLVideoElement* video, BackingStoreCopy backingStoreCopy, ExceptionCode& ec)
+PassRefPtr<Image> WebGLRenderingContext::videoFrameToImage(HTMLVideoElement* video, BackingStoreCopy backingStoreCopy, ExceptionCode&)
 {
-    if (!video || !video->videoWidth() || !video->videoHeight()) {
-        synthesizeGLError(GraphicsContext3D::INVALID_VALUE, "texImage2D", "no video");
-        return 0;
-    }
     IntSize size(video->videoWidth(), video->videoHeight());
     ImageBuffer* buf = m_videoCache.imageBuffer(size);
     if (!buf) {
         synthesizeGLError(GraphicsContext3D::OUT_OF_MEMORY, "texImage2D", "out of memory");
         return 0;
     }
-    if (wouldTaintOrigin(video)) {
-        ec = SECURITY_ERR;
-        return 0;
-    }
     IntRect destRect(0, 0, size.width(), size.height());
     // FIXME: Turn this into a GPU-GPU texture copy instead of CPU readback.
     video->paintCurrentFrameInContext(buf->context(), destRect);
@@ -3919,6 +3911,37 @@ void WebGLRenderingContext::texImage2D(GC3Denum target, GC3Dint level, GC3Denum
     ec = 0;
     if (isContextLost())
         return;
+    if (!video || !video->videoWidth() || !video->videoHeight()) {
+        synthesizeGLError(GraphicsContext3D::INVALID_VALUE, "texImage2D", "no video");
+        return;
+    }
+    if (wouldTaintOrigin(video)) {
+        ec = SECURITY_ERR;
+        return;
+    }
+    if (!validateTexFuncFormatAndType("texImage2D", format, type, level))
+        return;
+    if (!validateSettableTexFormat("texImage2D", format))
+        return;
+
+    // Go through the fast path doing a GPU-GPU textures copy without a readback to system memory if possible.
+    // Otherwise, it will fall back to the normal SW path.
+    // FIXME: The current restrictions require that format shoud be RGB or RGBA,
+    // type should be UNSIGNED_BYTE and level should be 0. It may be lifted in the future.
+    WebGLTexture* texture = validateTextureBinding("texImage2D", target, true);
+    if (GraphicsContext3D::TEXTURE_2D == target && texture
+        && (format == GraphicsContext3D::RGB || format == GraphicsContext3D::RGBA)
+        && type == GraphicsContext3D::UNSIGNED_BYTE
+        && (texture->getType(target, level) == GraphicsContext3D::UNSIGNED_BYTE || !texture->isValid(target, level))
+        && !level) {
+        if (video->copyVideoTextureToPlatformTexture(m_context.get(), texture->object(), level, type, internalformat, m_unpackPremultiplyAlpha, m_unpackFlipY)) {
+            texture->setLevelInfo(target, level, internalformat, video->videoWidth(), video->videoHeight(), type);
+            cleanupAfterGraphicsCall(false);
+            return;
+        }
+    }
+
+    // Normal pure SW path.
     RefPtr<Image> image = videoFrameToImage(video, ImageBuffer::fastCopyImageMode(), ec);
     if (!image)
         return;
@@ -4170,6 +4193,20 @@ void WebGLRenderingContext::texSubImage2D(GC3Denum target, GC3Dint level, GC3Din
     ec = 0;
     if (isContextLost())
         return;
+    if (!video || !video->videoWidth() || !video->videoHeight()) {
+        synthesizeGLError(GraphicsContext3D::INVALID_VALUE, "texSubImage2D", "no video");
+        return;
+    }
+
+    if (wouldTaintOrigin(video)) {
+        ec = SECURITY_ERR;
+        return;
+    }
+    if (!validateTexFuncFormatAndType("texSubImage2D", format, type, level))
+        return;
+    if (!validateSettableTexFormat("texSubImage2D", format))
+        return;
+
     RefPtr<Image> image = videoFrameToImage(video, ImageBuffer::fastCopyImageMode(), ec);
     if (!image)
         return;
index 9ad58b4..634d074 100644 (file)
@@ -207,6 +207,14 @@ GC3Dsizei WebGLTexture::getHeight(GC3Denum target, GC3Dint level) const
     return info->height;
 }
 
+bool WebGLTexture::isValid(GC3Denum target, GC3Dint level) const
+{
+    const LevelInfo* info = getLevelInfo(target, level);
+    if (!info)
+        return 0;
+    return info->valid;
+}
+
 bool WebGLTexture::isNPOT(GC3Dsizei width, GC3Dsizei height)
 {
     ASSERT(width >= 0 && height >= 0);
index d1db951..a4c1507 100644 (file)
@@ -58,6 +58,7 @@ public:
     GC3Denum getType(GC3Denum target, GC3Dint level) const;
     GC3Dsizei getWidth(GC3Denum target, GC3Dint level) const;
     GC3Dsizei getHeight(GC3Denum target, GC3Dint level) const;
+    bool isValid(GC3Denum target, GC3Dint level) const;
 
     // Whether width/height is NotPowerOfTwo.
     static bool isNPOT(GC3Dsizei, GC3Dsizei);
index 69b4d7b..7c94b2b 100644 (file)
@@ -727,6 +727,11 @@ void MediaPlayer::paintCurrentFrameInContext(GraphicsContext* p, const IntRect&
     m_private->paintCurrentFrameInContext(p, r);
 }
 
+bool MediaPlayer::copyVideoTextureToPlatformTexture(GraphicsContext3D* context, Platform3DObject texture, GC3Dint level, GC3Denum type, GC3Denum internalFormat, bool premultiplyAlpha, bool flipY)
+{
+    return m_private->copyVideoTextureToPlatformTexture(context, texture, level, type, internalFormat, premultiplyAlpha, flipY);
+}
+
 MediaPlayer::SupportsType MediaPlayer::supportsType(const ContentType& contentType, const String& keySystem, const KURL& url, const MediaPlayerSupportsTypeClient* client)
 {
     String type = contentType.type().lower();
index 4ba4f27..994c8cd 100644 (file)
@@ -27,7 +27,7 @@
 #define MediaPlayer_h
 
 #if ENABLE(VIDEO)
-
+#include "GraphicsTypes3D.h"
 #if ENABLE(PLUGIN_PROXY_FOR_VIDEO)
 #include "MediaPlayerProxy.h"
 #endif
@@ -105,6 +105,7 @@ class CachedResourceLoader;
 class ContentType;
 class FrameView;
 class GraphicsContext;
+class GraphicsContext3D;
 class IntRect;
 class IntSize;
 class MediaPlayer;
@@ -333,6 +334,21 @@ public:
     void paint(GraphicsContext*, const IntRect&);
     void paintCurrentFrameInContext(GraphicsContext*, const IntRect&);
 
+    // copyVideoTextureToPlatformTexture() is used to do the GPU-GPU textures copy without a readback to system memory.
+    // The first five parameters denote the corresponding GraphicsContext, destination texture, requested level, requested type and the required internalFormat for destination texture.
+    // The last two parameters premultiplyAlpha and flipY denote whether addtional premultiplyAlpha and flip operation are required during the copy.
+    // It returns true on success and false on failure.
+
+    // In the GPU-GPU textures copy, the source texture(Video texture) should have valid target, internalFormat and size, etc.
+    // The destination texture may need to be resized to to the dimensions of the source texture or re-defined to the required internalFormat.
+    // The current restrictions require that format shoud be RGB or RGBA, type should be UNSIGNED_BYTE and level should be 0. It may be lifted in the future.
+
+    // Each platform port can have its own implementation on this function. The default implementation for it is a single "return false" in MediaPlayerPrivate.h.
+    // In chromium, the implementation is based on GL_CHROMIUM_copy_texture extension which is documented at
+    // http://src.chromium.org/viewvc/chrome/trunk/src/gpu/GLES2/extensions/CHROMIUM/CHROMIUM_copy_texture.txt and implemented at
+    // http://src.chromium.org/viewvc/chrome/trunk/src/gpu/command_buffer/service/gles2_cmd_copy_texture_chromium.cc via shaders.
+    bool copyVideoTextureToPlatformTexture(GraphicsContext3D*, Platform3DObject texture, GC3Dint level, GC3Denum type, GC3Denum internalFormat, bool premultiplyAlpha, bool flipY);
+
     enum NetworkState { Empty, Idle, Loading, Loaded, FormatError, NetworkError, DecodeError };
     NetworkState networkState();
 
index 710daa7..857b495 100644 (file)
@@ -107,6 +107,7 @@ public:
     virtual void paint(GraphicsContext*, const IntRect&) = 0;
 
     virtual void paintCurrentFrameInContext(GraphicsContext* c, const IntRect& r) { paint(c, r); }
+    virtual bool copyVideoTextureToPlatformTexture(GraphicsContext3D*, Platform3DObject, GC3Dint, GC3Denum, GC3Denum, bool, bool) { return false; }
 
     virtual void setPreload(MediaPlayer::Preload) { }
 
index de0011b..6e9c8eb 100644 (file)
@@ -1,3 +1,17 @@
+2013-03-12  Jun Jiang  <jun.a.jiang@intel.com>
+
+        Enable GPU-GPU texture copy in texImage2D() for HTMLVideoElement if hardware accelerated video decode is in use
+        https://bugs.webkit.org/show_bug.cgi?id=111126
+
+        Reviewed by Kenneth Russell.
+
+        * public/WebMediaPlayer.h:
+        (WebKit::WebMediaPlayer::copyVideoTextureToPlatformTexture):
+        * src/WebMediaPlayerClientImpl.cpp:
+        (WebKit::WebMediaPlayerClientImpl::copyVideoTextureToPlatformTexture): do a GPU-GPU textures copy if possible.
+        * src/WebMediaPlayerClientImpl.h:
+        (WebMediaPlayerClientImpl):
+
 2013-03-12  Adam Barth  <abarth@webkit.org>
 
         [Chromium] WebFrame event listener APIs are unused and can be removed
index f88b714..6c41019 100644 (file)
@@ -47,6 +47,7 @@ class WebString;
 class WebURL;
 struct WebRect;
 struct WebSize;
+class WebGraphicsContext3D;
 
 class WebMediaPlayer {
 public:
@@ -180,6 +181,9 @@ public:
     // method should no longer be referenced after the call is made.
     virtual void putCurrentFrame(WebVideoFrame*) { }
 
+    // Do a GPU-GPU textures copy if possible.
+    virtual bool copyVideoTextureToPlatformTexture(WebGraphicsContext3D*, unsigned texture, unsigned level, unsigned internalFormat, bool premultiplyAlpha, bool flipY) { return false; }
+
     virtual void setStreamTextureClient(WebStreamTextureClient*) { }
 
     virtual WebAudioSourceProvider* audioSourceProvider() { return 0; }
index e20fda3..75cb0f2 100644 (file)
@@ -12,6 +12,7 @@
 #include "AudioSourceProviderClient.h"
 #include "Frame.h"
 #include "GraphicsContext.h"
+#include "GraphicsContext3DPrivate.h"
 #include "GraphicsLayerChromium.h"
 #include "HTMLMediaElement.h"
 #include "IntSize.h"
@@ -765,6 +766,17 @@ void WebMediaPlayerClientImpl::paintCurrentFrameInContext(GraphicsContext* conte
     }
 }
 
+bool WebMediaPlayerClientImpl::copyVideoTextureToPlatformTexture(WebCore::GraphicsContext3D* context, Platform3DObject texture, GC3Dint level, GC3Denum type, GC3Denum internalFormat, bool premultiplyAlpha, bool flipY)
+{
+    if (!context || !m_webMediaPlayer)
+        return false;
+    Extensions3D* extensions = context->getExtensions();
+    if (!extensions || !extensions->supports("GL_CHROMIUM_copy_texture") || !extensions->supports("GL_CHROMIUM_flipy") || !context->makeContextCurrent())
+        return false;
+    WebGraphicsContext3D* webGraphicsContext3D = GraphicsContext3DPrivate::extractWebGraphicsContext3D(context);
+    return m_webMediaPlayer->copyVideoTextureToPlatformTexture(webGraphicsContext3D, texture, level, internalFormat, premultiplyAlpha, flipY);
+}
+
 void WebMediaPlayerClientImpl::setPreload(MediaPlayer::Preload preload)
 {
     m_preload = preload;
index 99d2e8b..578f8e4 100644 (file)
@@ -132,6 +132,7 @@ public:
     virtual void setSize(const WebCore::IntSize&);
     virtual void paint(WebCore::GraphicsContext*, const WebCore::IntRect&);
     virtual void paintCurrentFrameInContext(WebCore::GraphicsContext*, const WebCore::IntRect&);
+    virtual bool copyVideoTextureToPlatformTexture(WebCore::GraphicsContext3D*, Platform3DObject texture, GC3Dint level, GC3Denum type, GC3Denum internalFormat, bool premultiplyAlpha, bool flipY);
     virtual void setPreload(WebCore::MediaPlayer::Preload);
     virtual bool hasSingleSecurityOrigin() const;
     virtual bool didPassCORSAccessCheck() const;