Bug 1049960 - WebGL generate invalid operation errors when copyTex[Sub]Image2D is called with a format that is not a subset of the FBO format. r=jgilbert

dom/canvas/WebGLContext.h

@@ -1051,6 +1051,7 @@ protected:
     bool ValidateGLSLCharacter(char16_t c);
     bool ValidateGLSLString(const nsAString& string, const char *info);
 
+    bool ValidateCopyTexImage(GLenum format, WebGLTexImageFunc func);
     bool ValidateTexImage(GLuint dims, GLenum target,
                           GLint level, GLint internalFormat,
                           GLint xoffset, GLint yoffset, GLint zoffset,

dom/canvas/WebGLContextGL.cpp

@@ -370,6 +370,12 @@ WebGLContext::CopyTexSubImage2D_base(GLenum target,
         return;
     }
 
+    if (!ValidateCopyTexImage(internalformat, func))
+        return;
+
+    if (!mBoundFramebuffer)
+        ClearBackbufferIfNeeded();
+
     MakeContextCurrent();
 
     WebGLTexture *tex = activeBoundTextureForTarget(target);
@@ -473,27 +479,11 @@ WebGLContext::CopyTexImage2D(GLenum target,
         return;
     }
 
-    if (mBoundFramebuffer) {
+    if (!ValidateCopyTexImage(format, func))
-        if (!mBoundFramebuffer->CheckAndInitializeAttachments())
+        return;
-            return ErrorInvalidFramebufferOperation("copyTexImage2D: incomplete framebuffer");
 
-        GLenum readPlaneBits = LOCAL_GL_COLOR_BUFFER_BIT;
+    if (!mBoundFramebuffer)
-        if (!mBoundFramebuffer->HasCompletePlanes(readPlaneBits)) {
+        ClearBackbufferIfNeeded();
-            return ErrorInvalidOperation("copyTexImage2D: Read source attachment doesn't have the"
-                                         " correct color/depth/stencil type.");
-        }
-    } else {
-      ClearBackbufferIfNeeded();
-    }
-
-    bool texFormatRequiresAlpha = format == LOCAL_GL_RGBA ||
-                                  format == LOCAL_GL_ALPHA ||
-                                  format == LOCAL_GL_LUMINANCE_ALPHA;
-    bool fboFormatHasAlpha = mBoundFramebuffer ? mBoundFramebuffer->ColorAttachment(0).HasAlpha()
-                                               : bool(gl->GetPixelFormat().alpha > 0);
-    if (texFormatRequiresAlpha && !fboFormatHasAlpha)
-        return ErrorInvalidOperation("copyTexImage2D: texture format requires an alpha channel "
-                                     "but the framebuffer doesn't have one");
 
     // check if the memory size of this texture may change with this call
     bool sizeMayChange = true;
@@ -580,33 +570,14 @@ WebGLContext::CopyTexSubImage2D(GLenum target,
     if (yoffset + height > texHeight || yoffset + height < 0)
       return ErrorInvalidValue("copyTexSubImage2D: yoffset+height is too large");
 
-    if (mBoundFramebuffer) {
+    if (!mBoundFramebuffer)
-        if (!mBoundFramebuffer->CheckAndInitializeAttachments())
-            return ErrorInvalidFramebufferOperation("copyTexSubImage2D: incomplete framebuffer");
-
-        GLenum readPlaneBits = LOCAL_GL_COLOR_BUFFER_BIT;
-        if (!mBoundFramebuffer->HasCompletePlanes(readPlaneBits)) {
-            return ErrorInvalidOperation("copyTexSubImage2D: Read source attachment doesn't have the"
-                                         " correct color/depth/stencil type.");
-        }
-    } else {
         ClearBackbufferIfNeeded();
-    }
-
-    GLenum webGLFormat = imageInfo.WebGLFormat();
-    bool texFormatRequiresAlpha = FormatHasAlpha(webGLFormat);
-    bool fboFormatHasAlpha = mBoundFramebuffer ? mBoundFramebuffer->ColorAttachment(0).HasAlpha()
-                                               : bool(gl->GetPixelFormat().alpha > 0);
-
-    if (texFormatRequiresAlpha && !fboFormatHasAlpha)
-        return ErrorInvalidOperation("copyTexSubImage2D: texture format requires an alpha channel "
-                                     "but the framebuffer doesn't have one");
 
     if (imageInfo.HasUninitializedImageData()) {
         tex->DoDeferredImageInitialization(target, level);
     }
 
-    return CopyTexSubImage2D_base(target, level, webGLFormat, xoffset, yoffset, x, y, width, height, true);
+    return CopyTexSubImage2D_base(target, level, imageInfo.WebGLFormat(), xoffset, yoffset, x, y, width, height, true);
 }
 
 

dom/canvas/WebGLContextUtils.cpp

@@ -22,6 +22,7 @@
 #include "WebGLProgram.h"
 #include "WebGLTexture.h"
 #include "WebGLVertexArray.h"
+#include "WebGLContextUtils.h"
 
 #include "mozilla/dom/ScriptSettings.h"
 
@@ -55,6 +56,47 @@ FormatHasAlpha(GLenum webGLFormat)
            webGLFormat == LOCAL_GL_SRGB_ALPHA;
 }
 
+GLComponents::GLComponents(GLenum format)
+{
+    mComponents = 0;
+
+    switch (format) {
+        case LOCAL_GL_RGBA:
+        case LOCAL_GL_RGBA4:
+        case LOCAL_GL_RGBA8:
+        case LOCAL_GL_RGB5_A1:
+        // Luminance + Alpha can be converted
+        // to and from RGBA
+        case LOCAL_GL_LUMINANCE_ALPHA:
+            mComponents |= Components::Alpha;
+        // Drops through
+        case LOCAL_GL_RGB:
+        case LOCAL_GL_RGB565:
+        // Luminance can be converted to and from RGB
+        case LOCAL_GL_LUMINANCE:
+            mComponents |= Components::Red | Components::Green | Components::Blue;
+            break;
+        case LOCAL_GL_ALPHA:
+            mComponents |= Components::Alpha;
+            break;
+        case LOCAL_GL_DEPTH_COMPONENT:
+            mComponents |= Components::Depth;
+            break;
+        case LOCAL_GL_DEPTH_STENCIL:
+            mComponents |= Components::Stencil;
+            break;
+        default:
+            MOZ_ASSERT(false, "Unhandled case - GLComponents");
+            break;
+    }
+}
+
+bool
+GLComponents::IsSubsetOf(const GLComponents& other) const
+{
+    return (mComponents | other.mComponents) == other.mComponents;
+}
+
 /**
  * Convert WebGL/ES format and type into GL format and GL internal
  * format valid for underlying driver.

dom/canvas/WebGLContextUtils.h

@@ -19,6 +19,30 @@ void DriverFormatsFromFormatAndType(gl::GLContext* gl, GLenum webGLFormat, GLenu
                                     GLenum* out_driverInternalFormat, GLenum* out_driverFormat);
 GLenum DriverTypeFromType(gl::GLContext* gl, GLenum webGLType);
 
+struct GLComponents
+{
+    unsigned char mComponents;
+
+    enum Components {
+        Red     = (1 << 0),
+        Green   = (1 << 1),
+        Blue    = (1 << 2),
+        Alpha   = (1 << 3),
+        Stencil = (1 << 4),
+        Depth   = (1 << 5),
+    };
+
+    GLComponents()
+        : mComponents(0)
+    { }
+
+    GLComponents(GLenum format);
+
+    // Returns true iff other has all (or more) of
+    // the components present in this GLComponents
+    bool IsSubsetOf(const GLComponents& other) const;
+};
+
 template <typename WebGLObjectType>
 JS::Value
 WebGLContext::WebGLObjectAsJSValue(JSContext *cx, const WebGLObjectType *object, ErrorResult& rv) const

dom/canvas/WebGLContextValidate.cpp

@@ -15,6 +15,7 @@
 #include "WebGLVertexArray.h"
 #include "GLContext.h"
 #include "CanvasUtils.h"
+#include "WebGLContextUtils.h"
 
 #include "mozilla/CheckedInt.h"
 #include "mozilla/Preferences.h"
@@ -1256,6 +1257,51 @@ WebGLContext::ValidateTexInputData(GLenum type, int jsArrayType, WebGLTexImageFu
     return validInput;
 }
 
+/**
+ * Checks specific for the CopyTex[Sub]Image2D functions.
+ * Verifies:
+ * - Framebuffer is complete and has valid read planes
+ * - Copy format is a subset of framebuffer format (i.e. all required components are available)
+ */
+bool
+WebGLContext::ValidateCopyTexImage(GLenum format, WebGLTexImageFunc func)
+{
+    MOZ_ASSERT(IsCopyFunc(func));
+
+    // Default framebuffer format
+    GLenum fboFormat = bool(gl->GetPixelFormat().alpha > 0) ? LOCAL_GL_RGBA : LOCAL_GL_RGB;
+
+    if (mBoundFramebuffer) {
+        if (!mBoundFramebuffer->CheckAndInitializeAttachments()) {
+            ErrorInvalidFramebufferOperation("%s: incomplete framebuffer", InfoFrom(func));
+            return false;
+        }
+
+        GLenum readPlaneBits = LOCAL_GL_COLOR_BUFFER_BIT;
+        if (!mBoundFramebuffer->HasCompletePlanes(readPlaneBits)) {
+            ErrorInvalidOperation("%s: Read source attachment doesn't have the"
+                                  " correct color/depth/stencil type.", InfoFrom(func));
+            return false;
+        }
+
+        // Get the correct format for the framebuffer, as it's not the default one
+        const WebGLFramebuffer::Attachment& color0 = mBoundFramebuffer->GetAttachment(LOCAL_GL_COLOR_ATTACHMENT0);
+        fboFormat = mBoundFramebuffer->GetFormatForAttachment(color0);
+    }
+
+    // Make sure the format of the framebuffer is a superset of
+    // the format requested by the CopyTex[Sub]Image2D functions.
+    const GLComponents formatComps = GLComponents(format);
+    const GLComponents fboComps = GLComponents(fboFormat);
+    if (!formatComps.IsSubsetOf(fboComps)) {
+        ErrorInvalidOperation("%s: format %s is not a subset of the current framebuffer format, which is %s.",
+                              InfoFrom(func), EnumName(format), EnumName(fboFormat));
+        return false;
+    }
+
+    return true;
+}
+
 /**
  * Test the gl(Copy|Compressed)?Tex[Sub]?Image[23]() parameters for errors.
  * Verifies each of the parameters against the WebGL standard and enabled extensions.

dom/canvas/WebGLFramebuffer.cpp

@@ -76,6 +76,26 @@ WebGLFramebuffer::Attachment::HasAlpha() const
     return FormatHasAlpha(format);
 }
 
+GLenum
+WebGLFramebuffer::GetFormatForAttachment(const WebGLFramebuffer::Attachment& attachment) const
+{
+    MOZ_ASSERT(attachment.IsDefined());
+    MOZ_ASSERT(attachment.Texture() || attachment.Renderbuffer());
+
+    if (attachment.Texture()) {
+        const WebGLTexture& tex = *attachment.Texture();
+        MOZ_ASSERT(tex.HasImageInfoAt(tex.Target(), 0));
+
+        const WebGLTexture::ImageInfo& imgInfo = tex.ImageInfoAt(tex.Target(), 0);
+        return imgInfo.WebGLFormat();
+    }
+
+    if (attachment.Renderbuffer())
+        return attachment.Renderbuffer()->InternalFormat();
+
+    return LOCAL_GL_NONE;
+}
+
 bool
 WebGLFramebuffer::Attachment::IsReadableFloat() const
 {

dom/canvas/WebGLFramebuffer.h

@@ -117,6 +117,7 @@ public:
     bool AllImageRectsMatch() const;
     GLenum PrecheckFramebufferStatus() const;
     GLenum CheckFramebufferStatus() const;
+    GLenum GetFormatForAttachment(const WebGLFramebuffer::Attachment& attachment) const;
 
     bool HasDepthStencilConflict() const {
         return int(mDepthAttachment.IsDefined()) +