Bug 860466: Properly deal with SharedTextureHostOGL. r=nical

There were a number of issues with the current implementation. It couldn't deal
with NULL mSharedHandles. Additionally it was binding things to the wrong
samplers. Finally it was not properly setting the texture transform (which
defaults to all 0). Finally in a debug build an assert would be hit upon client
shutdown due to it not clearing mDescriptor.

This patch fixes all of those issues which makes flash work again.

--HG--
extra : rebase_source : 733f242d2cd82c145ef6e3af9584f832af3c1f5d

gfx/layers/Effects.h

@@ -39,7 +39,6 @@ enum EffectTypes
   EFFECT_RGBX,
   EFFECT_BGRA,
   EFFECT_RGBA,
-  EFFECT_RGBA_EXTERNAL,
   EFFECT_YCBCR,
   EFFECT_COMPONENT_ALPHA,
   EFFECT_SOLID_COLOR,
@@ -176,23 +175,6 @@ struct EffectRGBA : public TexturedEffect
 #endif
 };
 
-struct EffectRGBAExternal : public TexturedEffect
-{
-  EffectRGBAExternal(TextureSource *aRGBATexture,
-                     const gfx::Matrix4x4 &aTextureTransform,
-                     bool aPremultiplied,
-                     gfx::Filter aFilter)
-    : TexturedEffect(EFFECT_RGBA_EXTERNAL, aRGBATexture, aPremultiplied, aFilter)
-    , mTextureTransform(aTextureTransform)
-  {}
-
-#ifdef MOZ_LAYERS_HAVE_LOG
-  virtual const char* Name() { return "EffectRGBAExternal"; }
-#endif
-
-  gfx::Matrix4x4 mTextureTransform;
-};
-
 struct EffectYCbCr : public TexturedEffect
 {
   EffectYCbCr(TextureSource *aSource, gfx::Filter aFilter)

gfx/layers/opengl/CompositorOGL.cpp

@@ -934,7 +934,6 @@ CompositorOGL::GetProgramTypeForEffect(Effect *aEffect) const
   case EFFECT_SOLID_COLOR:
     return gl::ColorLayerProgramType;
   case EFFECT_RGBA:
-  case EFFECT_RGBA_EXTERNAL:
   case EFFECT_RGBX:
   case EFFECT_BGRA:
   case EFFECT_BGRX:
@@ -1045,8 +1044,7 @@ CompositorOGL::DrawQuad(const Rect& aRect, const Rect& aClipRect,
   case EFFECT_BGRA:
   case EFFECT_BGRX:
   case EFFECT_RGBA:
-  case EFFECT_RGBX:
+  case EFFECT_RGBX: {
-  case EFFECT_RGBA_EXTERNAL: {
       TexturedEffect* texturedEffect =
           static_cast<TexturedEffect*>(aEffectChain.mPrimaryEffect.get());
       Rect textureCoords;
@@ -1058,10 +1056,8 @@ CompositorOGL::DrawQuad(const Rect& aRect, const Rect& aClipRect,
       }
 
       source->AsSourceOGL()->BindTexture(LOCAL_GL_TEXTURE0);
-      if (aEffectChain.mPrimaryEffect->mType == EFFECT_RGBA_EXTERNAL) {
+      if (programType == gl::RGBALayerExternalProgramType) {
-        EffectRGBAExternal* effectRGBAExternal =
+        program->SetTextureTransform(source->AsSourceOGL()->GetTextureTransform());
-          static_cast<EffectRGBAExternal*>(aEffectChain.mPrimaryEffect.get());
-        program->SetTextureTransform(effectRGBAExternal->mTextureTransform);
       }
 
       mGLContext->ApplyFilterToBoundTexture(source->AsSourceOGL()->GetTextureTarget(),

gfx/layers/opengl/TextureClientOGL.cpp

@@ -25,10 +25,9 @@ TextureClientSharedOGL::ReleaseResources()
     return;
   }
   MOZ_ASSERT(mDescriptor.type() == SurfaceDescriptor::TSharedTextureDescriptor);
-  SharedTextureDescriptor handle = mDescriptor.get_SharedTextureDescriptor();
+  mDescriptor = SurfaceDescriptor();
-  if (mGL && handle.handle()) {
+  // It's important our handle gets released! SharedTextureHostOGL will take
-    mGL->ReleaseSharedHandle(handle.shareType(), handle.handle());
+  // care of this for us though.
-  }
 }
 
 void

gfx/layers/opengl/TextureHostOGL.cpp

@@ -51,20 +51,19 @@ CreateTextureHostOGL(SurfaceDescriptorType aDescriptorType,
 }
 
 static void
-MakeTextureIfNeeded(gl::GLContext* gl, GLenum aTarget, GLuint& aTexture)
+MakeTextureIfNeeded(gl::GLContext* gl, GLuint& aTexture)
 {
   if (aTexture != 0)
     return;
 
   gl->fGenTextures(1, &aTexture);
 
-  gl->fActiveTexture(LOCAL_GL_TEXTURE0);
+  gl->fBindTexture(LOCAL_GL_TEXTURE_2D, aTexture);
-  gl->fBindTexture(aTarget, aTexture);
 
-  gl->fTexParameteri(aTarget, LOCAL_GL_TEXTURE_MIN_FILTER, LOCAL_GL_LINEAR);
+  gl->fTexParameteri(LOCAL_GL_TEXTURE_2D, LOCAL_GL_TEXTURE_MIN_FILTER, LOCAL_GL_LINEAR);
-  gl->fTexParameteri(aTarget, LOCAL_GL_TEXTURE_MAG_FILTER, LOCAL_GL_LINEAR);
+  gl->fTexParameteri(LOCAL_GL_TEXTURE_2D, LOCAL_GL_TEXTURE_MAG_FILTER, LOCAL_GL_LINEAR);
-  gl->fTexParameteri(aTarget, LOCAL_GL_TEXTURE_WRAP_S, LOCAL_GL_CLAMP_TO_EDGE);
+  gl->fTexParameteri(LOCAL_GL_TEXTURE_2D, LOCAL_GL_TEXTURE_WRAP_S, LOCAL_GL_CLAMP_TO_EDGE);
-  gl->fTexParameteri(aTarget, LOCAL_GL_TEXTURE_WRAP_T, LOCAL_GL_CLAMP_TO_EDGE);
+  gl->fTexParameteri(LOCAL_GL_TEXTURE_2D, LOCAL_GL_TEXTURE_WRAP_T, LOCAL_GL_CLAMP_TO_EDGE);
 }
 
 static gl::TextureImage::Flags
@@ -251,21 +250,27 @@ SharedTextureHostOGL::SwapTexturesImpl(const SurfaceDescriptor& aImage,
   if (texture.inverted()) {
     mFlags |= NeedsYFlip;
   }
+
+  if (mSharedHandle && mSharedHandle != newHandle) {
+    mGL->ReleaseSharedHandle(mShareType, mSharedHandle);
+  }
+
   mShareType = texture.shareType();
   mSharedHandle = newHandle;
 
   GLContext::SharedHandleDetails handleDetails;
-  if (mGL->GetSharedHandleDetails(mShareType, mSharedHandle, handleDetails)) {
+  if (mSharedHandle && mGL->GetSharedHandleDetails(mShareType, mSharedHandle, handleDetails)) {
     mTextureTarget = handleDetails.mTarget;
     mShaderProgram = handleDetails.mProgramType;
     mFormat = FormatFromShaderType(mShaderProgram);
+    mTextureTransform = handleDetails.mTextureTransform;
   }
 }
 
 bool
 SharedTextureHostOGL::Lock()
 {
-  MakeTextureIfNeeded(mGL, mTextureTarget, mTextureHandle);
+  MakeTextureIfNeeded(mGL, mTextureHandle);
 
   mGL->fActiveTexture(LOCAL_GL_TEXTURE0);
   mGL->fBindTexture(mTextureTarget, mTextureHandle);

gfx/layers/opengl/TextureHostOGL.h

@@ -48,8 +48,10 @@ public:
   virtual gl::ShaderProgramType GetShaderProgram() const {
     MOZ_NOT_REACHED("unhandled shader type");
   }
+  // TODO: Noone's implementing this anymore, should see if we need this.
   virtual GLenum GetTextureTarget() const { return LOCAL_GL_TEXTURE_2D; }
   virtual GLenum GetWrapMode() const = 0;// { return LOCAL_GL_CLAMP_TO_EDGE; } // default
+  virtual gfx3DMatrix GetTextureTransform() const { return gfx3DMatrix(); }
 };
 
 inline gl::ShaderProgramType
@@ -320,7 +322,7 @@ public:
 
   virtual TextureSourceOGL* AsSourceOGL() MOZ_OVERRIDE { return this; }
 
-  bool IsValid() const MOZ_OVERRIDE { return GetFormat() != gfx::FORMAT_UNKNOWN; }
+  bool IsValid() const MOZ_OVERRIDE { return !!mSharedHandle; }
 
   // override from TextureHost, we support both buffered
   // and unbuffered operation.
@@ -343,16 +345,11 @@ public:
     return mSize;
   }
 
-  virtual GLenum GetTextureTarget() const MOZ_OVERRIDE
-  {
-    return mTextureTarget;
-  }
-
   void BindTexture(GLenum activetex) MOZ_OVERRIDE
   {
     MOZ_ASSERT(mGL);
-    mGL->fActiveTexture(activetex);
+    // Lock already bound us!
-    mGL->fBindTexture(mTextureTarget, mTextureHandle);
+    MOZ_ASSERT(activetex == LOCAL_GL_TEXTURE0);
   }
   void ReleaseTexture() {}
   GLuint GetTextureID() { return mTextureHandle; }
@@ -363,6 +360,11 @@ public:
              gfxASurface::CONTENT_COLOR;
   }
 
+  virtual gfx3DMatrix GetTextureTransform() const MOZ_OVERRIDE
+  {
+    return mTextureTransform;
+  }
+
 #ifdef MOZ_LAYERS_HAVE_LOG
   virtual const char* Name() { return "SharedTextureHostOGL"; }
 #endif
@@ -378,6 +380,7 @@ protected:
   gl::SharedTextureHandle mSharedHandle;
   gl::ShaderProgramType mShaderProgram;
   gl::GLContext::SharedTextureShareType mShareType;
+  gfx3DMatrix mTextureTransform;
 };
 
 class SurfaceStreamHostOGL : public TextureHost