Bug 766345 - Part 1 - Implement DEAA Antialiasing for transformed layers (v5 Patch). r=vladimir, r=djg

gfx/layers/Compositor.h

@@ -297,10 +297,24 @@ public:
    * drawn is specified by aEffectChain. aRect is the quad to draw, in user space.
    * aTransform transforms from user space to screen space. If texture coords are
    * required, these will be in the primary effect in the effect chain.
+   * aVisibleRect is used to determine which edges should be antialiased,
+   * without applying the effect to the inner edges of a tiled layer.
    */
   virtual void DrawQuad(const gfx::Rect& aRect, const gfx::Rect& aClipRect,
                         const EffectChain& aEffectChain,
-                        gfx::Float aOpacity, const gfx::Matrix4x4 &aTransform) = 0;
+                        gfx::Float aOpacity, const gfx::Matrix4x4& aTransform,
+                        const gfx::Rect& aVisibleRect) = 0;
+
+  /**
+   * Overload of DrawQuad, with aVisibleRect defaulted to the value of aRect.
+   * Use this when you are drawing a single quad that is not part of a tiled
+   * layer.
+   */
+  void DrawQuad(const gfx::Rect& aRect, const gfx::Rect& aClipRect,
+                        const EffectChain& aEffectChain,
+                        gfx::Float aOpacity, const gfx::Matrix4x4& aTransform) {
+      DrawQuad(aRect, aClipRect, aEffectChain, aOpacity, aTransform, aRect);
+  }
 
   /*
    * Clear aRect on current render target.

gfx/layers/basic/BasicCompositor.cpp

@@ -323,7 +323,8 @@ BasicCompositor::DrawQuad(const gfx::Rect& aRect,
                           const gfx::Rect& aClipRect,
                           const EffectChain &aEffectChain,
                           gfx::Float aOpacity,
-                          const gfx::Matrix4x4 &aTransform)
+                          const gfx::Matrix4x4& aTransform,
+                          const gfx::Rect& aVisibleRect)
 {
   RefPtr<DrawTarget> buffer = mRenderTarget->mDrawTarget;
 

gfx/layers/basic/BasicCompositor.h

@@ -84,7 +84,8 @@ public:
                         const gfx::Rect& aClipRect,
                         const EffectChain &aEffectChain,
                         gfx::Float aOpacity,
-                        const gfx::Matrix4x4 &aTransform) override;
+                        const gfx::Matrix4x4& aTransform,
+                        const gfx::Rect& aVisibleRect) override;
 
   virtual void ClearRect(const gfx::Rect& aRect) override;
 

gfx/layers/composite/TiledContentHost.cpp

@@ -435,7 +435,8 @@ TiledContentHost::RenderTile(TileHost& aTile,
                              const gfx::Rect& aClipRect,
                              const nsIntRegion& aScreenRegion,
                              const IntPoint& aTextureOffset,
-                             const nsIntSize& aTextureBounds)
+                             const nsIntSize& aTextureBounds,
+                             const gfx::Rect& aVisibleRect)
 {
   if (aTile.IsPlaceholderTile()) {
     // This shouldn't ever happen, but let's fail semi-gracefully. No need
@@ -490,7 +491,7 @@ TiledContentHost::RenderTile(TileHost& aTile,
                                   textureRect.y / aTextureBounds.height,
                                   textureRect.width / aTextureBounds.width,
                                   textureRect.height / aTextureBounds.height);
-    mCompositor->DrawQuad(graphicsRect, aClipRect, aEffectChain, aOpacity, aTransform);
+    mCompositor->DrawQuad(graphicsRect, aClipRect, aEffectChain, aOpacity, aTransform, aVisibleRect);
   }
   DiagnosticFlags flags = DiagnosticFlags::CONTENT | DiagnosticFlags::TILE;
   if (aTile.mTextureHostOnWhite) {
@@ -582,7 +583,9 @@ TiledContentHost::RenderLayerBuffer(TiledLayerBufferComposite& aLayerBuffer,
           gfx::IntSize tileSize = aLayerBuffer.GetTileSize();
           RenderTile(tileTexture, aBackgroundColor, aEffectChain, aOpacity, aTransform,
                      aFilter, aClipRect, tileDrawRegion, tileOffset,
-                     nsIntSize(tileSize.width, tileSize.height));
+                     nsIntSize(tileSize.width, tileSize.height),
+                     gfx::Rect(visibleRect.x, visibleRect.y,
+                               visibleRect.width, visibleRect.height));
           if (tileTexture.mTextureHostOnWhite) {
             componentAlphaDiagnostic = DiagnosticFlags::COMPONENT_ALPHA;
           }

gfx/layers/composite/TiledContentHost.h

@@ -283,7 +283,8 @@ private:
                   const gfx::Rect& aClipRect,
                   const nsIntRegion& aScreenRegion,
                   const gfx::IntPoint& aTextureOffset,
-                  const gfx::IntSize& aTextureBounds);
+                  const gfx::IntSize& aTextureBounds,
+                  const gfx::Rect& aVisibleRect);
 
   void EnsureTileStore() {}
 

gfx/layers/d3d11/CompositorD3D11.cpp

@@ -741,7 +741,8 @@ CompositorD3D11::DrawQuad(const gfx::Rect& aRect,
                           const gfx::Rect& aClipRect,
                           const EffectChain& aEffectChain,
                           gfx::Float aOpacity,
-                          const gfx::Matrix4x4& aTransform)
+                          const gfx::Matrix4x4& aTransform,
+                          const gfx::Rect& aVisibleRect)
 {
   if (mCurrentClip.IsEmpty()) {
     return;

gfx/layers/d3d11/CompositorD3D11.h

@@ -94,7 +94,8 @@ public:
                         const gfx::Rect &aClipRect,
                         const EffectChain &aEffectChain,
                         gfx::Float aOpacity,
-                        const gfx::Matrix4x4 &aTransform) override;
+                        const gfx::Matrix4x4& aTransform,
+                        const gfx::Rect& aVisibleRect) override;
 
   /* Helper for when the primary effect is VR_DISTORTION */
   void DrawVRDistortion(const gfx::Rect &aRect,

gfx/layers/d3d9/CompositorD3D9.cpp

@@ -241,7 +241,8 @@ CompositorD3D9::DrawQuad(const gfx::Rect &aRect,
                          const gfx::Rect &aClipRect,
                          const EffectChain &aEffectChain,
                          gfx::Float aOpacity,
-                         const gfx::Matrix4x4 &aTransform)
+                         const gfx::Matrix4x4& aTransform,
+                         const gfx::Rect& aVisibleRect)
 {
   if (!mDeviceManager) {
     return;

gfx/layers/d3d9/CompositorD3D9.h

@@ -57,7 +57,8 @@ public:
                         const gfx::Rect &aClipRect,
                         const EffectChain &aEffectChain,
                         gfx::Float aOpacity,
-                        const gfx::Matrix4x4 &aTransform) override;
+                        const gfx::Matrix4x4& aTransform,
+                        const gfx::Rect& aVisibleRect) override;
 
   virtual void BeginFrame(const nsIntRegion& aInvalidRegion,
                           const gfx::Rect *aClipRectIn,

gfx/layers/opengl/CompositorOGL.cpp

@@ -87,7 +87,7 @@ CompositorOGL::CompositorOGL(nsIWidget *aWidget, int aSurfaceWidth,
   , mUseExternalSurfaceSize(aUseExternalSurfaceSize)
   , mFrameInProgress(false)
   , mDestroyed(false)
-  , mHeight(0)
+  , mViewportSize(0, 0)
   , mCurrentProgram(nullptr)
 {
   MOZ_COUNT_CTOR(CompositorOGL);
@@ -441,7 +441,7 @@ CompositorOGL::PrepareViewport(const gfx::IntSize& aSize)
   // Set the viewport correctly.
   mGLContext->fViewport(0, 0, aSize.width, aSize.height);
 
-  mHeight = aSize.height;
+  mViewportSize = aSize;
 
   // We flip the view matrix around so that everything is right-side up; we're
   // drawing directly into the window's back buffer, so this keeps things
@@ -575,7 +575,7 @@ void
 CompositorOGL::ClearRect(const gfx::Rect& aRect)
 {
   // Map aRect to OGL coordinates, origin:bottom-left
-  GLint y = mHeight - (aRect.y + aRect.height);
+  GLint y = mViewportSize.height - (aRect.y + aRect.height);
 
   ScopedGLState scopedScissorTestState(mGLContext, LOCAL_GL_SCISSOR_TEST, true);
   ScopedScissorRect autoScissorRect(mGLContext, aRect.x, y, aRect.width, aRect.height);
@@ -770,7 +770,8 @@ ShaderConfigOGL
 CompositorOGL::GetShaderConfigFor(Effect *aEffect,
                                   MaskType aMask,
                                   gfx::CompositionOp aOp,
-                                  bool aColorMatrix) const
+                                  bool aColorMatrix,
+                                  bool aDEAAEnabled) const
 {
   ShaderConfigOGL config;
 
@@ -821,6 +822,7 @@ CompositorOGL::GetShaderConfigFor(Effect *aEffect,
   config.SetColorMatrix(aColorMatrix);
   config.SetMask2D(aMask == MaskType::Mask2d);
   config.SetMask3D(aMask == MaskType::Mask3d);
+  config.SetDEAA(aDEAAEnabled);
   return config;
 }
 
@@ -902,12 +904,41 @@ static bool SetBlendMode(GLContext* aGL, gfx::CompositionOp aBlendMode, bool aIs
   return true;
 }
 
+gfx::Point3D
+CompositorOGL::GetLineCoefficients(const gfx::Point3D& aPoint1,
+                                   const gfx::Point3D& aPoint2)
+{
+  // Return standard coefficients for a line between aPoint1 and aPoint2
+  // for standard line equation:
+  //
+  // Ax + By + C = 0
+  //
+  // A = (p1.y – p2.y)
+  // B = (p2.x – p1.x)
+  // C = (p1.x * p2.y) – (p2.x * p1.y)
+
+  gfx::Point3D coeffecients;
+  coeffecients.x = aPoint1.y - aPoint2.y;
+  coeffecients.y = aPoint2.x - aPoint1.x;
+  coeffecients.z = aPoint1.x * aPoint2.y - aPoint2.x * aPoint1.y;
+
+  coeffecients *= 1.0f / sqrtf(coeffecients.x * coeffecients.x +
+                               coeffecients.y * coeffecients.y);
+
+  // Offset outwards by 0.5 pixel as the edge is considered to be 1 pixel
+  // wide and included within the interior of the polygon
+  coeffecients.z += 0.5f;
+
+  return coeffecients;
+}
+
 void
 CompositorOGL::DrawQuad(const Rect& aRect,
                         const Rect& aClipRect,
                         const EffectChain &aEffectChain,
                         Float aOpacity,
-                        const gfx::Matrix4x4 &aTransform)
+                        const gfx::Matrix4x4& aTransform,
+                        const gfx::Rect& aVisibleRect)
 {
   PROFILER_LABEL("CompositorOGL", "DrawQuad",
     js::ProfileEntry::Category::GRAPHICS);
@@ -1000,8 +1031,15 @@ CompositorOGL::DrawQuad(const Rect& aRect,
     blendMode = blendEffect->mBlendMode;
   }
 
+  // Only apply DEAA to quads that have been transformed such that aliasing
+  // could be visible
+  bool bEnableAA = gfxPrefs::LayersDEAAEnabled() &&
+                   !aTransform.Is2DIntegerTranslation();
+
   bool colorMatrix = aEffectChain.mSecondaryEffects[EffectTypes::COLOR_MATRIX];
-  ShaderConfigOGL config = GetShaderConfigFor(aEffectChain.mPrimaryEffect, maskType, blendMode, colorMatrix);
+  ShaderConfigOGL config = GetShaderConfigFor(aEffectChain.mPrimaryEffect,
+                                              maskType, blendMode, colorMatrix,
+                                              bEnableAA);
   config.SetOpacity(aOpacity != 1.f);
   ShaderProgramOGL *program = GetShaderProgramFor(config);
   ActivateProgram(program);
@@ -1028,6 +1066,74 @@ CompositorOGL::DrawQuad(const Rect& aRect,
     program->SetTexCoordMultiplier(source->GetSize().width, source->GetSize().height);
   }
 
+  // XXX kip - These calculations could be performed once per layer rather than
+  //           for every tile.  This might belong in Compositor.cpp once DEAA
+  //           is implemented for DirectX.
+  if (bEnableAA) {
+    // Calculate the transformed vertices of aVisibleRect in screen space
+    // pixels, mirroring the calculations in the vertex shader
+    Point4D quadVerts[4];
+    quadVerts[0] = Point4D(aVisibleRect.x, aVisibleRect.y, 0.0, 1.0);
+    quadVerts[1] = Point4D(aVisibleRect.x + aVisibleRect.width, aVisibleRect.y, 0.0, 1.0);
+    quadVerts[2] = Point4D(aVisibleRect.x + aVisibleRect.width, aVisibleRect.y + aVisibleRect.height, 0.0, 1.0);
+    quadVerts[3] = Point4D(aVisibleRect.x, aVisibleRect.y + aVisibleRect.height, 0.0, 1.0);
+    for (int i = 0; i < 4; i++) {
+      quadVerts[i] = aTransform * quadVerts[i];
+      quadVerts[i] -= Point4D(offset.x, offset.y, 0.0f, 0.0f) * quadVerts[i].w;
+      quadVerts[i] /= quadVerts[i].w;
+      quadVerts[i] = mProjMatrix * quadVerts[i];
+      quadVerts[i].x = (quadVerts[i].x * 0.5f + 0.5f) * mViewportSize.width;
+      quadVerts[i].y = (quadVerts[i].y * 0.5f + 0.5f) * mViewportSize.height;
+    }
+
+    // Calculate the line coefficients used by the DEAA shader to determine the
+    // sub-pixel coverage of the edge pixels
+    Point3D coefficients[4];
+    // Use shoelace formula on first triangle in quad to determine if winding
+    // order is reversed
+    float winding = (quadVerts[1].x - quadVerts[0].x) * (quadVerts[1].y + quadVerts[0].y) +
+                    (quadVerts[2].x - quadVerts[1].x) * (quadVerts[2].y + quadVerts[1].y) +
+                    (quadVerts[0].x - quadVerts[2].x) * (quadVerts[0].y + quadVerts[2].y);
+
+    if (winding >= 0) {
+      // This quad is front-facing
+      coefficients[0] = GetLineCoefficients(
+        Point3D(quadVerts[3].x, quadVerts[3].y, 0.0f),
+        Point3D(quadVerts[2].x, quadVerts[2].y, 0.0f));
+      coefficients[1] = GetLineCoefficients(
+        Point3D(quadVerts[2].x, quadVerts[2].y, 0.0f),
+        Point3D(quadVerts[1].x, quadVerts[1].y, 0.0f));
+      coefficients[2] = GetLineCoefficients(
+        Point3D(quadVerts[1].x, quadVerts[1].y, 0.0f),
+        Point3D(quadVerts[0].x, quadVerts[0].y, 0.0f));
+      coefficients[3] = GetLineCoefficients(
+        Point3D(quadVerts[0].x, quadVerts[0].y, 0.0f),
+        Point3D(quadVerts[3].x, quadVerts[3].y, 0.0f));
+    } else {
+      // This quad is rear-facing
+      coefficients[0] = GetLineCoefficients(
+        Point3D(quadVerts[2].x, quadVerts[2].y, 0.0f),
+        Point3D(quadVerts[3].x, quadVerts[3].y, 0.0f));
+      coefficients[1] = GetLineCoefficients(
+        Point3D(quadVerts[1].x, quadVerts[1].y, 0.0f),
+        Point3D(quadVerts[2].x, quadVerts[2].y, 0.0f));
+      coefficients[2] = GetLineCoefficients(
+        Point3D(quadVerts[0].x, quadVerts[0].y, 0.0f),
+        Point3D(quadVerts[1].x, quadVerts[1].y, 0.0f));
+      coefficients[3] = GetLineCoefficients(
+        Point3D(quadVerts[3].x, quadVerts[3].y, 0.0f),
+        Point3D(quadVerts[0].x, quadVerts[0].y, 0.0f));
+    }
+
+    // Set uniforms required by DEAA shader
+    Matrix4x4 transformInverted = aTransform;
+    transformInverted.Invert();
+    program->SetLayerTransformInverse(transformInverted);
+    program->SetDEAAEdges(coefficients);
+    program->SetVisibleCenter(aVisibleRect.Center());
+    program->SetViewportSize(mViewportSize);
+  }
+
   bool didSetBlendMode = false;
 
   switch (aEffectChain.mPrimaryEffect->mType) {
@@ -1126,9 +1232,7 @@ CompositorOGL::DrawQuad(const Rect& aRect,
       program->SetTextureUnit(0);
 
       if (maskType != MaskType::MaskNone) {
-        sourceMask->BindTexture(LOCAL_GL_TEXTURE1, gfx::Filter::LINEAR);
+        BindMaskForProgram(program, sourceMask, LOCAL_GL_TEXTURE1, maskQuadTransform);
-        program->SetMaskTextureUnit(1);
-        program->SetMaskLayerTransform(maskQuadTransform);
       }
 
       if (config.mFeatures & ENABLE_TEXTURE_RECT) {

gfx/layers/opengl/CompositorOGL.h

@@ -229,7 +229,8 @@ public:
                         const gfx::Rect& aClipRect,
                         const EffectChain &aEffectChain,
                         gfx::Float aOpacity,
-                        const gfx::Matrix4x4 &aTransform) override;
+                        const gfx::Matrix4x4& aTransform,
+                        const gfx::Rect& aVisibleRect) override;
 
   virtual void EndFrame() override;
   virtual void SetDispAcquireFence(Layer* aLayer) override;
@@ -335,7 +336,7 @@ private:
   gfx::Matrix4x4 mProjMatrix;
 
   /** The size of the surface we are rendering to */
-  nsIntSize mSurfaceSize;
+  gfx::IntSize mSurfaceSize;
 
   ScreenPoint mRenderOffset;
 
@@ -387,7 +388,8 @@ private:
   ShaderConfigOGL GetShaderConfigFor(Effect *aEffect,
                                      MaskType aMask = MaskType::MaskNone,
                                      gfx::CompositionOp aOp = gfx::CompositionOp::OP_OVER,
-                                     bool aColorMatrix = false) const;
+                                     bool aColorMatrix = false,
+                                     bool aDEAAEnabled = false) const;
   ShaderProgramOGL* GetShaderProgramFor(const ShaderConfigOGL &aConfig);
 
   /**
@@ -418,7 +420,8 @@ private:
                                       const gfx::Rect& aRect,
                                       const gfx::Rect& aTexCoordRect,
                                       TextureSource *aTexture);
-
+  gfx::Point3D GetLineCoefficients(const gfx::Point3D& aPoint1,
+                                   const gfx::Point3D& aPoint2);
   void ActivateProgram(ShaderProgramOGL *aProg);
   void CleanupResources();
 
@@ -437,7 +440,7 @@ private:
    * y-axis pointing downwards, for good reason as Web pages are typically
    * scrolled downwards. So, some flipping has to take place; FlippedY does it.
    */
-  GLint FlipY(GLint y) const { return mHeight - y; }
+  GLint FlipY(GLint y) const { return mViewportSize.height - y; }
 
   RefPtr<CompositorTexturePoolOGL> mTexturePool;
 
@@ -446,10 +449,10 @@ private:
   bool mDestroyed;
 
   /**
-   * Height of the OpenGL context's primary framebuffer in pixels. Used by
+   * Size of the OpenGL context's primary framebuffer in pixels. Used by
-   * FlipY for the y-flipping calculation.
+   * FlipY for the y-flipping calculation and by the DEAA shader.
    */
-  GLint mHeight;
+  gfx::IntSize mViewportSize;
 
   FenceHandle mReleaseFenceHandle;
   ShaderProgramOGL *mCurrentProgram;

gfx/layers/opengl/OGLShaderProgram.cpp

@@ -30,6 +30,7 @@ AddUniforms(ProgramProfileOGL& aProfile)
     // This needs to be kept in sync with the KnownUniformName enum
     static const char *sKnownUniformNames[] = {
         "uLayerTransform",
+        "uLayerTransformInverse",
         "uMaskTransform",
         "uLayerRects",
         "uMatrixProj",
@@ -53,6 +54,9 @@ AddUniforms(ProgramProfileOGL& aProfile)
         "uBlurOffset",
         "uBlurAlpha",
         "uBlurGaussianKernel",
+        "uSSEdges",
+        "uViewportSize",
+        "uVisibleCenter",
         nullptr
     };
 
@@ -142,6 +146,12 @@ ShaderConfigOGL::SetPremultiply(bool aEnabled)
   SetFeature(ENABLE_PREMULTIPLY, aEnabled);
 }
 
+void
+ShaderConfigOGL::SetDEAA(bool aEnabled)
+{
+  SetFeature(ENABLE_DEAA, aEnabled);
+}
+
 /* static */ ProgramProfileOGL
 ProgramProfileOGL::GetProfileFor(ShaderConfigOGL aConfig)
 {
@@ -153,8 +163,13 @@ ProgramProfileOGL::GetProfileFor(ShaderConfigOGL aConfig)
   vs << "uniform mat4 uMatrixProj;" << endl;
   vs << "uniform vec4 uLayerRects[4];" << endl;
   vs << "uniform mat4 uLayerTransform;" << endl;
-  vs << "uniform vec4 uRenderTargetOffset;" << endl;
+  if (aConfig.mFeatures & ENABLE_DEAA) {
-
+    vs << "uniform mat4 uLayerTransformInverse;" << endl;
+    vs << "uniform vec3 uSSEdges[4];" << endl;
+    vs << "uniform vec2 uVisibleCenter;" << endl;
+    vs << "uniform vec2 uViewportSize;" << endl;
+  }
+  vs << "uniform vec2 uRenderTargetOffset;" << endl;
   vs << "attribute vec4 aCoord;" << endl;
 
   if (!(aConfig.mFeatures & ENABLE_RENDER_COLOR)) {
@@ -174,27 +189,78 @@ ProgramProfileOGL::GetProfileFor(ShaderConfigOGL aConfig)
   vs << "  vec4 layerRect = uLayerRects[vertexID];" << endl;
   vs << "  vec4 finalPosition = vec4(aCoord.xy * layerRect.zw + layerRect.xy, 0.0, 1.0);" << endl;
   vs << "  finalPosition = uLayerTransform * finalPosition;" << endl;
-  vs << "  finalPosition.xyz /= finalPosition.w;" << endl;
 
-  if (aConfig.mFeatures & ENABLE_MASK_3D) {
+  if (aConfig.mFeatures & ENABLE_DEAA) {
-    vs << "  vMaskCoord.xy = (uMaskTransform * vec4(finalPosition.xyz, 1.0)).xy;" << endl;
+    // XXX kip - The DEAA shader could be made simpler if we switch to
-    // correct for perspective correct interpolation, see comment in D3D10 shader
+    //           using dynamic vertex buffers instead of sending everything
-    vs << "  vMaskCoord.z = 1.0;" << endl;
+    //           in through uniforms.  This would enable passing information
-    vs << "  vMaskCoord *= finalPosition.w;" << endl;
+    //           about how to dilate each vertex explicitly and eliminate the
-  } else if (aConfig.mFeatures & ENABLE_MASK_2D) {
+    //           need to extrapolate this with the sub-pixel coverage
-    vs << "  vMaskCoord.xy = (uMaskTransform * finalPosition).xy;" << endl;
+    //           calculation in the vertex shader.
-  }
 
-  vs << "  finalPosition = finalPosition - uRenderTargetOffset;" << endl;
+    // Calculate the screen space position of this vertex, in screen pixels
-  vs << "  finalPosition.xyz *= finalPosition.w;" << endl;
+    vs << "  vec4 ssPos = finalPosition;" << endl;
-  vs << "  finalPosition = uMatrixProj * finalPosition;" << endl;
+    vs << "  ssPos.xy -= uRenderTargetOffset * finalPosition.w;" << endl;
+    vs << "  ssPos = uMatrixProj * ssPos;" << endl;
+    vs << "  ssPos.xy = ((ssPos.xy/ssPos.w)*0.5+0.5)*uViewportSize;" << endl;
 
-  if (!(aConfig.mFeatures & ENABLE_RENDER_COLOR)) {
+    if (aConfig.mFeatures & ENABLE_MASK_2D ||
+        aConfig.mFeatures & ENABLE_MASK_3D ||
+        !(aConfig.mFeatures & ENABLE_RENDER_COLOR)) {
+      vs << "  vec4 coordAdjusted;" << endl;
+      vs << "  coordAdjusted.xy = aCoord.xy;" << endl;
+    }
+
+    // It is necessary to dilate edges away from uVisibleCenter to ensure that
+    // fragments with less than 50% sub-pixel coverage will be shaded.
+    // This offset is applied when the sub-pixel coverage of the vertex is
+    // less than 100%.  Expanding by 0.5 pixels in screen space is sufficient
+    // to include these pixels.
+    vs << "  if (dot(uSSEdges[0], vec3(ssPos.xy, 1.0)) < 1.5 ||" << endl;
+    vs << "      dot(uSSEdges[1], vec3(ssPos.xy, 1.0)) < 1.5 ||" << endl;
+    vs << "      dot(uSSEdges[2], vec3(ssPos.xy, 1.0)) < 1.5 ||" << endl;
+    vs << "      dot(uSSEdges[3], vec3(ssPos.xy, 1.0)) < 1.5) {" << endl;
+    // If the shader reaches this branch, then this vertex is on the edge of
+    // the layer's visible rect and should be dilated away from the center of
+    // the visible rect.  We don't want to hit this for inner facing
+    // edges between tiles, as the pixels may be covered twice without clipping
+    // against uSSEdges.  If all edges were dilated, it would result in
+    // artifacts visible within semi-transparent layers with multiple tiles.
+    vs << "    vec4 visibleCenter = uLayerTransform * vec4(uVisibleCenter, 0.0, 1.0);" << endl;
+    vs << "    vec2 dilateDir = finalPosition.xy / finalPosition.w - visibleCenter.xy / visibleCenter.w;" << endl;
+    vs << "    vec2 offset = sign(dilateDir) * 0.5;" << endl;
+    vs << "    finalPosition.xy += offset * finalPosition.w;" << endl;
+    if (!(aConfig.mFeatures & ENABLE_RENDER_COLOR)) {
+      // We must adjust the texture coordinates to compensate for the dilation
+      vs << "    coordAdjusted = uLayerTransformInverse * finalPosition;" << endl;
+      vs << "    coordAdjusted /= coordAdjusted.w;" << endl;
+      vs << "    coordAdjusted.xy -= layerRect.xy;" << endl;
+      vs << "    coordAdjusted.xy /= layerRect.zw;" << endl;
+    }
+    vs << "  }" << endl;
+
+    if (!(aConfig.mFeatures & ENABLE_RENDER_COLOR)) {
+      vs << "  vec4 textureRect = uTextureRects[vertexID];" << endl;
+      vs << "  vec2 texCoord = coordAdjusted.xy * textureRect.zw + textureRect.xy;" << endl;
+      vs << "  vTexCoord = (uTextureTransform * vec4(texCoord, 0.0, 1.0)).xy;" << endl;
+    }
+
+  } else if (!(aConfig.mFeatures & ENABLE_RENDER_COLOR)) {
     vs << "  vec4 textureRect = uTextureRects[vertexID];" << endl;
     vs << "  vec2 texCoord = aCoord.xy * textureRect.zw + textureRect.xy;" << endl;
     vs << "  vTexCoord = (uTextureTransform * vec4(texCoord, 0.0, 1.0)).xy;" << endl;
   }
-
+  if (aConfig.mFeatures & ENABLE_MASK_2D ||
+      aConfig.mFeatures & ENABLE_MASK_3D) {
+    vs << "  vMaskCoord.xy = (uMaskTransform * (finalPosition / finalPosition.w)).xy;" << endl;
+    if (aConfig.mFeatures & ENABLE_MASK_3D) {
+      // correct for perspective correct interpolation, see comment in D3D10 shader
+      vs << "  vMaskCoord.z = 1.0;" << endl;
+      vs << "  vMaskCoord *= finalPosition.w;" << endl;
+    }
+  }
+  vs << "  finalPosition.xy -= uRenderTargetOffset * finalPosition.w;" << endl;
+  vs << "  finalPosition = uMatrixProj * finalPosition;" << endl;
   vs << "  gl_Position = finalPosition;" << endl;
   vs << "}" << endl;
 
@@ -261,6 +327,10 @@ ProgramProfileOGL::GetProfileFor(ShaderConfigOGL aConfig)
     fs << "uniform sampler2D uMaskTexture;" << endl;
   }
 
+  if (aConfig.mFeatures & ENABLE_DEAA) {
+    fs << "uniform vec3 uSSEdges[4];" << endl;
+  }
+
   if (!(aConfig.mFeatures & ENABLE_RENDER_COLOR)) {
     fs << "vec4 sample(vec2 coord) {" << endl;
     fs << "  vec4 color;" << endl;
@@ -353,6 +423,16 @@ For [0,1] instead of [0,255], and to 5 places:
       fs << " color.rgb *= color.a;" << endl;
     }
   }
+  if (aConfig.mFeatures & ENABLE_DEAA) {
+    // Calculate the sub-pixel coverage of the pixel and modulate its opacity
+    // by that amount to perform DEAA.
+    fs << "  vec3 ssPos = vec3(gl_FragCoord.xy, 1.0);" << endl;
+    fs << "  float deaaCoverage = clamp(dot(uSSEdges[0], ssPos), 0.0, 1.0);" << endl;
+    fs << "  deaaCoverage *= clamp(dot(uSSEdges[1], ssPos), 0.0, 1.0);" << endl;
+    fs << "  deaaCoverage *= clamp(dot(uSSEdges[2], ssPos), 0.0, 1.0);" << endl;
+    fs << "  deaaCoverage *= clamp(dot(uSSEdges[3], ssPos), 0.0, 1.0);" << endl;
+    fs << "  color *= deaaCoverage;" << endl;
+  }
   if (aConfig.mFeatures & ENABLE_MASK_3D) {
     fs << "  vec2 maskCoords = vMaskCoord.xy / vMaskCoord.z;" << endl;
     fs << "  COLOR_PRECISION float mask = texture2D(uMaskTexture, maskCoords).r;" << endl;

gfx/layers/opengl/OGLShaderProgram.h

@@ -40,7 +40,8 @@ enum ShaderFeatures {
   ENABLE_COLOR_MATRIX=0x200,
   ENABLE_MASK_2D=0x400,
   ENABLE_MASK_3D=0x800,
-  ENABLE_PREMULTIPLY=0x1000
+  ENABLE_PREMULTIPLY=0x1000,
+  ENABLE_DEAA=0x2000
 };
 
 class KnownUniform {
@@ -50,6 +51,7 @@ public:
     NotAKnownUniform = -1,
 
     LayerTransform = 0,
+    LayerTransformInverse,
     MaskTransform,
     LayerRects,
     MatrixProj,
@@ -73,6 +75,9 @@ public:
     BlurOffset,
     BlurAlpha,
     BlurGaussianKernel,
+    SSEdges,
+    ViewportSize,
+    VisibleCenter,
 
     KnownUniformCount
   };
@@ -163,6 +168,30 @@ public:
     return false;
   }
 
+  bool UpdateArrayUniform(int cnt, const gfx::Point3D* points) {
+    if (mLocation == -1) return false;
+    if (cnt > 4) {
+      return false;
+    }
+
+    float fp[12];
+    float *d = fp;
+    for(int i=0; i < cnt; i++) {
+      // Note: Do not want to make assumptions about .x, .y, .z member packing.
+      // If gfx::Point3D is updated to make this guarantee, SIMD optimizations
+      // may be possible
+      *d++ = points[i].x;
+      *d++ = points[i].y;
+      *d++ = points[i].z;
+    }
+
+    if (memcmp(mValue.f16v, fp, sizeof(float) * cnt * 3) != 0) {
+      memcpy(mValue.f16v, fp, sizeof(float) * cnt * 3);
+      return true;
+    }
+    return false;
+  }
+
   KnownUniformName mName;
   const char *mNameString;
   int32_t mLocation;
@@ -192,6 +221,7 @@ public:
   void SetMask2D(bool aEnabled);
   void SetMask3D(bool aEnabled);
   void SetPremultiply(bool aEnabled);
+  void SetDEAA(bool aEnabled);
 
   bool operator< (const ShaderConfigOGL& other) const {
     return mFeatures < other.mFeatures;
@@ -303,10 +333,28 @@ public:
     SetMatrixUniform(KnownUniform::LayerTransform, aMatrix);
   }
 
+  void SetLayerTransformInverse(const gfx::Matrix4x4& aMatrix) {
+    SetMatrixUniform(KnownUniform::LayerTransformInverse, aMatrix);
+  }
+
   void SetMaskLayerTransform(const gfx::Matrix4x4& aMatrix) {
     SetMatrixUniform(KnownUniform::MaskTransform, aMatrix);
   }
 
+  void SetDEAAEdges(const gfx::Point3D* aEdges) {
+    SetArrayUniform(KnownUniform::SSEdges, 4, aEdges);
+  }
+
+  void SetViewportSize(const gfx::IntSize& aSize) {
+    float vals[2] = { (float)aSize.width, (float)aSize.height };
+    SetUniform(KnownUniform::ViewportSize, 2, vals);
+  }
+
+  void SetVisibleCenter(const gfx::Point& aVisibleCenter) {
+    float vals[2] = { aVisibleCenter.x, aVisibleCenter.y };
+    SetUniform(KnownUniform::VisibleCenter, 2, vals);
+  }
+
   void SetLayerRects(const gfx::Rect* aRects) {
     float vals[16] = { aRects[0].x, aRects[0].y, aRects[0].width, aRects[0].height,
                        aRects[1].x, aRects[1].y, aRects[1].width, aRects[1].height,
@@ -333,13 +381,13 @@ public:
   }
 
   void SetRenderOffset(const nsIntPoint& aOffset) {
-    float vals[4] = { float(aOffset.x), float(aOffset.y), 0.0f, 0.0f };
+    float vals[4] = { float(aOffset.x), float(aOffset.y) };
-    SetUniform(KnownUniform::RenderTargetOffset, 4, vals);
+    SetUniform(KnownUniform::RenderTargetOffset, 2, vals);
   }
 
   void SetRenderOffset(float aX, float aY) {
-    float vals[4] = { aX, aY, 0.0f, 0.0f };
+    float vals[2] = { aX, aY };
-    SetUniform(KnownUniform::RenderTargetOffset, 4, vals);
+    SetUniform(KnownUniform::RenderTargetOffset, 2, vals);
   }
 
   void SetLayerOpacity(float aOpacity) {
@@ -497,6 +545,17 @@ protected:
     }
   }
 
+  void SetArrayUniform(KnownUniform::KnownUniformName aKnownUniform, int aLength, const gfx::Point3D *aPointValues)
+  {
+    ASSERT_THIS_PROGRAM;
+    NS_ASSERTION(aKnownUniform >= 0 && aKnownUniform < KnownUniform::KnownUniformCount, "Invalid known uniform");
+
+    KnownUniform& ku(mProfile.mUniforms[aKnownUniform]);
+    if (ku.UpdateArrayUniform(aLength, aPointValues)) {
+      mGL->fUniform3fv(ku.mLocation, aLength, ku.mValue.f16v);
+    }
+  }
+
   void SetUniform(KnownUniform::KnownUniformName aKnownUniform, GLint aIntValue) {
     ASSERT_THIS_PROGRAM;
     NS_ASSERTION(aKnownUniform >= 0 && aKnownUniform < KnownUniform::KnownUniformCount, "Invalid known uniform");

gfx/thebes/gfxPrefs.h

@@ -293,6 +293,7 @@ private:
   DECL_GFX_PREF(Live, "layers.composer2d.enabled",             Composer2DCompositionEnabled, bool, false);
   DECL_GFX_PREF(Once, "layers.d3d11.disable-warp",             LayersD3D11DisableWARP, bool, false);
   DECL_GFX_PREF(Once, "layers.d3d11.force-warp",               LayersD3D11ForceWARP, bool, false);
+  DECL_GFX_PREF(Live, "layers.deaa.enabled",                   LayersDEAAEnabled, bool, false);
   DECL_GFX_PREF(Live, "layers.draw-bigimage-borders",          DrawBigImageBorders, bool, false);
   DECL_GFX_PREF(Live, "layers.draw-borders",                   DrawLayerBorders, bool, false);
   DECL_GFX_PREF(Live, "layers.draw-tile-borders",              DrawTileBorders, bool, false);

modules/libpref/init/all.js

@@ -4155,6 +4155,9 @@ pref("layers.acceleration.force-enabled", false);
 
 pref("layers.acceleration.draw-fps", false);
 
+// Enable DEAA antialiasing for transformed layers in the compositor
+pref("layers.deaa.enabled", false);
+
 pref("layers.dump", false);
 #ifdef MOZ_DUMP_PAINTING
 // If we're dumping layers, also dump the texture data