Bug 654950. Fix scissor rect calculations for descendants of container layers with intermediate surfaces. r=bas

gfx/layers/Layers.cpp

@@ -45,6 +45,7 @@
 #include "gfxPlatform.h"
 #include "ReadbackLayer.h"
 #include "gfxUtils.h"
+#include "mozilla/Util.h"
 
 using namespace mozilla::layers;
 
@@ -304,44 +305,55 @@ Layer::SnapTransform(const gfx3DMatrix& aTransform,
 }
 
 nsIntRect 
-Layer::CalculateScissorRect(bool aIntermediate,
-                            const nsIntRect& aVisibleRect,
-                            const nsIntRect& aParentScissor,
-                            const gfxMatrix& aTransform)
+Layer::CalculateScissorRect(const nsIntRect& aCurrentScissorRect,
+                            const gfxMatrix* aWorldTransform)
 {
-  nsIntRect scissorRect(aVisibleRect);
+  ContainerLayer* container = GetParent();
+  NS_ASSERTION(container, "This can't be called on the root!");
+
+  // Establish initial clip rect: it's either the one passed in, or
+  // if the parent has an intermediate surface, it's the extents of that surface.
+  nsIntRect currentClip;
+  if (container->UseIntermediateSurface()) {
+    currentClip.SizeTo(container->GetIntermediateSurfaceRect().Size());
+  } else {
+    currentClip = aCurrentScissorRect;
+  }
 
   const nsIntRect *clipRect = GetEffectiveClipRect();
+  if (!clipRect)
+    return currentClip;
 
-  if (!aIntermediate && !clipRect) {
-    return aParentScissor;
+  if (clipRect->IsEmpty()) {
+    // We might have a non-translation transform in the container so we can't
+    // use the code path below.
+    return nsIntRect(currentClip.TopLeft(), nsIntSize(0, 0));
   }
 
-  if (clipRect) {
-    if (clipRect->IsEmpty()) {
-      return *clipRect;
-    }
-    scissorRect = *clipRect;
-    if (!aIntermediate) {
-      gfxRect r(scissorRect.x, scissorRect.y, scissorRect.width, scissorRect.height);
-      gfxRect trScissor = aTransform.TransformBounds(r);
-      trScissor.Round();
-      if (!gfxUtils::GfxRectToIntRect(trScissor, &scissorRect)) {
-        scissorRect = aVisibleRect;
-      }
-    }
-  }
-    
-  if (aIntermediate) {
-    scissorRect.MoveBy(- aVisibleRect.TopLeft());
-  } else if (clipRect) {
-    scissorRect.IntersectRect(scissorRect, aParentScissor);
-  }
+  nsIntRect scissor = *clipRect;
+  if (!container->UseIntermediateSurface()) {
+    gfxMatrix matrix;
+    DebugOnly<bool> is2D = container->GetEffectiveTransform().Is2D(&matrix);
+    // See DefaultComputeEffectiveTransforms below
+    NS_ASSERTION(is2D && !matrix.HasNonIntegerTranslation(),
+                 "Non-integer-translation transform with clipped child should have forced intermediate surface");
+    scissor.MoveBy(nsIntPoint(PRInt32(matrix.x0), PRInt32(matrix.y0)));
 
-  NS_ASSERTION(scissorRect.x >= 0 && scissorRect.y >= 0,
-               "Attempting to scissor out of bounds!");
-
-  return scissorRect;
+    // Find the nearest ancestor with an intermediate surface
+    do {
+      container = container->GetParent();
+    } while (container && !container->UseIntermediateSurface());
+  }
+  if (container) {
+    scissor.MoveBy(-container->GetIntermediateSurfaceRect().TopLeft());
+  } else if (aWorldTransform) {
+    gfxRect r(scissor.x, scissor.y, scissor.width, scissor.height);
+    gfxRect trScissor = aWorldTransform->TransformBounds(r);
+    trScissor.Round();
+    if (!gfxUtils::GfxRectToIntRect(trScissor, &scissor))
+      return nsIntRect(currentClip.TopLeft(), nsIntSize(0, 0));
+  }
+  return currentClip.Intersect(scissor);
 }
 
 const gfx3DMatrix&
@@ -396,12 +408,12 @@ ContainerLayer::DefaultComputeEffectiveTransforms(const gfx3DMatrix& aTransformT
     useIntermediateSurface = PR_FALSE;
     gfxMatrix contTransform;
     if (!mEffectiveTransform.Is2D(&contTransform) ||
-        !contTransform.PreservesAxisAlignedRectangles()) {
+        contTransform.HasNonIntegerTranslation()) {
       for (Layer* child = GetFirstChild(); child; child = child->GetNextSibling()) {
         const nsIntRect *clipRect = child->GetEffectiveClipRect();
         /* We can't (easily) forward our transform to children with a non-empty clip
-         * rect since it would need to be adjusted for the transform.
-         * TODO: This is easily solvable for translation/scaling transforms.
+         * rect since it would need to be adjusted for the transform. See
+         * the calculations performed by CalculateScissorRect above.
          */
         if (clipRect && !clipRect->IsEmpty() && !child->GetVisibleRegion().IsEmpty()) {
           useIntermediateSurface = PR_TRUE;

gfx/layers/Layers.h

@@ -809,17 +809,16 @@ public:
   
   /**
    * Calculate the scissor rect required when rendering this layer.
-   *
-   * @param aIntermediate true if the layer is being rendered to an
-   * intermediate surface, false otherwise.
-   * @param aVisibleRect The bounds of the parent's visible region.
-   * @param aParentScissor The existing scissor rect set for the parent.
-   * @param aTransform The current 2d transform of the parent.
+   * Returns a rectangle relative to the intermediate surface belonging to the
+   * nearest ancestor that has an intermediate surface, or relative to the root
+   * viewport if no ancestor has an intermediate surface, corresponding to the
+   * clip rect for this layer intersected with aCurrentScissorRect.
+   * If no ancestor has an intermediate surface, the clip rect is transformed
+   * by aWorldTransform before being combined with aCurrentScissorRect, if
+   * aWorldTransform is non-null.
    */
-  nsIntRect CalculateScissorRect(bool aIntermediate,
-                                 const nsIntRect& aVisibleRect,
-                                 const nsIntRect& aParentScissor,
-                                 const gfxMatrix& aTransform);
+  nsIntRect CalculateScissorRect(const nsIntRect& aCurrentScissorRect,
+                                 const gfxMatrix* aWorldTransform);
 
   virtual const char* Name() const =0;
   virtual LayerType GetType() const =0;
@@ -1059,6 +1058,16 @@ public:
    */
   PRBool UseIntermediateSurface() { return mUseIntermediateSurface; }
 
+  /**
+   * Returns the rectangle covered by the intermediate surface,
+   * in this layer's coordinate system
+   */
+  nsIntRect GetIntermediateSurfaceRect()
+  {
+    NS_ASSERTION(mUseIntermediateSurface, "Must have intermediate surface");
+    return mVisibleRegion.GetBounds();
+  }
+
   /**
    * Returns true if this container has more than one non-empty child
    */

gfx/layers/d3d10/ContainerLayerD3D10.cpp

@@ -188,7 +188,6 @@ ContainerLayerD3D10::RenderLayer()
 
   gfx3DMatrix oldViewMatrix;
 
-  gfxMatrix contTransform;
   if (useIntermediate) {
     device()->OMGetRenderTargets(1, getter_AddRefs(previousRTView), NULL);
  
@@ -253,9 +252,6 @@ ContainerLayerD3D10::RenderLayer()
 
     previousViewportSize = mD3DManager->GetViewport();
     mD3DManager->SetViewport(nsIntSize(visibleRect.Size()));
-  } else {
-    PRBool is2d = GetEffectiveTransform().Is2D(&contTransform);
-    NS_ASSERTION(is2d, "Transform must be 2D");
   }
     
   D3D10_RECT oldD3D10Scissor;
@@ -280,11 +276,7 @@ ContainerLayerD3D10::RenderLayer()
     }
     
     nsIntRect scissorRect =
-      layerToRender->GetLayer()->CalculateScissorRect(useIntermediate,
-                                                      visibleRect,
-                                                      oldScissor,
-                                                      contTransform);
-
+        layerToRender->GetLayer()->CalculateScissorRect(oldScissor, nsnull);
     if (scissorRect.IsEmpty()) {
       continue;
     }
@@ -298,7 +290,7 @@ ContainerLayerD3D10::RenderLayer()
 
     layerToRender->RenderLayer();
   }
-      
+
   device()->RSSetScissorRects(1, &oldD3D10Scissor);
 
   if (useIntermediate) {

gfx/layers/d3d9/ContainerLayerD3D9.cpp

@@ -194,7 +194,6 @@ ContainerLayerD3D9::RenderLayer()
   PRBool useIntermediate = UseIntermediateSurface();
 
   mSupportsComponentAlphaChildren = PR_FALSE;
-  gfxMatrix contTransform;
   if (useIntermediate) {
     device()->GetRenderTarget(0, getter_AddRefs(previousRenderTarget));
     device()->CreateTexture(visibleRect.width, visibleRect.height, 1,
@@ -252,11 +251,6 @@ ContainerLayerD3D9::RenderLayer()
     device()->GetVertexShaderConstantF(CBmProjection, &oldViewMatrix[0][0], 4);
     device()->SetVertexShaderConstantF(CBmProjection, &viewMatrix._11, 4);
   } else {
-#ifdef DEBUG
-    PRBool is2d =
-#endif
-    GetEffectiveTransform().Is2D(&contTransform);
-    NS_ASSERTION(is2d, "Transform must be 2D");
     mSupportsComponentAlphaChildren = (GetContentFlags() & CONTENT_OPAQUE) ||
         (mParent && mParent->SupportsComponentAlphaChildren());
   }
@@ -273,12 +267,7 @@ ContainerLayerD3D9::RenderLayer()
     }
     
     nsIntRect scissorRect =
-      layerToRender->GetLayer()->CalculateScissorRect(useIntermediate,
-                                                      visibleRect,
-                                                      oldScissor,
-                                                      contTransform);
-
-
+      layerToRender->GetLayer()->CalculateScissorRect(oldScissor, nsnull);
     if (scissorRect.IsEmpty()) {
       continue;
     }

gfx/layers/opengl/ContainerLayerOGL.cpp

@@ -168,8 +168,6 @@ ContainerRender(Container* aContainer,
   nsIntPoint childOffset(aOffset);
   nsIntRect visibleRect = aContainer->GetEffectiveVisibleRegion().GetBounds();
 
-  gfxMatrix worldTransform = aManager->GetWorldTransform();
-
   nsIntRect cachedScissor = aContainer->gl()->ScissorRect();
   aContainer->gl()->PushScissorRect();
   aContainer->mSupportsComponentAlphaChildren = PR_FALSE;
@@ -233,12 +231,8 @@ ContainerRender(Container* aContainer,
       continue;
     }
 
-    nsIntRect scissorRect = 
-      layerToRender->GetLayer()->CalculateScissorRect(needsFramebuffer,
-                                                      visibleRect,
-                                                      cachedScissor,
-                                                      contTransform * worldTransform);
-
+    nsIntRect scissorRect = layerToRender->GetLayer()->
+        CalculateScissorRect(cachedScissor, &aManager->GetWorldTransform());
     if (scissorRect.IsEmpty()) {
       continue;
     }

layout/reftests/bugs/654950-1-ref.html

@@ -0,0 +1,10 @@
+<!DOCTYPE HTML>
+<html>
+<body>
+<div style="opacity:0.8; background-color: gray;">
+  <span style="display:inline-block; width:256px; height:256px; background:lime; position:relative; top:10px;">
+    <div style="background:yellow; height:50px;"></div>
+  </span>
+</div>
+</body>
+</html>

layout/reftests/bugs/654950-1.html

@@ -0,0 +1,17 @@
+<!DOCTYPE HTML>
+<html>
+<body>
+<div style="opacity:0.8; background-color: gray;">
+  <div style="overflow: hidden; -moz-transform: translate(0,10px);">
+    <canvas id="canvas" width="256" height="256"></canvas>
+  </div>
+</div>
+<script>
+var ctx = document.getElementById("canvas").getContext("2d");
+ctx.fillStyle = "yellow";
+ctx.fillRect(0, 0, 256, 50);
+ctx.fillStyle = "lime";
+ctx.fillRect(0, 50, 256, 206);
+</script>
+</body>
+</html>

layout/reftests/bugs/reftest.list

@@ -1631,8 +1631,9 @@ HTTP(..) == 635639-2.html 635639-2-ref.html
 == 641770-1.html 641770-1-ref.html
 == 641856-1.html 641856-1-ref.html
 == 645491-1.html 645491-1-ref.html
-== 653930-1.html 653930-1-ref.html
 fails-if(layersGPUAccelerated&&cocoaWidget) == 650228-1.html 650228-1-ref.html # Quartz alpha blending doesn't match GL alpha blending
+== 653930-1.html 653930-1-ref.html
 HTTP(..) == 654057-1.html 654057-1-ref.html
+== 654950-1.html 654950-1-ref.html
 == 652775-1.html 652775-1-ref.html
 != 656875.html about:blank