Bug 1214212 - Remove clips from separator transform items. r=roc

gfx/layers/Layers.h

@@ -1457,10 +1457,13 @@ public:
   bool Extend3DContext() {
     return GetContentFlags() & CONTENT_EXTEND_3D_CONTEXT;
   }
-  bool Is3DContextLeaf() {
+  bool Combines3DTransformWithAncestors() {
-    return !Extend3DContext() && GetParent() &&
+    return GetParent() &&
       reinterpret_cast<Layer*>(GetParent())->Extend3DContext();
   }
+  bool Is3DContextLeaf() {
+    return !Extend3DContext() && Combines3DTransformWithAncestors();
+  }
   /**
    * It is true if the user can see the back of the layer and the
    * backface is hidden.  The compositor should skip the layer if the

gfx/layers/basic/BasicLayerManager.cpp

@@ -864,6 +864,10 @@ InstallLayerClipPreserves3D(gfxContext* aTarget, Layer* aLayer)
   if (!clipRect) {
     return;
   }
+  MOZ_ASSERT(!aLayer->Extend3DContext() ||
+             !aLayer->Combines3DTransformWithAncestors(),
+             "Layers in a preserve 3D context have no clip"
+             " except leaves and the estabisher!");
 
   Layer* parent = aLayer->GetParent();
   Matrix4x4 transform3d =

layout/base/nsDisplayList.cpp

@@ -5414,9 +5414,11 @@ nsDisplayItem::LayerState
 nsDisplayTransform::GetLayerState(nsDisplayListBuilder* aBuilder,
                                   LayerManager* aManager,
                                   const ContainerLayerParameters& aParameters) {
-  // If the transform is 3d, or the layer takes part in preserve-3d sorting
+  // If the transform is 3d, the layer takes part in preserve-3d
-  // then we *always* want this to be an active layer.
+  // sorting, or the layer is a separator then we *always* want this
-  if (!GetTransform().Is2D() || mFrame->Combines3DTransformWithAncestors()) {
+  // to be an active layer.
+  if (!GetTransform().Is2D() || mFrame->Combines3DTransformWithAncestors() ||
+      mIsTransformSeparator) {
     return LAYER_ACTIVE_FORCE;
   }
   // Here we check if the *post-transform* bounds of this item are big enough
@@ -5587,7 +5589,7 @@ nsDisplayTransform::GetBounds(nsDisplayListBuilder* aBuilder, bool* aSnap)
     return mBounds;
   }
 
-  if (mFrame->Extend3DContext()) {
+  if (mFrame->Extend3DContext() && !mIsTransformSeparator) {
     return nsRect();
   }
 

layout/base/nsDisplayList.h

@@ -37,6 +37,7 @@
 
 class nsIContent;
 class nsRenderingContext;
+class nsDisplayList;
 class nsDisplayTableItem;
 class nsISelection;
 class nsDisplayLayerEventRegions;

layout/generic/nsFrame.cpp

@@ -1902,6 +1902,39 @@ CheckForApzAwareEventHandlers(nsDisplayListBuilder* aBuilder, nsIFrame* aFrame)
   }
 }
 
+/**
+ * True if aDescendant participates the context aAncestor participating.
+ */
+static bool
+Participate3DContextFrame(nsIFrame* aAncestor, nsIFrame* aDescendant) {
+  MOZ_ASSERT(aAncestor != aDescendant);
+  MOZ_ASSERT(aAncestor->Extend3DContext());
+  nsIFrame* frame;
+  for (frame = nsLayoutUtils::GetCrossDocParentFrame(aDescendant);
+       frame && aAncestor != frame;
+       frame = nsLayoutUtils::GetCrossDocParentFrame(frame)) {
+    if (!frame->Extend3DContext()) {
+      return false;
+    }
+  }
+  MOZ_ASSERT(frame == aAncestor);
+  return true;
+}
+
+static void
+WrapSeparatorTransform(nsDisplayListBuilder* aBuilder, nsIFrame* aFrame,
+                       nsRect& aDirtyRect,
+                       nsDisplayList* aSource, nsDisplayList* aTarget,
+                       int aIndex) {
+  if (!aSource->IsEmpty()) {
+    nsDisplayTransform *sepIdItem =
+      new (aBuilder) nsDisplayTransform(aBuilder, aFrame, aSource,
+                                        aDirtyRect, Matrix4x4(), aIndex);
+    sepIdItem->SetNoExtendContext();
+    aTarget->AppendToTop(sepIdItem);
+  }
+}
+
 void
 nsIFrame::BuildDisplayListForStackingContext(nsDisplayListBuilder* aBuilder,
                                              const nsRect&         aDirtyRect,
@@ -2161,6 +2194,36 @@ nsIFrame::BuildDisplayListForStackingContext(nsDisplayListBuilder* aBuilder,
    * we find all the correct children.
    */
   if (isTransformed && !resultList.IsEmpty()) {
+    if (!resultList.IsEmpty() && Extend3DContext()) {
+      // Install dummy nsDisplayTransform as a leaf containing
+      // descendants not participating this 3D rendering context.
+      nsDisplayList nonparticipants;
+      nsDisplayList participants;
+      int index = 1;
+
+      while (nsDisplayItem* item = resultList.RemoveBottom()) {
+        if (item->GetType() == nsDisplayItem::TYPE_TRANSFORM &&
+            Participate3DContextFrame(this, item->Frame())) {
+          // The frame of this item participates the same 3D context.
+          WrapSeparatorTransform(aBuilder, this, dirtyRect,
+                                 &nonparticipants, &participants, index++);
+          participants.AppendToTop(item);
+        } else {
+          // The frame of the item doesn't participate the current
+          // context, or has no transform.
+          //
+          // For items participating but not transformed, they are add
+          // to nonparticipants to get a separator layer for handling
+          // clips, if there is, on an intermediate surface.
+          // \see ContainerLayer::DefaultComputeEffectiveTransforms().
+          nonparticipants.AppendToTop(item);
+        }
+      }
+      WrapSeparatorTransform(aBuilder, this, dirtyRect,
+                             &nonparticipants, &participants, index++);
+      resultList.AppendToTop(&participants);
+    }
+
     // Restore clip state now so nsDisplayTransform is clipped properly.
     clipState.Restore();
     // Revert to the dirtyrect coming in from the parent, without our transform
@@ -2177,51 +2240,6 @@ nsIFrame::BuildDisplayListForStackingContext(nsDisplayListBuilder* aBuilder,
     nsDisplayTransform *transformItem =
       new (aBuilder) nsDisplayTransform(aBuilder, this, &resultList, dirtyRect);
     resultList.AppendNewToTop(transformItem);
-
-    /*
-     * Create an additional transform item as a separator layer
-     * between current and parent's 3D context if necessary.
-     *
-     * Separator layers avoid improperly exteding 3D context by
-     * children.
-     */
-    {
-      bool needAdditionalTransform = false;
-      if (Extend3DContext()) {
-        if (outerReferenceFrame->Extend3DContext()) {
-          for (nsIFrame *f = nsLayoutUtils::GetCrossDocParentFrame(this);
-               f && f != outerReferenceFrame && !f->IsTransformed();
-               f = nsLayoutUtils::GetCrossDocParentFrame(f)) {
-            if (!f->Extend3DContext()) {
-              // The first one with transform in it's 3D context chain,
-              // and it is different 3D context with the outer reference
-              // frame.
-              needAdditionalTransform = true;
-              break;
-            }
-          }
-        }
-      } else if (outerReferenceFrame->Extend3DContext() &&
-                 outerReferenceFrame != nsLayoutUtils::GetCrossDocParentFrame(this)) {
-        // The content should be transformed and drawn on a buffer,
-        // then tranformed and drawn again for outerReferenceFrame.
-        // So, a separator layer is required.
-        needAdditionalTransform = true;
-      }
-      if (needAdditionalTransform) {
-        nsRect sepDirty = dirtyRectOutsideTransform;
-        // The separator item is with ID transform and is out of this
-        // frame, so it is in the coordination of the outer reference
-        // frame.  Here translate the dirty rect back.
-        sepDirty.MoveBy(toOuterReferenceFrame);
-        nsDisplayTransform *sepIdItem =
-          new (aBuilder) nsDisplayTransform(aBuilder, this, &resultList,
-                                            sepDirty,
-                                            Matrix4x4(), 1);
-        sepIdItem->SetNoExtendContext();
-        resultList.AppendNewToTop(sepIdItem);
-      }
-    }
   }
 
   /* If we're doing VR rendering, then we need to wrap everything in a nsDisplayVR

layout/reftests/bugs/reftest.list

@@ -1719,7 +1719,7 @@ skip-if(B2G||Mulet) == 729143-1.html 729143-1-ref.html # Initial mulet triage: p
 == 731521-1.html 731521-1-ref.html
 needs-focus == 731726-1.html 731726-1-ref.html
 == 735481-1.html 735481-1-ref.html
-== 745934-1.html 745934-1-ref.html
+fuzzy-if(cocoaWidget,1,300000) == 745934-1.html 745934-1-ref.html
 == 748692-1a.html 748692-1-ref.html
 == 748692-1b.html 748692-1-ref.html
 skip-if(B2G||Mulet) == 748803-1.html 748803-1-ref.html # Initial mulet triage: parity with B2G/B2G Desktop

layout/reftests/outline/reftest.list

@@ -1,7 +1,7 @@
 == outline-and-box-shadow.html outline-and-box-shadow-ref.html
 == outline-and-3d-transform-1a.html outline-and-3d-transform-1-ref.html
 == outline-and-3d-transform-1b.html outline-and-3d-transform-1-ref.html
-fuzzy-if(Android,255,356) fuzzy-if(d2d,16,96) fuzzy-if(cocoaWidget,255,120) fuzzy-if(B2G,128,60) fuzzy-if(gtkWidget,128,120) fuzzy-if(winWidget,255,120) == outline-and-3d-transform-2.html outline-and-3d-transform-2-ref.html
+fuzzy-if(Mulet||gtkWidget,136,120) fuzzy-if(Android,255,356) fuzzy-if(d2d,16,96) fuzzy-if(cocoaWidget,255,120) fuzzy-if(B2G,128,60) fuzzy-if(winWidget,255,215) == outline-and-3d-transform-2.html outline-and-3d-transform-2-ref.html
 == outline-overflow-block-abspos.html outline-overflow-block-ref.html
 == outline-overflow-block-float.html outline-overflow-block-ref.html
 == outline-overflow-inlineblock-abspos.html outline-overflow-inlineblock-ref.html

layout/reftests/transform-3d/reftest.list

@@ -18,7 +18,7 @@ fuzzy-if(gtkWidget||winWidget,8,376) fuzzy-if(Android,8,441) fuzzy-if(cocoaWidge
 == preserve3d-2b.html preserve3d-2-ref.html
 == preserve3d-2c.html preserve3d-2-ref.html
 == preserve3d-2d.html preserve3d-2-ref.html
-fuzzy(4,100) == preserve3d-3a.html preserve3d-3-ref.html
+== preserve3d-3a.html preserve3d-3-ref.html
 skip-if(B2G||Mulet) == preserve3d-4a.html green-rect.html # Initial mulet triage: parity with B2G/B2G Desktop
 fuzzy-if(gtkWidget,4,200) fuzzy-if(Android&&AndroidVersion>=15,4,300) == preserve3d-5a.html preserve3d-5-ref.html
 == scale3d-z.html scalez-1-ref.html