Bug 1226904 - Fix boundary checking for leaves collecting. r=roc

layout/base/nsDisplayList.cpp

@@ -1895,11 +1895,9 @@ void nsDisplayList::HitTest(nsDisplayListBuilder* aBuilder, const nsRect& aRect,
   if (aState->mInPreserves3D) {
     // Collect leaves of the current 3D rendering context.
     for (item = GetBottom(); item; item = item->GetAbove()) {
-      MOZ_ASSERT(item->GetType() == nsDisplayTransform::TYPE_TRANSFORM ||
-                 item->GetType() == nsDisplayTransform::TYPE_PERSPECTIVE);
-      if (item->Frame()->Extend3DContext() &&
-          (item->GetType() == nsDisplayTransform::TYPE_PERSPECTIVE ||
-           !static_cast<nsDisplayTransform*>(item)->IsTransformSeparator())) {
+      auto itemType = item->GetType();
+      if (itemType != nsDisplayItem::TYPE_TRANSFORM ||
+          !static_cast<nsDisplayTransform*>(item)->IsLeafOf3DContext()) {
         item->HitTest(aBuilder, aRect, aState, aOutFrames);
       } else {
         // One of leaves in the current 3D rendering context.
@@ -1923,33 +1921,33 @@ void nsDisplayList::HitTest(nsDisplayListBuilder* aBuilder, const nsRect& aRect,
 
     bool snap;
     nsRect r = item->GetBounds(aBuilder, &snap).Intersect(aRect);
+    auto itemType = item->GetType();
     bool alwaysIntersect =
-      item->GetType() == nsDisplayItem::TYPE_TRANSFORM &&
-      (item->Frame()->Combines3DTransformWithAncestors() ||
-       item->Frame()->Extend3DContext() ||
-       static_cast<nsDisplayTransform*>(item)->IsTransformSeparator());
+      (itemType == nsDisplayItem::TYPE_TRANSFORM &&
+       static_cast<nsDisplayTransform*>(item)->IsParticipating3DContext()) ||
+      (itemType == nsDisplayItem::TYPE_PERSPECTIVE &&
+       static_cast<nsDisplayPerspective*>(item)->Frame()->Extend3DContext());
+    if (alwaysIntersect &&
+        !static_cast<nsDisplayTransform*>(item)->IsLeafOf3DContext()) {
+      nsAutoTArray<nsIFrame*, 1> neverUsed;
+      // Start gethering leaves of the 3D rendering context, and
+      // append leaves at the end of mItemBuffer.  Leaves are
+      // processed at following iterations.
+      aState->mInPreserves3D = true;
+      item->HitTest(aBuilder, aRect, aState, &neverUsed);
+      aState->mInPreserves3D = false;
+      i = aState->mItemBuffer.Length();
+      continue;
+    }
     if (alwaysIntersect || item->GetClip().MayIntersect(r)) {
       nsAutoTArray<nsIFrame*, 16> outFrames;
-      if (item->Frame()->Extend3DContext() &&
-          item->GetType() == nsDisplayItem::TYPE_TRANSFORM &&
-          !static_cast<nsDisplayTransform*>(item)->IsTransformSeparator()) {
-        // Start gethering leaves of the 3D rendering context, and
-        // append leaves at the end of mItemBuffer.  Leaves are
-        // processed at following iterations.
-        aState->mInPreserves3D = true;
-        item->HitTest(aBuilder, aRect, aState, &outFrames);
-        aState->mInPreserves3D = false;
-        i = aState->mItemBuffer.Length();
-        continue;
-      } else {
-        item->HitTest(aBuilder, aRect, aState, &outFrames);
-      }
+      item->HitTest(aBuilder, aRect, aState, &outFrames);
 
       // For 3d transforms with preserve-3d we add hit frames into the temp list
       // so we can sort them later, otherwise we add them directly to the output list.
       nsTArray<nsIFrame*> *writeFrames = aOutFrames;
       if (item->GetType() == nsDisplayItem::TYPE_TRANSFORM &&
-          item->Frame()->Combines3DTransformWithAncestors()) {
+          static_cast<nsDisplayTransform*>(item)->IsLeafOf3DContext()) {
         if (outFrames.Length()) {
           nsDisplayTransform *transform = static_cast<nsDisplayTransform*>(item);
           nsPoint point = aRect.TopLeft();
@@ -1958,6 +1956,7 @@ void nsDisplayList::HitTest(nsDisplayListBuilder* aBuilder, const nsRect& aRect,
             point = aRect.Center();
           }
           temp.AppendElement(FramesWithDepth(transform->GetHitDepthAtPoint(aBuilder, point)));
+          printf("depth %p %f\n", transform, transform->GetHitDepthAtPoint(aBuilder, point));
           writeFrames = &temp[temp.Length() - 1].mFrames;
         }
       } else {
@@ -5570,14 +5569,14 @@ nsDisplayTransform::GetTransformForRendering()
 const Matrix4x4&
 nsDisplayTransform::GetAccumulatedPreserved3DTransform(nsDisplayListBuilder* aBuilder)
 {
+  MOZ_ASSERT(!mFrame->Extend3DContext() || IsLeafOf3DContext());
   // XXX: should go back to fix mTransformGetter.
   if (!mTransformPreserves3DInited) {
     mTransformPreserves3DInited = true;
-    if (!mFrame->Combines3DTransformWithAncestors()) {
+    if (!IsLeafOf3DContext()) {
       mTransformPreserves3D = GetTransform();
       return mTransformPreserves3D;
     }
-    MOZ_ASSERT(!mFrame->Extend3DContext() || IsTransformSeparator());
 
     const nsIFrame* establisher; // Establisher of the 3D rendering context.
     for (establisher = nsLayoutUtils::GetCrossDocParentFrame(mFrame);
@@ -5814,7 +5813,6 @@ nsDisplayTransform::GetHitDepthAtPoint(nsDisplayListBuilder* aBuilder, const nsP
   inverse.Invert();
   Point4D point = inverse.ProjectPoint(Point(NSAppUnitsToFloatPixels(aPoint.x, factor),
                                              NSAppUnitsToFloatPixels(aPoint.y, factor)));
-  NS_ASSERTION(point.HasPositiveWCoord(), "Why are we trying to get the depth for a point we didn't hit?");
 
   Point point2d = point.As2DPoint();
 

layout/base/nsDisplayList.h

@@ -4085,6 +4085,14 @@ public:
             (!mFrame->Extend3DContext() &&
              mFrame->Combines3DTransformWithAncestors()));
   }
+  /**
+   * The backing frame of this item participates a 3D rendering
+   * context.
+   */
+  bool IsParticipating3DContext() {
+    return mFrame->Extend3DContext() ||
+      mFrame->Combines3DTransformWithAncestors();
+  }
 
 private:
   void ComputeBounds(nsDisplayListBuilder* aBuilder);
@@ -4199,6 +4207,11 @@ public:
 
   nsIFrame* TransformFrame() { return mTransformFrame; }
 
+  virtual void
+  DoUpdateBoundsPreserves3D(nsDisplayListBuilder* aBuilder) override {
+    static_cast<nsDisplayTransform*>(mList.GetChildren()->GetTop())->DoUpdateBoundsPreserves3D(aBuilder);
+  }
+
 private:
   nsDisplayWrapList mList;
   nsIFrame* mTransformFrame;

layout/base/tests/bug1226904.html

@@ -0,0 +1,35 @@
+<!DOCTYPE HTML>
+<html>
+<!--
+https://bugzilla.mozilla.org/show_bug.cgi?id=684759
+-->
+<head>
+  <title>Test for Bug 684759</title>
+</head>
+<style>
+  #container {
+  transform-style: preserve-3d;
+  transform: translate3d(400px, 0px, 10px);
+  background-color: black;
+  }
+  #separator {
+  width: 400px;
+  height: 400px;
+  background-color: green;
+  }
+  #transformed {
+  width: 400px;
+  height: 400px;
+  background-color: blue;
+  transform: translate3d(-400px, 0px, 10px);
+  }
+</style>
+<body onload="run()">
+<div>
+  <div id="container">
+    <div id="separator"></div>
+    <div id="transformed"></div>
+  </div>
+</div>
+</body>
+</html>

layout/base/tests/mochitest.ini

@@ -263,3 +263,5 @@ support-files = bug1153130_inner.html
 support-files =
   bug1162990_inner_1.html
   bug1162990_inner_2.html
+[test_bug1226904.html]
+support-files = bug1226904.html

layout/base/tests/preserve3d_sorting_hit_testing_iframe.html

@@ -12,14 +12,14 @@
     width: 1000px;
     height: 1000px;
     background-color: #995C7F;
-    -moz-transform: rotatey(45deg);
+    -moz-transform: translate3d(0px, 0px, 350px) rotatey(45deg);
   }
 
   #small {
     width: 100px;
     height: 100px;
     background-color: #835A99;
-    -moz-transform: translate3d(600px, 200px, 0px);
+    -moz-transform: translate3d(600px, 200px, 350px);
   }
 
 </style>

layout/base/tests/test_bug1226904.html

@@ -0,0 +1,44 @@
+<!DOCTYPE HTML>
+<html>
+<!--
+https://bugzilla.mozilla.org/show_bug.cgi?id=684759
+-->
+<head>
+  <title>Test for Bug 684759</title>
+  <script type="application/javascript" src="/tests/SimpleTest/SimpleTest.js"></script>
+  <link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css"/>
+</head>
+<body onload="run()">
+<a target="_blank" href="https://bugzilla.mozilla.org/show_bug.cgi?id=1226904">Mozilla Bug 1226904</a>
+<iframe src="bug1226904.html" id="iframe" height="1000" width="1000" style="border:none"></iframe>
+<pre id="test">
+<script type="application/javascript">
+
+/** Test for Bug 1226904 **/
+
+SimpleTest.waitForExplicitFinish();
+
+function run() {
+  var iframe = document.getElementById("iframe");
+  var doc = iframe.contentDocument;
+  var container = doc.getElementById("container");
+  var separator = doc.getElementById("separator");
+  var transformed = doc.getElementById("transformed");
+
+  function check(x, y, expected_element, description)
+  {
+	is(doc.elementFromPoint(x, y), expected_element,
+	   "point (" + x + ", " + y + "): " + description);
+  }
+
+  check(600, 200, separator, "Separator object should be at right side");
+  check(900, 200, container, "Check bounds of separator object");
+  check(200, 600, transformed, "Transformed object should be at left side");
+
+  SimpleTest.finish();
+}
+
+</script>
+</pre>
+</body>
+</html>

layout/base/tests/test_preserve3d_sorting_hit_testing.html

@@ -29,7 +29,7 @@ function run() {
 
   function check(x, y, expected_element, description)
   {
-	is(doc.elementFromPoint(x, y), expected_element,
+	is(doc.elementFromPoint(x, y).id, expected_element.id,
 	   "point (" + x + ", " + y + "): " + description);
   }
 

layout/generic/nsFrame.cpp

@@ -1923,6 +1923,9 @@ ItemParticipatesIn3DContext(nsIFrame* aAncestor, nsDisplayItem* aItem)
   } else {
     return false;
   }
+  if (aAncestor == transformFrame) {
+    return true;
+  }
   return FrameParticipatesIn3DContext(aAncestor, transformFrame);
 }