Bug 806029: Have remote content drop their buffers when they're hidden. r=mattwoodrow sr=roc

dom/ipc/TabChild.cpp

@@ -1792,6 +1792,22 @@ TabChild::IsAsyncPanZoomEnabled()
     return mScrolling == ASYNC_PAN_ZOOM;
 }
 
+void
+TabChild::MakeVisible()
+{
+    if (mWidget) {
+        mWidget->Show(true);
+    }
+}
+
+void
+TabChild::MakeHidden()
+{
+    if (mWidget) {
+        mWidget->Show(false);
+    }
+}
+
 NS_IMETHODIMP
 TabChild::GetMessageManager(nsIContentFrameMessageManager** aResult)
 {

dom/ipc/TabChild.h

@@ -292,6 +292,14 @@ public:
 
     bool IsAsyncPanZoomEnabled();
 
+    /**
+     * Signal to this TabChild that it should be made visible:
+     * activated widget, retained layer tree, etc.  (Respectively,
+     * made not visible.)
+     */
+    void MakeVisible();
+    void MakeHidden();
+
 protected:
     virtual PRenderFrameChild* AllocPRenderFrame(ScrollingBehavior* aScrolling,
                                                  LayersBackend* aBackend,

gfx/layers/Layers.h

@@ -430,6 +430,24 @@ public:
     return static_cast<LayerUserData*>(mUserData.Get(static_cast<gfx::UserDataKey*>(aKey)));
   }
 
+  /**
+   * Must be called outside of a layers transaction.
+   *
+   * For the subtree rooted at |aSubtree|, this attempts to free up
+   * any free-able resources like retained buffers, but may do nothing
+   * at all.  After this call, the layer tree is left in an undefined
+   * state; the layers in |aSubtree|'s subtree may no longer have
+   * buffers with valid content and may no longer be able to draw
+   * their visible and valid regions.
+   *
+   * In general, a painting or forwarding transaction on |this| must
+   * complete on the tree before it returns to a valid state.
+   *
+   * Resource freeing begins from |aSubtree| or |mRoot| if |aSubtree|
+   * is null.  |aSubtree|'s manager must be this.
+   */
+  virtual void ClearCachedResources(Layer* aSubtree = nullptr) {}
+
   /**
    * Flag the next paint as the first for a document.
    */

gfx/layers/basic/BasicCanvasLayer.cpp

@@ -316,6 +316,11 @@ public:
   virtual void Initialize(const Data& aData);
   virtual void Paint(gfxContext* aContext, Layer* aMaskLayer);
 
+  virtual void ClearCachedResources() MOZ_OVERRIDE
+  {
+    DestroyBackBuffer();
+  }
+
   virtual void FillSpecificAttributes(SpecificLayerAttributes& aAttrs)
   {
     aAttrs = CanvasLayerAttributes(mFilter);

gfx/layers/basic/BasicImageLayer.cpp

@@ -181,6 +181,11 @@ public:
 
   virtual void Paint(gfxContext* aContext, Layer* aMaskLayer);
 
+  virtual void ClearCachedResources() MOZ_OVERRIDE
+  {
+    DestroyBackBuffer();
+  }
+
   virtual void FillSpecificAttributes(SpecificLayerAttributes& aAttrs)
   {
     aAttrs = ImageLayerAttributes(mFilter, mForceSingleTile);

gfx/layers/basic/BasicLayerManager.cpp

@@ -998,12 +998,14 @@ BasicLayerManager::PaintLayer(gfxContext* aTarget,
 }
 
 void
-BasicLayerManager::ClearCachedResources()
+BasicLayerManager::ClearCachedResources(Layer* aSubtree)
 {
-  if (mRoot) {
+  MOZ_ASSERT(!aSubtree || aSubtree->Manager() == this);
+  if (aSubtree) {
+    ClearLayer(aSubtree);
+  } else if (mRoot) {
     ClearLayer(mRoot);
   }
-
   mCachedSurface.Expire();
 }
 void
@@ -1283,6 +1285,16 @@ BasicShadowLayerManager::SetIsFirstPaint()
   ShadowLayerForwarder::SetIsFirstPaint();
 }
 
+void
+BasicShadowLayerManager::ClearCachedResources(Layer* aSubtree)
+{
+  MOZ_ASSERT(!HasShadowManager() || !aSubtree);
+  if (PLayersChild* manager = GetShadowManager()) {
+    manager->SendClearCachedResources();
+  }
+  BasicLayerManager::ClearCachedResources(aSubtree);
+}
+
 bool
 BasicShadowLayerManager::ProgressiveUpdateCallback(bool aHasPendingNewThebesContent,
                                                    gfx::Rect& aViewport,

gfx/layers/basic/BasicLayers.h

@@ -138,8 +138,8 @@ public:
   virtual const char* Name() const { return "Basic"; }
 #endif // MOZ_LAYERS_HAVE_LOG
 
-  // Clear the cached contents of this layer.
-  void ClearCachedResources();
+  // Clear the cached contents of this layer tree.
+  virtual void ClearCachedResources(Layer* aSubtree = nullptr) MOZ_OVERRIDE;
 
   void SetTransactionIncomplete() { mTransactionIncomplete = true; }
   bool IsTransactionIncomplete() { return mTransactionIncomplete; }
@@ -271,6 +271,10 @@ public:
 
   virtual void SetIsFirstPaint() MOZ_OVERRIDE;
 
+  // Drop cached resources and ask our shadow manager to do the same,
+  // if we have one.
+  virtual void ClearCachedResources(Layer* aSubtree = nullptr) MOZ_OVERRIDE;
+
   void SetRepeatTransaction() { mRepeatTransaction = true; }
 
   bool IsRepeatTransaction() { return mIsRepeatTransaction; }

gfx/layers/basic/BasicThebesLayer.h

@@ -131,6 +131,15 @@ public:
                            void* aCallbackData,
                            ReadbackProcessor* aReadback);
 
+  virtual void ClearCachedResources() MOZ_OVERRIDE
+  {
+    BasicThebesLayer::ClearCachedResources();
+    DestroyBackBuffer();
+    // Don't try to read back from this, it soon may be invalid.
+    mROFrontBuffer = null_t();
+    mFrontAndBackBufferDiffer = false;
+  }
+
   virtual void FillSpecificAttributes(SpecificLayerAttributes& aAttrs)
   {
     aAttrs = ThebesLayerAttributes(GetValidRegion());

gfx/layers/ipc/PLayers.ipdl

@@ -328,6 +328,10 @@ parent:
   // no transaction operate require a swap.
   async UpdateNoSwap(Edit[] cset, TargetConfig targetConfig, bool isFirstPaint);
 
+  // Drop any front buffers that might be retained on the compositor
+  // side.
+  async ClearCachedResources();
+
   async __delete__();
 };
 

gfx/layers/ipc/ShadowLayersParent.cpp

@@ -481,6 +481,18 @@ ShadowLayersParent::RecvUpdate(const InfallibleTArray<Edit>& cset,
   return true;
 }
 
+bool
+ShadowLayersParent::RecvClearCachedResources()
+{
+  if (mRoot) {
+    // NB: |mRoot| here is the *child* context's root.  In this parent
+    // context, it's just a subtree root.  We need to scope the clear
+    // of resources to exactly that subtree, so we specify it here.
+    mLayerManager->ClearCachedResources(mRoot);
+  }
+  return true;
+}
+
 PGrallocBufferParent*
 ShadowLayersParent::AllocPGrallocBuffer(const gfxIntSize& aSize,
                                         const gfxContentType& aContent,

gfx/layers/ipc/ShadowLayersParent.h

@@ -56,6 +56,8 @@ protected:
                                 const TargetConfig& targetConfig,
                                 const bool& isFirstPaint) MOZ_OVERRIDE;
 
+  virtual bool RecvClearCachedResources() MOZ_OVERRIDE;
+
   virtual PGrallocBufferParent*
   AllocPGrallocBuffer(const gfxIntSize& aSize, const gfxContentType& aContent,
                       MaybeMagicGrallocBufferHandle* aOutHandle) MOZ_OVERRIDE;

gfx/layers/opengl/ContainerLayerOGL.cpp

@@ -493,6 +493,7 @@ ShadowRefLayerOGL::RenderLayer(int aPreviousFrameBuffer,
 void
 ShadowRefLayerOGL::CleanupResources()
 {
+  MOZ_ASSERT(!mFirstChild);
 }
 
 } /* layers */

gfx/layers/opengl/LayerManagerOGL.cpp

@@ -300,6 +300,9 @@ LayerManagerOGL::LayerManagerOGL(nsIWidget *aWidget, int aSurfaceWidth, int aSur
   , mBackBufferSize(-1, -1)
   , mHasBGRA(0)
   , mIsRenderingToEGLSurface(aIsRenderingToEGLSurface)
+#ifdef DEBUG
+  , mMaybeInvalidTree(false)
+#endif
 {
 }
 
@@ -594,12 +597,18 @@ void
 LayerManagerOGL::BeginTransaction()
 {
   mInTransaction = true;
+#ifdef DEBUG
+  mMaybeInvalidTree = false;
+#endif
 }
 
 void
 LayerManagerOGL::BeginTransactionWithTarget(gfxContext *aTarget)
 {
   mInTransaction = true;
+#ifdef DEBUG
+  mMaybeInvalidTree = false;
+#endif
 
 #ifdef MOZ_LAYERS_HAVE_LOG
   MOZ_LAYERS_LOG(("[----- BeginTransaction"));
@@ -617,6 +626,10 @@ LayerManagerOGL::BeginTransactionWithTarget(gfxContext *aTarget)
 bool
 LayerManagerOGL::EndEmptyTransaction(EndTransactionFlags aFlags)
 {
+  // NB: this makes the somewhat bogus assumption that pure
+  // compositing txns don't call BeginTransaction(), because that's
+  // the behavior of CompositorParent.
+  MOZ_ASSERT(!mMaybeInvalidTree);
   mInTransaction = false;
 
   if (!mRoot)
@@ -739,6 +752,41 @@ LayerManagerOGL::CreateCanvasLayer()
   return layer.forget();
 }
 
+static LayerOGL*
+ToLayerOGL(Layer* aLayer)
+{
+  return static_cast<LayerOGL*>(aLayer->ImplData());
+}
+
+static void ClearSubtree(Layer* aLayer)
+{
+  ToLayerOGL(aLayer)->CleanupResources();
+  for (Layer* child = aLayer->GetFirstChild(); child;
+       child = child->GetNextSibling()) {
+    ClearSubtree(child);
+  }
+}
+
+void
+LayerManagerOGL::ClearCachedResources(Layer* aSubtree)
+{
+  MOZ_ASSERT(!aSubtree || aSubtree->Manager() == this);
+  Layer* subtree = aSubtree ? aSubtree : mRoot.get();
+  if (!subtree) {
+    return;
+  }
+
+  ClearSubtree(subtree);
+#ifdef DEBUG
+  // If this subtree is reachable from the root layer, then it's
+  // possibly onscreen, and the resource clear means that composites
+  // until the next received transaction may draw garbage to the
+  // framebuffer.
+  for(; subtree && subtree != mRoot; subtree = subtree->GetParent());
+  mMaybeInvalidTree = (subtree == mRoot);
+#endif  // DEBUG
+}
+
 LayerOGL*
 LayerManagerOGL::RootLayer() const
 {
@@ -747,7 +795,7 @@ LayerManagerOGL::RootLayer() const
     return nullptr;
   }
 
-  return static_cast<LayerOGL*>(mRoot->ImplData());
+  return ToLayerOGL(mRoot);
 }
 
 bool LayerManagerOGL::sDrawFPS = false;

gfx/layers/opengl/LayerManagerOGL.h

@@ -145,6 +145,8 @@ public:
   virtual already_AddRefed<gfxASurface>
     CreateOptimalMaskSurface(const gfxIntSize &aSize);
 
+  virtual void ClearCachedResources(Layer* aSubtree = nullptr) MOZ_OVERRIDE;
+
   /**
    * Helper methods.
    */
@@ -464,6 +466,13 @@ private:
   void *mThebesLayerCallbackData;
   gfxMatrix mWorldMatrix;
   nsAutoPtr<FPSState> mFPS;
+#ifdef DEBUG
+  // NB: only interesting when this is a purely compositing layer
+  // manager.  True after possibly onscreen layers have had their
+  // cached resources cleared outside of a transaction, and before the
+  // next forwarded transaction that re-validates their buffers.
+  bool mMaybeInvalidTree;
+#endif
 
   static bool sDrawFPS;
   static bool sFrameCounter;

layout/base/nsPresShell.cpp

@@ -9070,6 +9070,38 @@ PresShell::SetIsActive(bool aIsActive)
     }
   }
 #endif
+
+  // We have this odd special case here because remote content behaves
+  // differently from same-process content when "hidden".  In
+  // desktop-type "browser UIs", hidden "tabs" have documents that are
+  // part of the chrome tree.  When the tabs are hidden, their content
+  // is no longer part of the visible document tree, and the layers
+  // for the content are naturally released.
+  //
+  // Remote content is its own top-level tree in its subprocess.  When
+  // it's "hidden", there's no transaction in which the document
+  // thinks it's not visible, so layers can be retained forever.  This
+  // is problematic when those layers uselessly hold on to precious
+  // resources like directly texturable memory.
+  //
+  // PresShell::SetIsActive() is the first C++ entry point at which we
+  // (i) know that our parent process wants our content to be hidden;
+  // and (ii) has easy access to the TabChild.  So we use this
+  // notification to signal the TabChild to drop its layer tree and
+  // stop trying to repaint.
+  if (TabChild* tab = GetTabChildFrom(this)) {
+    if (aIsActive) {
+      tab->MakeVisible();
+      if (nsIFrame* root = mFrameConstructor->GetRootFrame()) {
+        FrameLayerBuilder::InvalidateAllLayersForFrame(
+          nsLayoutUtils::GetDisplayRootFrame(root));
+        root->SchedulePaint();
+      }
+    } else {
+      tab->MakeHidden();
+    }
+  }
+
   return rv;
 }
 

widget/xpwidgets/PuppetWidget.cpp

@@ -162,8 +162,17 @@ PuppetWidget::Show(bool aState)
   bool wasVisible = mVisible;
   mVisible = aState;
 
+  if (mChild) {
+    mChild->mVisible = aState;
+  }
+
+  if (!mVisible && mLayerManager) {
+    mLayerManager->ClearCachedResources();
+  }
+
   if (!wasVisible && mVisible) {
     Resize(mBounds.width, mBounds.height, false);
+    Invalidate(mBounds);
   }
 
   return NS_OK;
@@ -548,6 +557,12 @@ PuppetWidget::PaintTask::Run()
   return NS_OK;
 }
 
+bool
+PuppetWidget::NeedsPaint()
+{
+  return mVisible;
+}
+
 float
 PuppetWidget::GetDPI()
 {

widget/xpwidgets/PuppetWidget.h

@@ -172,6 +172,8 @@ public:
   // later on.
   virtual float GetDPI();
 
+  virtual bool NeedsPaint() MOZ_OVERRIDE;
+
   virtual TabChild* GetOwningTabChild() MOZ_OVERRIDE { return mTabChild; }
 
 private: