Handle device resets by recreating the compositor, rather than the compositor IPDL stack. (bug 1245765 part 7, r=mattwoodrow)

dom/ipc/TabChild.cpp

@@ -2880,6 +2880,17 @@ TabChild::InvalidateLayers()
   FrameLayerBuilder::InvalidateAllLayers(lm);
 }
 
+void
+TabChild::CompositorUpdated(const TextureFactoryIdentifier& aNewIdentifier)
+{
+  RefPtr<LayerManager> lm = mPuppetWidget->GetLayerManager();
+  ClientLayerManager* clm = lm->AsClientLayerManager();
+
+  mTextureFactoryIdentifier = aNewIdentifier;
+  clm->UpdateTextureFactoryIdentifier(aNewIdentifier);
+  FrameLayerBuilder::InvalidateAllLayers(clm);
+}
+
 NS_IMETHODIMP
 TabChild::OnShowTooltip(int32_t aXCoords, int32_t aYCoords, const char16_t *aTipText)
 {

dom/ipc/TabChild.h

@@ -546,6 +546,7 @@ public:
 
   void ClearCachedResources();
   void InvalidateLayers();
+  void CompositorUpdated(const TextureFactoryIdentifier& aNewIdentifier);
 
   static inline TabChild* GetFrom(nsIDOMWindow* aWindow)
   {

gfx/layers/client/ClientLayerManager.cpp

@@ -550,6 +550,12 @@ ClientLayerManager::FlushRendering()
   }
 }
 
+void
+ClientLayerManager::UpdateTextureFactoryIdentifier(const TextureFactoryIdentifier& aNewIdentifier)
+{
+  mForwarder->UpdateTextureFactoryIdentifier(aNewIdentifier);
+}
+
 void
 ClientLayerManager::SendInvalidRegion(const nsIntRegion& aRegion)
 {

gfx/layers/client/ClientLayerManager.h

@@ -97,6 +97,7 @@ public:
   virtual already_AddRefed<ColorLayer> CreateColorLayer() override;
   virtual already_AddRefed<RefLayer> CreateRefLayer() override;
 
+  void UpdateTextureFactoryIdentifier(const TextureFactoryIdentifier& aNewIdentifier);
   TextureFactoryIdentifier GetTextureFactoryIdentifier()
   {
     return mForwarder->GetTextureFactoryIdentifier();

gfx/layers/composite/ContainerLayerComposite.cpp

@@ -210,6 +210,9 @@ ContainerRenderVR(ContainerT* aContainer,
     if (!layer->IsVisible() && !layer->AsContainerLayer()) {
       continue;
     }
+    if (layerToRender->HasStaleCompositor()) {
+      continue;
+    }
 
     // We flip between pre-rendered and Gecko-rendered VR based on
     // whether the child layer of this VR container layer has
@@ -553,6 +556,10 @@ RenderLayers(ContainerT* aContainer,
     const RenderTargetIntRect& clipRect = preparedData.mClipRect;
     Layer* layer = layerToRender->GetLayer();
 
+    if (layerToRender->HasStaleCompositor()) {
+      continue;
+    }
+
     Color color;
     if (NeedToDrawCheckerboardingForLayer(layer, &color)) {
       if (gfxPrefs::APZHighlightCheckerboardedAreas()) {

gfx/layers/composite/LayerManagerComposite.cpp

@@ -1496,6 +1496,14 @@ LayerManagerComposite::AutoAddMaskEffect::~AutoAddMaskEffect()
   mCompositable->RemoveMaskEffect();
 }
 
+void
+LayerManagerComposite::ChangeCompositor(Compositor* aNewCompositor)
+{
+  mCompositor = aNewCompositor;
+  mTextRenderer = new TextRenderer(aNewCompositor);
+  mTwoPassTmpTarget = nullptr;
+}
+
 LayerComposite::LayerComposite(LayerManagerComposite *aManager)
   : mCompositeManager(aManager)
   , mCompositor(aManager->GetCompositor())
@@ -1573,6 +1581,12 @@ LayerComposite::GetFullyRenderedRegion() {
   }
 }
 
+bool
+LayerComposite::HasStaleCompositor() const
+{
+  return mCompositeManager->GetCompositor() != mCompositor;
+}
+
 #ifndef MOZ_HAVE_PLATFORM_SPECIFIC_LAYER_BUFFERS
 
 /*static*/ bool

gfx/layers/composite/LayerManagerComposite.h

@@ -225,6 +225,12 @@ public:
     return mCompositor;
   }
 
+  // Called by CompositorParent when a new compositor has been created due
+  // to a device reset. The layer manager must clear any cached resources
+  // attached to the old compositor, and make a best effort at ignoring
+  // layer or texture updates against the old compositor.
+  void ChangeCompositor(Compositor* aNewCompositor);
+
   /**
    * LayerManagerComposite provides sophisticated debug overlays
    * that can request a next frame.
@@ -332,6 +338,8 @@ private:
                                bool aInvertEffect,
                                float aContrastEffect);
 
+  void ChangeCompositorInternal(Compositor* aNewCompositor);
+
   float mWarningLevel;
   mozilla::TimeStamp mWarnTime;
   bool mUnusedApzTransformWarning;
@@ -482,6 +490,9 @@ public:
   bool HasLayerBeenComposited() { return mLayerComposited; }
   gfx::IntRect GetClearRect() { return mClearRect; }
 
+  // Returns false if the layer is attached to an older compositor.
+  bool HasStaleCompositor() const;
+
   /**
    * Return the part of the visible region that has been fully rendered.
    * While progressive drawing is in progress this region will be

gfx/layers/ipc/CompositableForwarder.h

@@ -136,6 +136,9 @@ public:
 
   void IdentifyTextureHost(const TextureFactoryIdentifier& aIdentifier);
 
+  void UpdateTextureFactoryIdentifier(const TextureFactoryIdentifier& aNewIdentifier) {
+    mTextureFactoryIdentifier = aNewIdentifier;
+  }
   virtual int32_t GetMaxTextureSize() const override
   {
     return mTextureFactoryIdentifier.mMaxTextureSize;

gfx/layers/ipc/CompositorChild.cpp

@@ -210,6 +210,21 @@ CompositorChild::RecvInvalidateLayers(const uint64_t& aLayersId)
   return true;
 }
 
+bool
+CompositorChild::RecvCompositorUpdated(const uint64_t& aLayersId,
+                                      const TextureFactoryIdentifier& aNewIdentifier)
+{
+  if (mLayerManager) {
+    // This case is handled directly by nsBaseWidget.
+    MOZ_ASSERT(aLayersId == 0);
+  } else if (aLayersId != 0) {
+    if (dom::TabChild* child = dom::TabChild::GetFrom(aLayersId)) {
+      child->CompositorUpdated(aNewIdentifier);
+    }
+  }
+  return true;
+}
+
 #if defined(XP_WIN) || defined(MOZ_WIDGET_GTK)
 static void CalculatePluginClip(const LayoutDeviceIntRect& aBounds,
                                 const nsTArray<LayoutDeviceIntRect>& aPluginClipRects,

gfx/layers/ipc/CompositorChild.h

@@ -81,6 +81,10 @@ public:
   virtual bool
   RecvInvalidateLayers(const uint64_t& aLayersId) override;
 
+  virtual bool
+  RecvCompositorUpdated(const uint64_t& aLayersId,
+                        const TextureFactoryIdentifier& aNewIdentifier) override;
+
   virtual bool
   RecvOverfill(const uint32_t &aOverfill) override;
 

gfx/layers/ipc/CompositorParent.cpp

@@ -684,6 +684,7 @@ CompositorParent::CompositorParent(nsIWidget* aWidget,
   , mEGLSurfaceSize(aSurfaceWidth, aSurfaceHeight)
   , mPauseCompositionMonitor("PauseCompositionMonitor")
   , mResumeCompositionMonitor("ResumeCompositionMonitor")
+  , mResetCompositorMonitor("ResetCompositorMonitor")
   , mRootLayerTreeID(AllocateLayerTreeId())
   , mOverrideComposeReadiness(false)
   , mForceCompositionTask(nullptr)
@@ -2054,6 +2055,92 @@ CompositorParent::InvalidateRemoteLayers()
   });
 }
 
+bool
+CompositorParent::ResetCompositor(const nsTArray<LayersBackend>& aBackendHints,
+                                  TextureFactoryIdentifier* aOutIdentifier)
+{
+  Maybe<TextureFactoryIdentifier> newIdentifier;
+  {
+    MonitorAutoLock lock(mResetCompositorMonitor);
+
+    CompositorLoop()->PostTask(FROM_HERE,
+      NewRunnableMethod(this,
+                        &CompositorParent::ResetCompositorTask,
+                        aBackendHints,
+                        &newIdentifier));
+
+    mResetCompositorMonitor.Wait();
+  }
+
+  if (!newIdentifier) {
+    return false;
+  }
+
+  *aOutIdentifier = newIdentifier.value();
+  return true;
+}
+
+// Invoked on the compositor thread. The main thread is waiting on the given
+// monitor.
+void
+CompositorParent::ResetCompositorTask(const nsTArray<LayersBackend>& aBackendHints,
+                                      Maybe<TextureFactoryIdentifier>* aOutNewIdentifier)
+{
+  // Perform the reset inside a lock, so the main thread can wake up as soon as
+  // possible. We notify child processes (if necessary) outside the lock.
+  Maybe<TextureFactoryIdentifier> newIdentifier;
+  {
+    MonitorAutoLock lock(mResetCompositorMonitor);
+
+    newIdentifier = ResetCompositorImpl(aBackendHints);
+    *aOutNewIdentifier = newIdentifier;
+
+    mResetCompositorMonitor.NotifyAll();
+  }
+
+  // NOTE: |aBackendHints|, and |aOutNewIdentifier| are now all invalid since
+  // they are allocated on ResetCompositor's stack on the main thread, which
+  // is no longer waiting on the lock.
+
+  if (!newIdentifier) {
+    // No compositor change; nothing to do.
+    return;
+  }
+
+  MonitorAutoLock lock(*sIndirectLayerTreesLock);
+  ForEachIndirectLayerTree([&] (LayerTreeState* lts, uint64_t layersId) -> void {
+    if (CrossProcessCompositorParent* cpcp = lts->mCrossProcessParent) {
+      Unused << cpcp->SendCompositorUpdated(layersId, newIdentifier.value());
+    }
+  });
+}
+
+Maybe<TextureFactoryIdentifier>
+CompositorParent::ResetCompositorImpl(const nsTArray<LayersBackend>& aBackendHints)
+{
+  if (!mLayerManager) {
+    return Nothing();
+  }
+
+  RefPtr<Compositor> compositor = NewCompositor(aBackendHints);
+  if (!compositor) {
+    return Nothing();
+  }
+
+  // Don't bother changing from basic->basic.
+  if (mCompositor &&
+      mCompositor->GetBackendType() == LayersBackend::LAYERS_BASIC &&
+      compositor->GetBackendType() == LayersBackend::LAYERS_BASIC)
+  {
+    return Nothing();
+  }
+
+  mCompositor = compositor;
+  mLayerManager->ChangeCompositor(compositor);
+
+  return Some(compositor->GetTextureFactoryIdentifier());
+}
+
 static void
 OpenCompositor(CrossProcessCompositorParent* aCompositor,
                Transport* aTransport, ProcessId aOtherPid,

gfx/layers/ipc/CompositorParent.h

@@ -22,6 +22,7 @@
 #include "base/thread.h"                // for Thread
 #include "mozilla/Assertions.h"         // for MOZ_ASSERT_HELPER2
 #include "mozilla/Attributes.h"         // for override
+#include "mozilla/Maybe.h"
 #include "mozilla/Monitor.h"            // for Monitor
 #include "mozilla/RefPtr.h"             // for RefPtr
 #include "mozilla/TimeStamp.h"          // for TimeStamp
@@ -275,6 +276,20 @@ public:
                                       const nsTArray<ScrollableLayerGuid>& aTargets) override;
   virtual AsyncCompositionManager* GetCompositionManager(LayerTransactionParent* aLayerTree) override { return mCompositionManager; }
 
+  /**
+   * Request that the compositor be recreated due to a shared device reset.
+   * This must be called on the main thread, and blocks until a task posted
+   * to the compositor thread has completed.
+   *
+   * Note that this posts a task directly, rather than using synchronous
+   * IPDL, and waits on a monitor notification from the compositor thread.
+   * We do this as a best-effort attempt to jump any IPDL messages that
+   * have not yet been posted (and are sitting around in the IO pipe), to
+   * minimize the amount of time the main thread is blocked.
+   */
+  bool ResetCompositor(const nsTArray<LayersBackend>& aBackendHints,
+                       TextureFactoryIdentifier* aOutIdentifier);
+
   /**
    * This forces the is-first-paint flag to true. This is intended to
    * be called by the widget code when it loses its viewport information
@@ -502,6 +517,9 @@ protected:
   void Invalidate();
 
   RefPtr<Compositor> NewCompositor(const nsTArray<LayersBackend>& aBackendHints);
+  void ResetCompositorTask(const nsTArray<LayersBackend>& aBackendHints,
+                           Maybe<TextureFactoryIdentifier>* aOutNewIdentifier);
+  Maybe<TextureFactoryIdentifier> ResetCompositorImpl(const nsTArray<LayersBackend>& aBackendHints);
 
   /**
    * Add a compositor to the global compositor map.
@@ -541,6 +559,7 @@ protected:
 
   mozilla::Monitor mPauseCompositionMonitor;
   mozilla::Monitor mResumeCompositionMonitor;
+  mozilla::Monitor mResetCompositorMonitor;
 
   uint64_t mCompositorID;
   const uint64_t mRootLayerTreeID;

gfx/layers/ipc/PCompositor.ipdl

@@ -46,6 +46,11 @@ child:
   // TextureSources are recreated.
   async InvalidateLayers(uint64_t layersId);
 
+  // The compositor type or device has changed, and a new texture factory
+  // identifier is available. Layers must be invalidated and the new identifier
+  // must be propagated.
+  async CompositorUpdated(uint64_t layersId, TextureFactoryIdentifier newIdentifier);
+
   // The compositor completed a layers transaction. id is the layers id
   // of the child layer tree that was composited (or 0 when notifying
   // the root layer tree).

layout/base/UnitTransforms.h

@@ -9,6 +9,7 @@
 
 #include "Units.h"
 #include "mozilla/gfx/Matrix.h"
+#include "mozilla/Maybe.h"
 #include "nsRegion.h"
 
 namespace mozilla {

widget/nsBaseWidget.cpp

@@ -62,6 +62,7 @@
 #include "TouchEvents.h"
 #include "WritingModes.h"
 #include "InputData.h"
+#include "FrameLayerBuilder.h"
 #ifdef ACCESSIBILITY
 #include "nsAccessibilityService.h"
 #endif
@@ -292,6 +293,47 @@ void nsBaseWidget::DestroyLayerManager()
   DestroyCompositor();
 }
 
+void
+nsBaseWidget::OnRenderingDeviceReset()
+{
+  if (!mLayerManager || !mCompositorParent) {
+    return;
+  }
+
+  nsTArray<LayersBackend> backendHints;
+  gfxPlatform::GetPlatform()->GetCompositorBackends(ComputeShouldAccelerate(), backendHints);
+
+  // If the existing compositor does not use acceleration, and this widget
+  // should not be accelerated, then there's no point in resetting.
+  //
+  // Note that if this widget should be accelerated, but instead has a basic
+  // compositor, we still reset just in case we're now in the position to get
+  // accelerated layers again.
+  RefPtr<ClientLayerManager> clm = mLayerManager->AsClientLayerManager();
+  if (!ComputeShouldAccelerate() &&
+      clm->GetTextureFactoryIdentifier().mParentBackend != LayersBackend::LAYERS_BASIC)
+  {
+    return;
+  }
+
+  // Recreate the compositor.
+  TextureFactoryIdentifier identifier;
+  if (!mCompositorParent->ResetCompositor(backendHints, &identifier)) {
+    // No action was taken, so we don't have to do anything.
+    return;
+  }
+
+  // Invalidate all layers.
+  FrameLayerBuilder::InvalidateAllLayers(mLayerManager);
+
+  // Update the texture factory identifier.
+  clm->UpdateTextureFactoryIdentifier(identifier);
+  if (ShadowLayerForwarder* lf = clm->AsShadowForwarder()) {
+    lf->IdentifyTextureHost(identifier);
+  }
+  ImageBridgeChild::IdentifyCompositorTextureHost(identifier);
+}
+
 void
 nsBaseWidget::FreeShutdownObserver()
 {

widget/nsBaseWidget.h

@@ -487,6 +487,9 @@ protected:
 
   nsIDocument* GetDocument() const;
 
+  // Notify the compositor that a device reset has occurred.
+  void OnRenderingDeviceReset();
+
 protected:
   /**
    * Starts the OMTC compositor destruction sequence.

widget/windows/nsWindow.cpp

@@ -6793,12 +6793,6 @@ bool nsWindow::AutoErase(HDC dc)
   return false;
 }
 
-void
-nsWindow::ClearCompositor(nsWindow* aWindow)
-{
-  aWindow->DestroyLayerManager();
-}
-
 bool
 nsWindow::IsPopup()
 {

widget/windows/nsWindow.h

@@ -261,8 +261,6 @@ public:
   virtual bool            AutoErase(HDC dc);
   bool ComputeShouldAccelerate() override;
 
-  static void             ClearCompositor(nsWindow* aWindow);
-
   void                    ForcePresent();
 
   /**

widget/windows/nsWindowGfx.cpp

@@ -185,7 +185,9 @@ bool nsWindow::OnPaint(HDC aDC, uint32_t aNestingLevel)
 
   if (gfxWindowsPlatform::GetPlatform()->DidRenderingDeviceReset()) {
     gfxWindowsPlatform::GetPlatform()->UpdateRenderMode();
-    EnumAllWindows(ClearCompositor);
+    EnumAllWindows([] (nsWindow* aWindow) -> void {
+      aWindow->OnRenderingDeviceReset();
+    });
     return false;
   }