diff --git a/gfx/layers/Layers.cpp b/gfx/layers/Layers.cpp
index 482b0e4d31e..50920b7a488 100644
--- a/gfx/layers/Layers.cpp
+++ b/gfx/layers/Layers.cpp
@@ -7,6 +7,7 @@
 
 #include "mozilla/DebugOnly.h"
 
+#include "mozilla/layers/AsyncPanZoomController.h"
 #include "mozilla/layers/PLayerTransaction.h"
 #include "mozilla/layers/LayerManagerComposite.h"
 #include "mozilla/Telemetry.h"
@@ -406,36 +407,21 @@ Layer::SetAnimations(const AnimationArray& aAnimations)
   Mutated();
 }
 
-static uint8_t sPanZoomUserDataKey;
-struct PanZoomUserData : public LayerUserData {
-  PanZoomUserData(AsyncPanZoomController* aController)
-    : mController(aController)
-  { }
-
-  // We don't keep a strong ref here because PanZoomUserData is only
-  // set transiently, and APZC is thread-safe refcounted so
-  // AddRef/Release is expensive.
-  AsyncPanZoomController* mController;
-};
-
 void
-Layer::SetAsyncPanZoomController(AsyncPanZoomController *controller)
+ContainerLayer::SetAsyncPanZoomController(AsyncPanZoomController *controller)
 {
-  if (controller) {
-    SetUserData(&sPanZoomUserDataKey, new PanZoomUserData(controller));
-  } else {
-    RemoveUserData(&sPanZoomUserDataKey);
-  }
+  mAPZC = controller;
 }
 
 AsyncPanZoomController*
-Layer::GetAsyncPanZoomController()
+ContainerLayer::GetAsyncPanZoomController()
 {
-  LayerUserData* data = GetUserData(&sPanZoomUserDataKey);
-  if (!data) {
-    return nullptr;
+#ifdef DEBUG
+  if (mAPZC) {
+    MOZ_ASSERT(GetFrameMetrics().IsScrollable());
   }
-  return static_cast<PanZoomUserData*>(data)->mController;
+#endif
+  return mAPZC;
 }
 
 void
@@ -702,6 +688,23 @@ Layer::ComputeEffectiveTransformForMaskLayer(const gfx3DMatrix& aTransformToSurf
   }
 }
 
+ContainerLayer::ContainerLayer(LayerManager* aManager, void* aImplData)
+  : Layer(aManager, aImplData),
+    mFirstChild(nullptr),
+    mLastChild(nullptr),
+    mPreXScale(1.0f),
+    mPreYScale(1.0f),
+    mInheritedXScale(1.0f),
+    mInheritedYScale(1.0f),
+    mUseIntermediateSurface(false),
+    mSupportsComponentAlphaChildren(false),
+    mMayHaveReadbackChild(false)
+{
+  mContentFlags = 0; // Clear NO_TEXT, NO_TEXT_OVER_TRANSPARENT
+}
+
+ContainerLayer::~ContainerLayer() {}
+
 void
 ContainerLayer::FillSpecificAttributes(SpecificLayerAttributes& aAttrs)
 {
diff --git a/gfx/layers/Layers.h b/gfx/layers/Layers.h
index cc26391dd66..1379aa932c4 100644
--- a/gfx/layers/Layers.h
+++ b/gfx/layers/Layers.h
@@ -907,11 +907,6 @@ public:
   const gfx::Margin& GetFixedPositionMargins() { return mMargins; }
   Layer* GetMaskLayer() { return mMaskLayer; }
 
-  // These functions allow attaching an AsyncPanZoomController to this layer,
-  // and can be used anytime.
-  void SetAsyncPanZoomController(AsyncPanZoomController *controller);
-  AsyncPanZoomController* GetAsyncPanZoomController();
-
   // Note that all lengths in animation data are either in CSS pixels or app
   // units and must be converted to device pixels by the compositor.
   AnimationArray& GetAnimations() { return mAnimations; }
@@ -1381,6 +1376,9 @@ protected:
  */
 class ContainerLayer : public Layer {
 public:
+
+  ~ContainerLayer();
+
   /**
    * CONSTRUCTION PHASE ONLY
    * Insert aChild into the child list of this container. aChild must
@@ -1418,6 +1416,12 @@ public:
     }
   }
 
+  // These functions allow attaching an AsyncPanZoomController to this layer,
+  // and can be used anytime.
+  // A container layer has an APZC only-if GetFrameMetrics().IsScrollable()
+  void SetAsyncPanZoomController(AsyncPanZoomController *controller);
+  AsyncPanZoomController* GetAsyncPanZoomController();
+
   void SetPreScale(float aXScale, float aYScale)
   {
     if (mPreXScale == aXScale && mPreYScale == aYScale) {
@@ -1503,20 +1507,7 @@ protected:
   void DidInsertChild(Layer* aLayer);
   void DidRemoveChild(Layer* aLayer);
 
-  ContainerLayer(LayerManager* aManager, void* aImplData)
-    : Layer(aManager, aImplData),
-      mFirstChild(nullptr),
-      mLastChild(nullptr),
-      mPreXScale(1.0f),
-      mPreYScale(1.0f),
-      mInheritedXScale(1.0f),
-      mInheritedYScale(1.0f),
-      mUseIntermediateSurface(false),
-      mSupportsComponentAlphaChildren(false),
-      mMayHaveReadbackChild(false)
-  {
-    mContentFlags = 0; // Clear NO_TEXT, NO_TEXT_OVER_TRANSPARENT
-  }
+  ContainerLayer(LayerManager* aManager, void* aImplData);
 
   /**
    * A default implementation of ComputeEffectiveTransforms for use by OpenGL
@@ -1534,6 +1525,7 @@ protected:
   Layer* mFirstChild;
   Layer* mLastChild;
   FrameMetrics mFrameMetrics;
+  nsRefPtr<AsyncPanZoomController> mAPZC;
   float mPreXScale;
   float mPreYScale;
   // The resolution scale inherited from the parent layer. This will already
diff --git a/gfx/layers/composite/AsyncCompositionManager.cpp b/gfx/layers/composite/AsyncCompositionManager.cpp
index ac6ddbff94d..44940d8c3a7 100644
--- a/gfx/layers/composite/AsyncCompositionManager.cpp
+++ b/gfx/layers/composite/AsyncCompositionManager.cpp
@@ -51,6 +51,7 @@ WalkTheTree(Layer* aLayer,
   if (RefLayer* ref = aLayer->AsRefLayer()) {
     if (const CompositorParent::LayerTreeState* state = CompositorParent::GetIndirectShadowTree(ref->GetReferentId())) {
       if (Layer* referent = state->mRoot) {
+        ContainerLayer *referentAsContainer = referent->AsContainerLayer();
         if (!ref->GetVisibleRegion().IsEmpty()) {
           ScreenOrientation chromeOrientation = aTargetConfig.orientation();
           ScreenOrientation contentOrientation = state->mTargetConfig.orientation();
@@ -62,12 +63,16 @@ WalkTheTree(Layer* aLayer,
 
         if (OP == Resolve) {
           ref->ConnectReferentLayer(referent);
-          if (AsyncPanZoomController* apzc = state->mController) {
-            referent->SetAsyncPanZoomController(apzc);
+          if (referentAsContainer) {
+            if (AsyncPanZoomController* apzc = state->mController) {
+              referentAsContainer->SetAsyncPanZoomController(apzc);
+            }
           }
         } else {
           ref->DetachReferentLayer(referent);
-          referent->SetAsyncPanZoomController(nullptr);
+          if (referentAsContainer) {
+            referentAsContainer->SetAsyncPanZoomController(nullptr);
+          }
         }
       }
     }
@@ -335,7 +340,7 @@ AsyncCompositionManager::ApplyAsyncContentTransformToTree(TimeStamp aCurrentFram
     return appliedTransform;
   }
 
-  if (AsyncPanZoomController* controller = aLayer->GetAsyncPanZoomController()) {
+  if (AsyncPanZoomController* controller = container->GetAsyncPanZoomController()) {
     LayerComposite* layerComposite = aLayer->AsLayerComposite();
 
     ViewTransform treeTransform;
diff --git a/gfx/layers/ipc/CompositorParent.cpp b/gfx/layers/ipc/CompositorParent.cpp
index be6dbf779c3..b17ed43da89 100644
--- a/gfx/layers/ipc/CompositorParent.cpp
+++ b/gfx/layers/ipc/CompositorParent.cpp
@@ -812,9 +812,10 @@ UpdateIndirectTree(uint64_t aId, Layer* aRoot, const TargetConfig& aTargetConfig
 {
   sIndirectLayerTrees[aId].mRoot = aRoot;
   sIndirectLayerTrees[aId].mTargetConfig = aTargetConfig;
-  if (ContainerLayer* root = aRoot->AsContainerLayer()) {
+  ContainerLayer* rootContainer = aRoot->AsContainerLayer();
+  if (rootContainer) {
     if (AsyncPanZoomController* apzc = sIndirectLayerTrees[aId].mController) {
-      apzc->NotifyLayersUpdated(root->GetFrameMetrics(), isFirstPaint);
+      apzc->NotifyLayersUpdated(rootContainer->GetFrameMetrics(), isFirstPaint);
     }
   }
 }
diff --git a/widget/android/nsWindow.cpp b/widget/android/nsWindow.cpp
index ffd4e5d7870..1ce1d8fdabc 100644
--- a/widget/android/nsWindow.cpp
+++ b/widget/android/nsWindow.cpp
@@ -2493,7 +2493,7 @@ public:
         Layer* targetLayer = GetLayerManager()->GetPrimaryScrollableLayer();
         AsyncPanZoomController* controller = nsWindow::GetPanZoomController();
         if (targetLayer && targetLayer->AsContainerLayer() && controller) {
-            targetLayer->SetAsyncPanZoomController(controller);
+            targetLayer->AsContainerLayer()->SetAsyncPanZoomController(controller);
             controller->NotifyLayersUpdated(targetLayer->AsContainerLayer()->GetFrameMetrics(), isFirstPaint);
         }
     }