diff --git a/dom/media/mediasource/MediaSource.cpp b/dom/media/mediasource/MediaSource.cpp
index ce61cc3f5ab..4b27233659f 100644
--- a/dom/media/mediasource/MediaSource.cpp
+++ b/dom/media/mediasource/MediaSource.cpp
@@ -233,11 +233,27 @@ MediaSource::AddSourceBuffer(const nsAString& aType, ErrorResult& aRv)
     return nullptr;
   }
   mSourceBuffers->Append(sourceBuffer);
-  mActiveSourceBuffers->Append(sourceBuffer);
   MSE_DEBUG("sourceBuffer=%p", sourceBuffer.get());
   return sourceBuffer.forget();
 }
 
+void
+MediaSource::SourceBufferIsActive(SourceBuffer* aSourceBuffer)
+{
+  MOZ_ASSERT(NS_IsMainThread());
+  mActiveSourceBuffers->ClearSimple();
+  bool found = false;
+  for (uint32_t i = 0; i < mSourceBuffers->Length(); i++) {
+    SourceBuffer* sourceBuffer = mSourceBuffers->IndexedGetter(i, found);
+    MOZ_ALWAYS_TRUE(found);
+    if (sourceBuffer == aSourceBuffer) {
+      mActiveSourceBuffers->Append(aSourceBuffer);
+    } else if (sourceBuffer->IsActive()) {
+      mActiveSourceBuffers->AppendSimple(sourceBuffer);
+    }
+  }
+}
+
 void
 MediaSource::RemoveSourceBuffer(SourceBuffer& aSourceBuffer, ErrorResult& aRv)
 {
diff --git a/dom/media/mediasource/MediaSource.h b/dom/media/mediasource/MediaSource.h
index 7eccca92734..90dcbf0401c 100644
--- a/dom/media/mediasource/MediaSource.h
+++ b/dom/media/mediasource/MediaSource.h
@@ -122,7 +122,7 @@ private:
   // MediaSourceDecoder uses DurationChange to set the duration
   // without hitting the checks in SetDuration.
   friend class mozilla::MediaSourceDecoder;
-  // SourceBuffer uses SetDuration
+  // SourceBuffer uses SetDuration and SourceBufferIsActive
   friend class mozilla::dom::SourceBuffer;
 
   ~MediaSource();
@@ -140,6 +140,9 @@ private:
   // SetDuration with no checks.
   void SetDuration(double aDuration, MSRangeRemovalAction aAction);
 
+  // Mark SourceBuffer as active and rebuild ActiveSourceBuffers.
+  void SourceBufferIsActive(SourceBuffer* aSourceBuffer);
+
   nsRefPtr<SourceBufferList> mSourceBuffers;
   nsRefPtr<SourceBufferList> mActiveSourceBuffers;
 
diff --git a/dom/media/mediasource/SourceBuffer.cpp b/dom/media/mediasource/SourceBuffer.cpp
index 3ca85dbdb0b..b62ca34f1cf 100644
--- a/dom/media/mediasource/SourceBuffer.cpp
+++ b/dom/media/mediasource/SourceBuffer.cpp
@@ -322,6 +322,7 @@ SourceBuffer::SourceBuffer(MediaSource* aMediaSource, const nsACString& aType)
   , mTimestampOffset(0)
   , mAppendMode(SourceBufferAppendMode::Segments)
   , mUpdating(false)
+  , mActive(false)
   , mUpdateID(0)
   , mType(aType)
 {
@@ -475,7 +476,11 @@ SourceBuffer::AppendDataCompletedWithSuccess(bool aGotMedia)
   }
 
   if (mTrackBuffer->HasInitSegment()) {
-    mMediaSource->QueueInitializationEvent();
+    if (!mActive) {
+      mActive = true;
+      mMediaSource->SourceBufferIsActive(this);
+      mMediaSource->QueueInitializationEvent();
+    }
   }
 
   if (aGotMedia) {
diff --git a/dom/media/mediasource/SourceBuffer.h b/dom/media/mediasource/SourceBuffer.h
index 85a1b60fba6..3968ea7caa2 100644
--- a/dom/media/mediasource/SourceBuffer.h
+++ b/dom/media/mediasource/SourceBuffer.h
@@ -117,6 +117,11 @@ public:
   // Actually remove data between aStart and aEnd
   void DoRangeRemoval(double aStart, double aEnd);
 
+  bool IsActive() const
+  {
+    return mActive;
+  }
+
 #if defined(DEBUG)
   void Dump(const char* aPath);
 #endif
@@ -174,6 +179,8 @@ private:
   SourceBufferAppendMode mAppendMode;
   bool mUpdating;
 
+  bool mActive;
+
   // Each time mUpdating is set to true, mUpdateID will be incremented.
   // This allows for a queued AppendData task to identify if it was earlier
   // aborted and another AppendData queued.
diff --git a/dom/media/mediasource/SourceBufferList.cpp b/dom/media/mediasource/SourceBufferList.cpp
index e4b955ece90..2a8ac69e008 100644
--- a/dom/media/mediasource/SourceBufferList.cpp
+++ b/dom/media/mediasource/SourceBufferList.cpp
@@ -61,6 +61,13 @@ SourceBufferList::Append(SourceBuffer* aSourceBuffer)
   QueueAsyncSimpleEvent("addsourcebuffer");
 }
 
+void
+SourceBufferList::AppendSimple(SourceBuffer* aSourceBuffer)
+{
+  MOZ_ASSERT(NS_IsMainThread());
+  mSourceBuffers.AppendElement(aSourceBuffer);
+}
+
 void
 SourceBufferList::Remove(SourceBuffer* aSourceBuffer)
 {
@@ -88,6 +95,13 @@ SourceBufferList::Clear()
   QueueAsyncSimpleEvent("removesourcebuffer");
 }
 
+void
+SourceBufferList::ClearSimple()
+{
+  MOZ_ASSERT(NS_IsMainThread());
+  mSourceBuffers.Clear();
+}
+
 bool
 SourceBufferList::IsEmpty()
 {
diff --git a/dom/media/mediasource/SourceBufferList.h b/dom/media/mediasource/SourceBufferList.h
index de02b914a43..685f67e0f0c 100644
--- a/dom/media/mediasource/SourceBufferList.h
+++ b/dom/media/mediasource/SourceBufferList.h
@@ -78,6 +78,13 @@ public:
   // Returns the highest end time of any of the Sourcebuffers.
   double GetHighestBufferedEndTime();
 
+  // Append a SourceBuffer to the list. No event is fired.
+  void AppendSimple(SourceBuffer* aSourceBuffer);
+
+  // Remove all SourceBuffers from mSourceBuffers.
+  //  No event is fired and no action is performed on the sourcebuffers.
+  void ClearSimple();
+
 #if defined(DEBUG)
   void Dump(const char* aPath);
 #endif