Bug 1032839: Backend support for PeerConnection ReplaceTrack() r=padenot,jib

content/media/MediaStreamGraph.h

@@ -855,6 +855,7 @@ protected:
 
   TrackData* FindDataForTrack(TrackID aID)
   {
+    mMutex.AssertCurrentThreadOwns();
     for (uint32_t i = 0; i < mUpdateTracks.Length(); ++i) {
       if (mUpdateTracks[i].mID == aID) {
         return &mUpdateTracks[i];

content/media/StreamBuffer.h

@@ -24,6 +24,7 @@ const StreamTime STREAM_TIME_MAX = MEDIA_TIME_MAX;
  */
 typedef int32_t TrackID;
 const TrackID TRACK_NONE = 0;
+const TrackID TRACK_INVALID = -1;
 
 inline TrackTicks RateConvertTicksRoundDown(TrackRate aOutRate,
                                             TrackRate aInRate,

dom/media/tests/mochitest/test_peerConnection_replaceTrack.html

@@ -14,7 +14,8 @@
 <script type="application/javascript;version=1.8">
   createHTML({
     bug: "1032839",
-    title: "Replace video track"
+    title: "Replace video track",
+    visible: true
   });
 
   function isSenderOfTrack(sender) {
@@ -28,8 +29,9 @@
     test = new PeerConnectionTest();
     test.setMediaConstraints([{video: true}], [{video: true}]);
     test.chain.removeAfter("PC_REMOTE_CHECK_MEDIA_FLOW_PRESENT");
-    test.chain.insertBefore("PC_LOCAL_CHECK_MEDIA_FLOW_PRESENT", [[
-      "PC_LOCAL_REPLACE_VIDEOTRACK",
+    var flowtest = test.chain.remove("PC_REMOTE_CHECK_MEDIA_FLOW_PRESENT");
+    test.chain.append(flowtest);
+    test.chain.append([["PC_LOCAL_REPLACE_VIDEOTRACK",
       function (test) {
         var stream = test.pcLocal._pc.getLocalStreams()[0];
         var track = stream.getVideoTracks()[0];
@@ -52,6 +54,7 @@
         });
       }
     ]]);
+    test.chain.append(flowtest);
 
     test.run();
   });

media/webrtc/signaling/src/media/VcmSIPCCBinding.cpp

@@ -2562,12 +2562,15 @@ static int vcmTxStartICE_m(cc_mcapid_t mcap_id,
   const char *mediaType;
   mozilla::RefPtr<mozilla::MediaSessionConduit> conduit;
   int err = VCM_ERROR;
+  bool is_video;
   if (CC_IS_AUDIO(mcap_id)) {
     mediaType = "audio";
     err = vcmTxCreateAudioConduit(level, payload, pc, attrs, conduit);
+    is_video = false;
   } else if (CC_IS_VIDEO(mcap_id)) {
     mediaType = "video";
     err = vcmTxCreateVideoConduit(level, payload, pc, attrs, conduit);
+    is_video = true;
   } else {
     CSFLogError(logTag, "%s: mcap_id unrecognized", __FUNCTION__);
   }
@@ -2586,6 +2589,7 @@ static int vcmTxStartICE_m(cc_mcapid_t mcap_id,
       stream->GetMediaStream(),
       pc_track_id,
       level,
+      is_video,
       conduit, rtp_flow, rtcp_flow);
 
   nsresult res = pipeline->Init();

media/webrtc/signaling/src/mediapipeline/MediaPipeline.cpp

@@ -627,11 +627,17 @@ void MediaPipeline::PacketReceived(TransportLayer *layer,
 }
 
 nsresult MediaPipelineTransmit::Init() {
+  AttachToTrack(track_id_);
+
+  return MediaPipeline::Init();
+}
+
+void MediaPipelineTransmit::AttachToTrack(TrackID track_id) {
   char track_id_string[11];
   ASSERT_ON_THREAD(main_thread_);
 
   // We can replace this when we are allowed to do streams or std::to_string
-  PR_snprintf(track_id_string, sizeof(track_id_string), "%d", track_id_);
+  PR_snprintf(track_id_string, sizeof(track_id_string), "%d", track_id);
 
   description_ = pc_ + "| ";
   description_ += conduit_->type() == MediaSessionConduit::AUDIO ?
@@ -657,8 +663,6 @@ nsresult MediaPipelineTransmit::Init() {
   // this enables the unit tests that can't fiddle with principals and the like
   listener_->SetEnabled(true);
 #endif
-
-  return MediaPipeline::Init();
 }
 
 #ifdef MOZILLA_INTERNAL_API
@@ -694,6 +698,23 @@ nsresult MediaPipelineTransmit::TransportReady_s(TransportInfo &info) {
   return NS_OK;
 }
 
+nsresult MediaPipelineTransmit::ReplaceTrack(DOMMediaStream *domstream,
+                                             TrackID track_id) {
+  // MainThread, checked in calls we make
+  MOZ_MTLOG(ML_DEBUG, "Reattaching pipeline to stream "
+            << static_cast<void *>(domstream->GetStream()) << " conduit type=" <<
+            (conduit_->type() == MediaSessionConduit::AUDIO ?"audio":"video"));
+
+  if (domstream_) { // may be excessive paranoia
+    DetachMediaStream();
+  }
+  domstream_ = domstream; // Detach clears it
+  stream_ = domstream->GetStream();
+  //track_id_ = track_id; not threadsafe to change this; and we don't need to
+  AttachToTrack(track_id);
+  return NS_OK;
+}
+
 void MediaPipeline::DisconnectTransport_s(TransportInfo &info) {
   MOZ_ASSERT(info.transport_);
   ASSERT_ON_THREAD(sts_thread_);
@@ -947,6 +968,10 @@ NotifyQueuedTrackChanges(MediaStreamGraph* graph, TrackID tid,
 #define CRSIZE(x,y) ((((x)+1) >> 1) * (((y)+1) >> 1))
 #define I420SIZE(x,y) (YSIZE((x),(y)) + 2 * CRSIZE((x),(y)))
 
+// XXX NOTE: this code will have to change when we get support for multiple tracks of type
+// in a MediaStream and especially in a PeerConnection stream.  bug 1056650
+// It should be matching on the "correct" track for the pipeline, not just "any video track".
+
 void MediaPipelineTransmit::PipelineListener::
 NewData(MediaStreamGraph* graph, TrackID tid,
         TrackRate rate,
@@ -958,15 +983,25 @@ NewData(MediaStreamGraph* graph, TrackID tid,
     return;
   }
 
+  if (track_id_ != TRACK_INVALID) {
+    if (tid != track_id_) {
+      return;
+    }
+  } else if (conduit_->type() !=
+             (media.GetType() == MediaSegment::AUDIO ? MediaSessionConduit::AUDIO :
+                                                       MediaSessionConduit::VIDEO)) {
+    // Ignore data in case we have a muxed stream
+    return;
+  } else {
+    // Don't lock during normal media flow except on first sample
+    MutexAutoLock lock(mMutex);
+    track_id_ = track_id_external_ = tid;
+  }
+
   // TODO(ekr@rtfm.com): For now assume that we have only one
   // track type and it's destined for us
   // See bug 784517
   if (media.GetType() == MediaSegment::AUDIO) {
-    if (conduit_->type() != MediaSessionConduit::AUDIO) {
-      // Ignore data in case we have a muxed stream
-      return;
-    }
-
     AudioSegment* audio = const_cast<AudioSegment *>(
         static_cast<const AudioSegment *>(&media));
 
@@ -978,11 +1013,6 @@ NewData(MediaStreamGraph* graph, TrackID tid,
     }
   } else if (media.GetType() == MediaSegment::VIDEO) {
 #ifdef MOZILLA_INTERNAL_API
-    if (conduit_->type() != MediaSessionConduit::VIDEO) {
-      // Ignore data in case we have a muxed stream
-      return;
-    }
-
     VideoSegment* video = const_cast<VideoSegment *>(
         static_cast<const VideoSegment *>(&media));
 

media/webrtc/signaling/src/mediapipeline/MediaPipeline.h

@@ -146,6 +146,7 @@ class MediaPipeline : public sigslot::has_slots<> {
   virtual Direction direction() const { return direction_; }
   virtual TrackID trackid() const { return track_id_; }
   virtual int level() const { return level_; }
+  virtual bool IsVideo() const { return false; }
 
   bool IsDoingRtcpMux() const {
     return (rtp_.type_ == MUX);
@@ -244,8 +245,10 @@ class MediaPipeline : public sigslot::has_slots<> {
   RefPtr<MediaStream> stream_;  // A pointer to the stream we are servicing.
                                 // Written on the main thread.
                                 // Used on STS and MediaStreamGraph threads.
+                                // May be changed by rtpSender.replaceTrack()
   TrackID track_id_;            // The track on the stream.
-                                // Written and used as the stream_;
+                                // Written and used as with the stream_;
+                                // Not used outside initialization in MediaPipelineTransmit
   int level_; // The m-line index (starting at 1, to match convention)
   RefPtr<MediaSessionConduit> conduit_;  // Our conduit. Written on the main
                                          // thread. Read on STS thread.
@@ -372,21 +375,34 @@ public:
                         nsCOMPtr<nsIEventTarget> main_thread,
                         nsCOMPtr<nsIEventTarget> sts_thread,
                         DOMMediaStream *domstream,
-                        TrackID track_id,
+                        int pipeline_index, // For PeerConnectionMedia/mPipelines
                         int level,
+                        bool is_video,
                         RefPtr<MediaSessionConduit> conduit,
                         RefPtr<TransportFlow> rtp_transport,
                         RefPtr<TransportFlow> rtcp_transport) :
       MediaPipeline(pc, TRANSMIT, main_thread, sts_thread,
-                    domstream->GetStream(), track_id, level,
+                    domstream->GetStream(), TRACK_INVALID, level,
                     conduit, rtp_transport, rtcp_transport),
       listener_(new PipelineListener(conduit)),
-      domstream_(domstream)
+      domstream_(domstream),
+      pipeline_index_(pipeline_index),
+      is_video_(is_video)
   {}
 
   // Initialize (stuff here may fail)
   virtual nsresult Init() MOZ_OVERRIDE;
 
+  virtual void AttachToTrack(TrackID track_id);
+
+  // Index used to refer to this before we know the TrackID
+  virtual TrackID pipeline_index() const { return pipeline_index_; }
+  // Note: unlike MediaPipeline::trackid(), this is threadsafe
+  // Not set until first media is received
+  virtual TrackID const trackid_locked() { return listener_->trackid(); }
+  // written and used from MainThread
+  virtual bool IsVideo() const MOZ_OVERRIDE { return is_video_; }
+
 #ifdef MOZILLA_INTERNAL_API
   // when the principal of the PeerConnection changes, it calls through to here
   // so that we can determine whether to enable stream transmission
@@ -407,12 +423,23 @@ public:
   // Override MediaPipeline::TransportReady.
   virtual nsresult TransportReady_s(TransportInfo &info);
 
+  // Replace a track with a different one
+  // In non-compliance with the likely final spec, allow the new
+  // track to be part of a different stream (since we don't support
+  // multiple tracks of a type in a stream yet).  bug 1056650
+  virtual nsresult ReplaceTrack(DOMMediaStream *domstream,
+                                TrackID track_id);
+
+
   // Separate class to allow ref counting
   class PipelineListener : public MediaStreamDirectListener {
    friend class MediaPipelineTransmit;
    public:
     PipelineListener(const RefPtr<MediaSessionConduit>& conduit)
       : conduit_(conduit),
+        track_id_(TRACK_INVALID),
+        mMutex("MediaPipelineTransmit::PipelineListener"),
+        track_id_external_(TRACK_INVALID),
         active_(false),
         enabled_(false),
         direct_connect_(false),
@@ -438,6 +465,10 @@ public:
 
     void SetActive(bool active) { active_ = active; }
     void SetEnabled(bool enabled) { enabled_ = enabled; }
+    TrackID trackid() {
+      MutexAutoLock lock(mMutex);
+      return track_id_external_;
+    }
 
     // Implement MediaStreamListener
     virtual void NotifyQueuedTrackChanges(MediaStreamGraph* graph, TrackID tid,
@@ -469,6 +500,13 @@ public:
 #endif
     RefPtr<MediaSessionConduit> conduit_;
 
+    // May be TRACK_INVALID until we see data from the track
+    TrackID track_id_; // this is the current TrackID this listener is attached to
+    Mutex mMutex;
+    // protected by mMutex
+    // May be TRACK_INVALID until we see data from the track
+    TrackID track_id_external_; // this is queried from other threads
+
     // active is true if there is a transport to send on
     mozilla::Atomic<bool> active_;
     // enabled is true if the media access control permits sending
@@ -495,6 +533,8 @@ public:
  private:
   RefPtr<PipelineListener> listener_;
   DOMMediaStream *domstream_;
+  int pipeline_index_; // for lookups in LocalSourceStreamInfo::mPipelines;
+  bool is_video_;
 };
 
 

media/webrtc/signaling/src/peerconnection/PeerConnectionImpl.cpp

@@ -1475,7 +1475,7 @@ PeerConnectionImpl::AddTrack(MediaStreamTrack& aTrack,
 
   // XXX Remove this check once addStream has an error callback
   // available and/or we have plumbing to handle multiple
-  // local audio streams.
+  // local audio streams.  bug 1056650
   if ((hints & DOMMediaStream::HINT_CONTENTS_AUDIO) &&
       mNumAudioStreams > 0) {
     CSFLogError(logTag, "%s: Only one local audio stream is supported for now",
@@ -1485,7 +1485,7 @@ PeerConnectionImpl::AddTrack(MediaStreamTrack& aTrack,
 
   // XXX Remove this check once addStream has an error callback
   // available and/or we have plumbing to handle multiple
-  // local video streams.
+  // local video streams. bug 1056650
   if ((hints & DOMMediaStream::HINT_CONTENTS_VIDEO) &&
       mNumVideoStreams > 0) {
     CSFLogError(logTag, "%s: Only one local video stream is supported for now",
@@ -1585,8 +1585,32 @@ PeerConnectionImpl::ReplaceTrack(MediaStreamTrack& aThisTrack,
   // Since a track may be replaced more than once, the track being replaced
   // may not be in the stream either, so we check neither arg right now.
 
-  // Insert magic here.
-  bool success = true;
+  // XXX This MUST be addressed when we add multiple tracks of a type!!
+  // This is needed because the track IDs used by MSG are from TrackUnion
+  // (for getUserMedia streams) and aren't the same as the values the source tracks
+  // have.  Solution is to have SIPCC/VcmSIPCCBinding read track ids and use those.
+
+  // Because DirectListeners see the SourceMediaStream's TrackID's, and not the
+  // TrackUnionStream's TrackID's, this value won't currently match what is used in
+  // MediaPipelineTransmit.  Bug 1056652
+  //  TrackID thisID = aThisTrack.GetTrackID();
+  TrackID withID = aWithTrack.GetTrackID();
+
+  bool success = false;
+  for(uint32_t i = 0; i < media()->LocalStreamsLength(); ++i) {
+    LocalSourceStreamInfo *info = media()->GetLocalStream(i);
+    // XXX use type instead of TrackID - bug 1056650
+    int pipeline = info->HasTrackType(&aStream, !!(aThisTrack.AsVideoStreamTrack()));
+    if (pipeline >= 0) {
+      // XXX GetStream() will likely be invalid once a track can be in more than one
+      info->ReplaceTrack(pipeline, aWithTrack.GetStream(), withID);
+      success = true;
+      break;
+    }
+  }
+  if (!success) {
+    return NS_ERROR_FAILURE;
+  }
 
   nsRefPtr<PeerConnectionObserver> pco = do_QueryObjectReferent(mPCObserver);
   if (!pco) {

media/webrtc/signaling/src/peerconnection/PeerConnectionMedia.cpp

@@ -102,6 +102,51 @@ void LocalSourceStreamInfo::DetachMedia_m()
   mMediaStream = nullptr;
 }
 
+#if 0
+// XXX  bug 1056652 makes this not very useful for transmit streams
+// NOTE: index is != the trackid in the MediaStream
+int LocalSourceStreamInfo::HasTrack(DOMMediaStream* aStream, TrackID aTrack)
+{
+  if (aStream != mMediaStream) {
+    return -1;
+  }
+  for (auto it = mPipelines.begin(); it != mPipelines.end(); ++it) {
+    if (it->second->trackid_locked() == aTrack) {
+      return it->first;
+    }
+  }
+  return -1;
+}
+#endif
+
+// NOTE: index is != the trackid in the MediaStream
+int LocalSourceStreamInfo::HasTrackType(DOMMediaStream* aStream, bool aIsVideo)
+{
+  if (aStream != mMediaStream) {
+    return -1;
+  }
+  for (auto it = mPipelines.begin(); it != mPipelines.end(); ++it) {
+    if (it->second->IsVideo() == aIsVideo) {
+      return it->first;
+    }
+  }
+  return -1;
+}
+
+// XXX revisit once we support multiple tracks of a type - bug 1056650
+nsresult LocalSourceStreamInfo::ReplaceTrack(int aIndex,
+                                             DOMMediaStream* aNewStream,
+                                             TrackID aNewTrack)
+{
+  // Note aIndex != aOldTrack!
+  mozilla::RefPtr<mozilla::MediaPipeline> pipeline = mPipelines[aIndex];
+  MOZ_ASSERT(pipeline);
+  if (NS_SUCCEEDED(static_cast<mozilla::MediaPipelineTransmit*>(pipeline.get())->ReplaceTrack(aNewStream, aNewTrack))) {
+    return NS_OK;
+  }
+  return NS_ERROR_FAILURE;
+}
+
 void RemoteSourceStreamInfo::DetachTransport_s()
 {
   ASSERT_ON_THREAD(mParent->GetSTSThread());
@@ -277,7 +322,7 @@ PeerConnectionMedia::AddStream(nsIDOMMediaStream* aMediaStream,
   // Now see if we already have this stream or another stream with
   // tracks of the same type, since we only allow one track of each type.
   // TODO(ekr@rtfm.com): remove this when multiple of each stream
-  // is allowed
+  // is allowed  bug 1056650
   nsRefPtr<LocalSourceStreamInfo> localSourceStream = nullptr;
 
   for (uint32_t u = 0; u < mLocalSourceStreams.Length(); u++) {
@@ -729,7 +774,7 @@ LocalSourceStreamInfo::StorePipeline(
     return;
   }
   //TODO: Revisit once we start supporting multiple streams or multiple tracks
-  // of same type
+  // of same type  bug 1056650
   mPipelines[aTrack] = aPipeline;
 }
 
@@ -746,7 +791,7 @@ RemoteSourceStreamInfo::StorePipeline(
   CSFLogDebug(logTag, "%s track %d %s = %p", __FUNCTION__, aTrack, aIsVideo ? "video" : "audio",
               aPipeline.get());
   // See if we have both audio and video here, and if so cross the streams and sync them
-  // XXX Needs to be adjusted when we support multiple streams of the same type
+  // XXX Needs to be adjusted when we support multiple streams of the same type  bug 1056650
   for (std::map<int, bool>::iterator it = mTypes.begin(); it != mTypes.end(); ++it) {
     if (it->second != aIsVideo) {
       // Ok, we have one video, one non-video - cross the streams!
@@ -764,7 +809,7 @@ RemoteSourceStreamInfo::StorePipeline(
     }
   }
   //TODO: Revisit once we start supporting multiple streams or multiple tracks
-  // of same type
+  // of same type bug 1056650
   mPipelines[aTrack] = aPipeline;
   //TODO: move to attribute on Pipeline
   mTypes[aTrack] = aIsVideo;

media/webrtc/signaling/src/peerconnection/PeerConnectionMedia.h

@@ -217,6 +217,17 @@ public:
   DOMMediaStream* GetMediaStream() {
     return mMediaStream;
   }
+  // Returns the mPipelines index for the track or -1.
+#if 0
+  int HasTrack(DOMMediaStream* aStream, mozilla::TrackID aTrack);
+#endif
+  int HasTrackType(DOMMediaStream* aStream, bool aIsVideo);
+  // XXX NOTE: does not change mMediaStream, even if it replaces the last track
+  // in a LocalSourceStreamInfo.  Revise when we have support for multiple tracks
+  // of a type.
+  // Note aIndex != aOldTrack!  It's the result of HasTrackType()
+  nsresult ReplaceTrack(int aIndex, DOMMediaStream* aNewStream, mozilla::TrackID aNewTrack);
+
   void StorePipeline(int aTrack,
                      mozilla::RefPtr<mozilla::MediaPipelineTransmit> aPipeline);
 

media/webrtc/signaling/src/sipcc/core/gsm/lsm.c

@@ -916,8 +916,12 @@ lsm_rx_start (lsm_lcb_t *lcb, const char *fname, fsmdef_media_t *media)
            have > 2 streams. (adam@nostrum.com): For now,
            we use all the same stream so pc_stream_id == 0
            and the tracks are assigned in order and are
-           equal to the level in the media objects */
+           equal to the level in the media objects  bug 1056650 */
+        /* See also ReplaceTrack in PeerConnectionMedia.cpp */
+        /* Possible solution is a map from MediaStreamTrack->GetTrackID to this, or
+           to find a way to make them match */
         pc_stream_id = 0;
+        // note: not a TrackID! (on receive, we may use it to create a TrackID though)
         pc_track_id = media->level;
 
         /*

media/webrtc/signaling/test/FakeMediaStreams.h

@@ -210,10 +210,14 @@ class Fake_SourceMediaStream : public Fake_MediaStream {
   nsCOMPtr<nsITimer> mTimer;
 };
 
+class Fake_DOMMediaStream;
+
 class Fake_MediaStreamTrack : public mozilla::RefCounted<Fake_MediaStreamTrack>
 {
 public:
   Fake_MediaStreamTrack(bool aIsVideo) : mIsVideo (aIsVideo) {}
+  mozilla::TrackID GetTrackID() { return mIsVideo ? 1 : 0; }
+  Fake_DOMMediaStream *GetStream() { return nullptr; }
   const Fake_MediaStreamTrack* AsVideoStreamTrack() const
   {
     return mIsVideo? this : nullptr;

media/webrtc/signaling/test/mediapipeline_unittest.cpp

@@ -272,6 +272,7 @@ class TestAgentSend : public TestAgent {
         audio_,
         1,
         1,
+        false,
         audio_conduit_,
         rtp,
         rtcp);