From ceac951ec3820a013debb0d1782206e52e28ebd7 Mon Sep 17 00:00:00 2001 From: Andreas Pehrson Date: Wed, 30 Sep 2015 09:32:06 +0800 Subject: [PATCH] Bug 1103188 - Break out MediaTrackListListener to an interface. r=roc Other modules than MediaTrackLists may want to receive updates on a DOMMediaStream's track set. This moves the MediaTrackListListener out of the MediaTrackList class into DOMMediaStream as a general interface. The logic for adding MediaTracks to the MediaTrackList when MediaStreamTracks are added or removed from a DOMMediaStream is moved to HTMLMediaElement as this fits the model better - HTMLMediaElement is the owner of the MediaTrackLists. --- dom/html/HTMLMediaElement.cpp | 139 ++++++++++++++++++++++++++++++- dom/html/HTMLMediaElement.h | 22 +++++ dom/media/DOMMediaStream.cpp | 149 +++++++++++----------------------- dom/media/DOMMediaStream.h | 60 ++++++++------ dom/media/MediaStreamTrack.h | 1 + dom/media/MediaTrackList.cpp | 27 ------ dom/media/MediaTrackList.h | 36 -------- 7 files changed, 239 insertions(+), 195 deletions(-) diff --git a/dom/html/HTMLMediaElement.cpp b/dom/html/HTMLMediaElement.cpp index be1ea06dfca..706da7811e7 100644 --- a/dom/html/HTMLMediaElement.cpp +++ b/dom/html/HTMLMediaElement.cpp @@ -71,6 +71,7 @@ #include "MediaSourceDecoder.h" #include "AudioStreamTrack.h" #include "VideoStreamTrack.h" +#include "MediaTrackList.h" #include "AudioChannelService.h" @@ -2105,6 +2106,7 @@ HTMLMediaElement::HTMLMediaElement(already_AddRefed& aNo mPlayingThroughTheAudioChannel(false), mDisableVideo(false), mPlayBlockedBecauseHidden(false), + mMediaStreamTrackListener(nullptr), mElementInTreeState(ELEMENT_NOT_INTREE), mHasUserInteraction(false) { @@ -3100,10 +3102,34 @@ public: mElement->NotifyMediaStreamTracksAvailable(aStream); } + private: HTMLMediaElement* mElement; }; +class HTMLMediaElement::MediaStreamTrackListener : + public DOMMediaStream::TrackListener +{ +public: + explicit MediaStreamTrackListener(HTMLMediaElement* aElement): + mElement(aElement) {} + + void NotifyTrackAdded(const nsRefPtr& aTrack) override + { + mElement->NotifyMediaStreamTrackAdded(aTrack); + } + + void NotifyTrackRemoved(const nsRefPtr& aTrack) override + { + mElement->NotifyMediaStreamTrackRemoved(aTrack); + } + +protected: + ~MediaStreamTrackListener() {} + + HTMLMediaElement* const mElement; +}; + void HTMLMediaElement::UpdateSrcMediaStreamPlaying(uint32_t aFlags) { if (!mSrcStream) { @@ -3198,14 +3224,14 @@ void HTMLMediaElement::SetupSrcMediaStreamPlayback(DOMMediaStream* aStream) UpdateSrcMediaStreamPlaying(); - // Note: we must call DisconnectTrackListListeners(...) before dropping - // mSrcStream. // If we pause this media element, track changes in the underlying stream // will continue to fire events at this element and alter its track list. // That's simpler than delaying the events, but probably confusing... - mSrcStream->ConstructMediaTracks(AudioTracks(), VideoTracks()); + ConstructMediaTracks(); mSrcStream->OnTracksAvailable(new MediaStreamTracksAvailableCallback(this)); + mMediaStreamTrackListener = new MediaStreamTrackListener(this); + mSrcStream->RegisterTrackListener(mMediaStreamTrackListener); ChangeNetworkState(nsIDOMHTMLMediaElement::NETWORK_IDLE); ChangeDelayLoadStatus(false); @@ -3220,11 +3246,116 @@ void HTMLMediaElement::EndSrcMediaStreamPlayback() UpdateSrcMediaStreamPlaying(REMOVING_SRC_STREAM); - mSrcStream->DisconnectTrackListListeners(AudioTracks(), VideoTracks()); + mSrcStream->UnregisterTrackListener(mMediaStreamTrackListener); + mMediaStreamTrackListener = nullptr; mSrcStream = nullptr; } +static already_AddRefed +CreateAudioTrack(AudioStreamTrack* aStreamTrack) +{ + nsAutoString id; + nsAutoString label; + aStreamTrack->GetId(id); + aStreamTrack->GetLabel(label); + + return MediaTrackList::CreateAudioTrack(id, NS_LITERAL_STRING("main"), + label, EmptyString(), + aStreamTrack->Enabled()); +} + +static already_AddRefed +CreateVideoTrack(VideoStreamTrack* aStreamTrack) +{ + nsAutoString id; + nsAutoString label; + aStreamTrack->GetId(id); + aStreamTrack->GetLabel(label); + + return MediaTrackList::CreateVideoTrack(id, NS_LITERAL_STRING("main"), + label, EmptyString()); +} + +void HTMLMediaElement::ConstructMediaTracks() +{ + nsTArray> tracks; + mSrcStream->GetTracks(tracks); + + int firstEnabledVideo = -1; + for (const nsRefPtr& track : tracks) { + if (track->Ended()) { + continue; + } + + if (AudioStreamTrack* t = track->AsAudioStreamTrack()) { + nsRefPtr audioTrack = CreateAudioTrack(t); + AudioTracks()->AddTrack(audioTrack); + } else if (VideoStreamTrack* t = track->AsVideoStreamTrack()) { + nsRefPtr videoTrack = CreateVideoTrack(t); + VideoTracks()->AddTrack(videoTrack); + firstEnabledVideo = (t->Enabled() && firstEnabledVideo < 0) + ? (VideoTracks()->Length() - 1) + : firstEnabledVideo; + } + } + + if (VideoTracks()->Length() > 0) { + // If media resource does not indicate a particular set of video tracks to + // enable, the one that is listed first in the element's videoTracks object + // must be selected. + int index = firstEnabledVideo >= 0 ? firstEnabledVideo : 0; + (*VideoTracks())[index]->SetEnabledInternal(true, MediaTrack::FIRE_NO_EVENTS); + } +} + +void +HTMLMediaElement::NotifyMediaStreamTrackAdded(const nsRefPtr& aTrack) +{ + MOZ_ASSERT(aTrack); + +#ifdef DEBUG + nsString id; + aTrack->GetId(id); + + LOG(LogLevel::Debug, ("%p, Adding MediaTrack with id %s", + this, NS_ConvertUTF16toUTF8(id).get())); +#endif + + if (AudioStreamTrack* t = aTrack->AsAudioStreamTrack()) { + nsRefPtr audioTrack = CreateAudioTrack(t); + AudioTracks()->AddTrack(audioTrack); + } else if (VideoStreamTrack* t = aTrack->AsVideoStreamTrack()) { + nsRefPtr videoTrack = CreateVideoTrack(t); + VideoTracks()->AddTrack(videoTrack); + } +} + +void +HTMLMediaElement::NotifyMediaStreamTrackRemoved(const nsRefPtr& aTrack) +{ + MOZ_ASSERT(aTrack); + + nsAutoString id; + aTrack->GetId(id); + + LOG(LogLevel::Debug, ("%p, Removing MediaTrack with id %s", + this, NS_ConvertUTF16toUTF8(id).get())); + + if (MediaTrack* t = AudioTracks()->GetTrackById(id)) { + AudioTracks()->RemoveTrack(t); + } else if (MediaTrack* t = VideoTracks()->GetTrackById(id)) { + VideoTracks()->RemoveTrack(t); + } else { + // XXX Uncomment this when DOMMediaStream doesn't call NotifyTrackRemoved + // multiple times for the same track, i.e., when it implements the + // "addtrack" and "removetrack" events. + // NS_ASSERTION(false, "MediaStreamTrack ended but did not exist in track lists"); + return; + } +} + + void HTMLMediaElement::ProcessMediaFragmentURI() { nsMediaFragmentURIParser parser(mLoadingSrc); diff --git a/dom/html/HTMLMediaElement.h b/dom/html/HTMLMediaElement.h index e7cd1f9f3b9..364767ce9f0 100644 --- a/dom/html/HTMLMediaElement.h +++ b/dom/html/HTMLMediaElement.h @@ -671,6 +671,7 @@ protected: class MediaLoadListener; class MediaStreamTracksAvailableCallback; + class MediaStreamTrackListener; class StreamListener; class StreamSizeListener; @@ -760,6 +761,25 @@ protected: enum { REMOVING_SRC_STREAM = 0x1 }; void UpdateSrcMediaStreamPlaying(uint32_t aFlags = 0); + /** + * If loading and playing a MediaStream, for each MediaStreamTrack in the + * MediaStream, create a corresponding AudioTrack or VideoTrack during the + * phase of resource fetching. + */ + void ConstructMediaTracks(); + + /** + * Called by our DOMMediaStream::TrackListener when a new MediaStreamTrack has + * been added to the playback stream of |mSrcStream|. + */ + void NotifyMediaStreamTrackAdded(const nsRefPtr& aTrack); + + /** + * Called by our DOMMediaStream::TrackListener when a MediaStreamTrack in + * |mSrcStream|'s playback stream has ended. + */ + void NotifyMediaStreamTrackRemoved(const nsRefPtr& aTrack); + /** * Returns an nsDOMMediaStream containing the played contents of this * element. When aFinishWhenEnded is true, when this element ends playback @@ -1429,6 +1449,8 @@ protected: nsRefPtr mVideoTrackList; + nsRefPtr mMediaStreamTrackListener; + enum ElementInTreeState { // The MediaElement is not in the DOM tree now. ELEMENT_NOT_INTREE, diff --git a/dom/media/DOMMediaStream.cpp b/dom/media/DOMMediaStream.cpp index 05e5d063d3f..48586c7302c 100644 --- a/dom/media/DOMMediaStream.cpp +++ b/dom/media/DOMMediaStream.cpp @@ -498,7 +498,7 @@ DOMMediaStream::AddTrack(MediaStreamTrack& aTrack) nsRefPtr trackPort = new TrackPort(inputPort, &aTrack, TrackPort::InputPortOwnership::OWNED); mTracks.AppendElement(trackPort.forget()); - NotifyMediaStreamTrackCreated(&aTrack); + NotifyTrackAdded(&aTrack); LOG(LogLevel::Debug, ("DOMMediaStream %p Added track %p", this, &aTrack)); } @@ -729,7 +729,7 @@ DOMMediaStream::CreateOwnDOMTrack(TrackID aTrackID, MediaSegment::Type aType) new TrackPort(mPlaybackPort, track, TrackPort::InputPortOwnership::EXTERNAL); mTracks.AppendElement(playbackTrackPort.forget()); - NotifyMediaStreamTrackCreated(track); + NotifyTrackAdded(track); return track; } @@ -788,7 +788,7 @@ DOMMediaStream::NotifyMediaStreamGraphShutdown() // to prevent leaks. mNotifiedOfMediaStreamGraphShutdown = true; mRunOnTracksAvailable.Clear(); - + mTrackListeners.Clear(); mConsumersToKeepAlive.Clear(); } @@ -812,7 +812,7 @@ DOMMediaStream::OnTracksAvailable(OnTracksAvailableCallback* aRunnable) } void -DOMMediaStream::TracksCreated() +DOMMediaStream::NotifyTracksCreated() { mTracksCreated = true; CheckTracksAvailable(); @@ -832,6 +832,49 @@ DOMMediaStream::CheckTracksAvailable() } } +void +DOMMediaStream::RegisterTrackListener(TrackListener* aListener) +{ + MOZ_ASSERT(NS_IsMainThread()); + + if (mNotifiedOfMediaStreamGraphShutdown) { + // No more tracks will ever be added, so just do nothing. + return; + } + mTrackListeners.AppendElement(aListener); +} + +void +DOMMediaStream::UnregisterTrackListener(TrackListener* aListener) +{ + MOZ_ASSERT(NS_IsMainThread()); + mTrackListeners.RemoveElement(aListener); +} + +void +DOMMediaStream::NotifyTrackAdded( + const nsRefPtr& aTrack) +{ + MOZ_ASSERT(NS_IsMainThread()); + + for (int32_t i = mTrackListeners.Length() - 1; i >= 0; --i) { + const nsRefPtr& listener = mTrackListeners[i]; + listener->NotifyTrackAdded(aTrack); + } +} + +void +DOMMediaStream::NotifyTrackRemoved( + const nsRefPtr& aTrack) +{ + MOZ_ASSERT(NS_IsMainThread()); + + for (int32_t i = mTrackListeners.Length() - 1; i >= 0; --i) { + const nsRefPtr& listener = mTrackListeners[i]; + listener->NotifyTrackRemoved(aTrack); + } +} + void DOMMediaStream::CreateAndAddPlaybackStreamListener(MediaStream* aStream) { @@ -840,104 +883,6 @@ DOMMediaStream::CreateAndAddPlaybackStreamListener(MediaStream* aStream) aStream->AddListener(mPlaybackListener); } -already_AddRefed -DOMMediaStream::CreateAudioTrack(AudioStreamTrack* aStreamTrack) -{ - nsAutoString id; - nsAutoString label; - aStreamTrack->GetId(id); - aStreamTrack->GetLabel(label); - - return MediaTrackList::CreateAudioTrack(id, NS_LITERAL_STRING("main"), - label, EmptyString(), - aStreamTrack->Enabled()); -} - -already_AddRefed -DOMMediaStream::CreateVideoTrack(VideoStreamTrack* aStreamTrack) -{ - nsAutoString id; - nsAutoString label; - aStreamTrack->GetId(id); - aStreamTrack->GetLabel(label); - - return MediaTrackList::CreateVideoTrack(id, NS_LITERAL_STRING("main"), - label, EmptyString()); -} - -void -DOMMediaStream::ConstructMediaTracks(AudioTrackList* aAudioTrackList, - VideoTrackList* aVideoTrackList) -{ - MediaTrackListListener audioListener(aAudioTrackList); - mMediaTrackListListeners.AppendElement(audioListener); - MediaTrackListListener videoListener(aVideoTrackList); - mMediaTrackListListeners.AppendElement(videoListener); - - int firstEnabledVideo = -1; - for (const nsRefPtr& info : mTracks) { - if (AudioStreamTrack* t = info->GetTrack()->AsAudioStreamTrack()) { - nsRefPtr track = CreateAudioTrack(t); - aAudioTrackList->AddTrack(track); - } else if (VideoStreamTrack* t = info->GetTrack()->AsVideoStreamTrack()) { - nsRefPtr track = CreateVideoTrack(t); - aVideoTrackList->AddTrack(track); - firstEnabledVideo = (t->Enabled() && firstEnabledVideo < 0) - ? (aVideoTrackList->Length() - 1) - : firstEnabledVideo; - } - } - - if (aVideoTrackList->Length() > 0) { - // If media resource does not indicate a particular set of video tracks to - // enable, the one that is listed first in the element's videoTracks object - // must be selected. - int index = firstEnabledVideo >= 0 ? firstEnabledVideo : 0; - (*aVideoTrackList)[index]->SetEnabledInternal(true, MediaTrack::FIRE_NO_EVENTS); - } -} - -void -DOMMediaStream::DisconnectTrackListListeners(const AudioTrackList* aAudioTrackList, - const VideoTrackList* aVideoTrackList) -{ - for (auto i = mMediaTrackListListeners.Length(); i > 0; ) { // unsigned! - --i; // 0 ... Length()-1 range - if (mMediaTrackListListeners[i].mMediaTrackList == aAudioTrackList || - mMediaTrackListListeners[i].mMediaTrackList == aVideoTrackList) { - mMediaTrackListListeners.RemoveElementAt(i); - } - } -} - -void -DOMMediaStream::NotifyMediaStreamTrackCreated(MediaStreamTrack* aTrack) -{ - MOZ_ASSERT(aTrack); - - for (uint32_t i = 0; i < mMediaTrackListListeners.Length(); ++i) { - if (AudioStreamTrack* t = aTrack->AsAudioStreamTrack()) { - nsRefPtr track = CreateAudioTrack(t); - mMediaTrackListListeners[i].NotifyMediaTrackCreated(track); - } else if (VideoStreamTrack* t = aTrack->AsVideoStreamTrack()) { - nsRefPtr track = CreateVideoTrack(t); - mMediaTrackListListeners[i].NotifyMediaTrackCreated(track); - } - } -} - -void -DOMMediaStream::NotifyMediaStreamTrackEnded(MediaStreamTrack* aTrack) -{ - MOZ_ASSERT(aTrack); - - nsAutoString id; - aTrack->GetId(id); - for (uint32_t i = 0; i < mMediaTrackListListeners.Length(); ++i) { - mMediaTrackListListeners[i].NotifyMediaTrackEnded(id); - } -} - DOMLocalMediaStream::~DOMLocalMediaStream() { if (mInputStream) { diff --git a/dom/media/DOMMediaStream.h b/dom/media/DOMMediaStream.h index 9d09d49a517..6cc6e789b78 100644 --- a/dom/media/DOMMediaStream.h +++ b/dom/media/DOMMediaStream.h @@ -182,11 +182,31 @@ class DOMMediaStream : public DOMEventTargetHelper typedef dom::VideoTrack VideoTrack; typedef dom::AudioTrackList AudioTrackList; typedef dom::VideoTrackList VideoTrackList; - typedef dom::MediaTrackListListener MediaTrackListListener; public: typedef dom::MediaTrackConstraints MediaTrackConstraints; - typedef uint8_t TrackTypeHints; + + class TrackListener { + NS_INLINE_DECL_REFCOUNTING(TrackListener) + + public: + /** + * Called when the DOMMediaStream has a new track added, either by + * JS (addTrack()) or the source creating one. + */ + virtual void + NotifyTrackAdded(const nsRefPtr& aTrack) {}; + + /** + * Called when the DOMMediaStream removes a track, either by + * JS (removeTrack()) or the source ending it. + */ + virtual void + NotifyTrackRemoved(const nsRefPtr& aTrack) {}; + + protected: + virtual ~TrackListener() {} + }; DOMMediaStream(); @@ -394,23 +414,8 @@ public: } } - /** - * If loading and playing a MediaStream in a media element, for each - * MediaStreamTrack in the MediaStream, create a corresponding AudioTrack or - * VideoTrack during the phase of resource fetching. - */ - void ConstructMediaTracks(AudioTrackList* aAudioTrackList, - VideoTrackList* aVideoTrackList); - - /** - * MUST call this before the AudioTrackList or VideoTrackList go away - */ - void DisconnectTrackListListeners(const AudioTrackList* aAudioTrackList, - const VideoTrackList* aVideoTrackList); - - virtual void NotifyMediaStreamTrackCreated(MediaStreamTrack* aTrack); - - virtual void NotifyMediaStreamTrackEnded(MediaStreamTrack* aTrack); + void RegisterTrackListener(TrackListener* aListener); + void UnregisterTrackListener(TrackListener* aListener); protected: virtual ~DOMMediaStream(); @@ -420,14 +425,18 @@ protected: void InitTrackUnionStream(nsIDOMWindow* aWindow, MediaStreamGraph* aGraph); void InitAudioCaptureStream(nsIDOMWindow* aWindow, MediaStreamGraph* aGraph); void InitStreamCommon(MediaStream* aStream, MediaStreamGraph* aGraph); - already_AddRefed CreateAudioTrack(AudioStreamTrack* aStreamTrack); - already_AddRefed CreateVideoTrack(VideoStreamTrack* aStreamTrack); + + void CheckTracksAvailable(); // Called when MediaStreamGraph has finished an iteration where tracks were // created. - void TracksCreated(); + void NotifyTracksCreated(); - void CheckTracksAvailable(); + // Dispatches NotifyTrackAdded() to all registered track listeners. + void NotifyTrackAdded(const nsRefPtr& aTrack); + + // Dispatches NotifyTrackRemoved() to all registered track listeners. + void NotifyTrackRemoved(const nsRefPtr& aTrack); class OwnedStreamListener; friend class OwnedStreamListener; @@ -488,9 +497,8 @@ protected: bool mNotifiedOfMediaStreamGraphShutdown; - // Send notifications to AudioTrackList or VideoTrackList, if this MediaStream - // is consumed by a HTMLMediaElement. - nsTArray mMediaTrackListListeners; + // The track listeners subscribe to changes in this stream's track set. + nsTArray> mTrackListeners; private: void NotifyPrincipalChanged(); diff --git a/dom/media/MediaStreamTrack.h b/dom/media/MediaStreamTrack.h index 89a1a6ad53a..bd31f5d7b38 100644 --- a/dom/media/MediaStreamTrack.h +++ b/dom/media/MediaStreamTrack.h @@ -61,6 +61,7 @@ public: already_AddRefed ApplyConstraints(const dom::MediaTrackConstraints& aConstraints, ErrorResult &aRv); + bool Ended() const { return mEnded; } // Notifications from the MediaStreamGraph void NotifyEnded() { mEnded = true; } diff --git a/dom/media/MediaTrackList.cpp b/dom/media/MediaTrackList.cpp index d6efb814be2..7247a5f7ecb 100644 --- a/dom/media/MediaTrackList.cpp +++ b/dom/media/MediaTrackList.cpp @@ -16,33 +16,6 @@ namespace mozilla { namespace dom { -void -MediaTrackListListener::NotifyMediaTrackCreated(MediaTrack* aTrack) -{ - if (!mMediaTrackList && !aTrack) { - return; - } - - if (aTrack->AsAudioTrack() && mMediaTrackList->AsAudioTrackList()) { - mMediaTrackList->AddTrack(aTrack); - } else if (aTrack->AsVideoTrack() && mMediaTrackList->AsVideoTrackList()) { - mMediaTrackList->AddTrack(aTrack); - } -} - -void -MediaTrackListListener::NotifyMediaTrackEnded(const nsAString& aId) -{ - if (!mMediaTrackList) { - return; - } - - const nsRefPtr track = mMediaTrackList->GetTrackById(aId); - if (track) { - mMediaTrackList->RemoveTrack(track); - } -} - MediaTrackList::MediaTrackList(nsPIDOMWindow* aOwnerWindow, HTMLMediaElement* aMediaElement) : DOMEventTargetHelper(aOwnerWindow) diff --git a/dom/media/MediaTrackList.h b/dom/media/MediaTrackList.h index e7386f0b094..ab99a98e2be 100644 --- a/dom/media/MediaTrackList.h +++ b/dom/media/MediaTrackList.h @@ -20,41 +20,6 @@ class AudioTrackList; class VideoTrackList; class AudioTrack; class VideoTrack; -class MediaTrackList; - -/** - * This is for the media resource to notify its audio track and video track, - * when a media-resource-specific track has ended, or whether it has enabled or - * not. All notification methods are called from the main thread. - */ -class MediaTrackListListener -{ -public: - friend class mozilla::DOMMediaStream; - - explicit MediaTrackListListener(MediaTrackList* aMediaTrackList) - : mMediaTrackList(aMediaTrackList) {}; - - ~MediaTrackListListener() - { - mMediaTrackList = nullptr; - }; - - // Notify mMediaTrackList that a track has created by the media resource, - // and this corresponding MediaTrack object should be added into - // mMediaTrackList, and fires a addtrack event. - void NotifyMediaTrackCreated(MediaTrack* aTrack); - - // Notify mMediaTrackList that a track has ended by the media resource, - // and this corresponding MediaTrack object should be removed from - // mMediaTrackList, and fires a removetrack event. - void NotifyMediaTrackEnded(const nsAString& aId); - -protected: - // A weak reference to a MediaTrackList object, its lifetime managed by its - // owner. - MediaTrackList* mMediaTrackList; -}; /** * Base class of AudioTrackList and VideoTrackList. The AudioTrackList and @@ -120,7 +85,6 @@ public: IMPL_EVENT_HANDLER(addtrack) IMPL_EVENT_HANDLER(removetrack) - friend class MediaTrackListListener; friend class AudioTrack; friend class VideoTrack;