/* -*- mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ /* vim: set ts=8 sts=2 et sw=2 tw=80: */ /* This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ #include "MediaSourceResource.h" #include "MediaSourceDecoder.h" #include "AbstractMediaDecoder.h" #include "MediaDecoderReader.h" #include "MediaDecoderStateMachine.h" #include "mozilla/Assertions.h" #include "mozilla/FloatingPoint.h" #include "mozilla/dom/HTMLMediaElement.h" #include "mozilla/dom/TimeRanges.h" #include "mozilla/mozalloc.h" #include "nsISupports.h" #include "nsIThread.h" #include "prlog.h" #include "MediaSource.h" #include "SubBufferDecoder.h" #include "SourceBufferResource.h" #include "VideoUtils.h" #ifdef PR_LOGGING extern PRLogModuleInfo* gMediaSourceLog; #define MSE_DEBUG(...) PR_LOG(gMediaSourceLog, PR_LOG_DEBUG, (__VA_ARGS__)) #else #define MSE_DEBUG(...) #endif namespace mozilla { namespace dom { class TimeRanges; } // namespace dom class MediaSourceReader : public MediaDecoderReader { public: MediaSourceReader(MediaSourceDecoder* aDecoder) : MediaDecoderReader(aDecoder) , mActiveVideoDecoder(-1) , mActiveAudioDecoder(-1) { } nsresult Init(MediaDecoderReader* aCloneDonor) MOZ_OVERRIDE { // Although we technically don't implement anything here, we return NS_OK // so that when the state machine initializes and calls this function // we don't return an error code back to the media element. return NS_OK; } bool IsWaitingMediaResources() MOZ_OVERRIDE { return mDecoders.IsEmpty() && mPendingDecoders.IsEmpty(); } bool DecodeAudioData() MOZ_OVERRIDE { if (!GetAudioReader()) { MSE_DEBUG("%p DecodeAudioFrame called with no audio reader", this); MOZ_ASSERT(mPendingDecoders.IsEmpty()); return false; } bool rv = GetAudioReader()->DecodeAudioData(); nsAutoTArray audio; GetAudioReader()->AudioQueue().GetElementsAfter(-1, &audio); for (uint32_t i = 0; i < audio.Length(); ++i) { AudioQueue().Push(audio[i]); } GetAudioReader()->AudioQueue().Empty(); return rv; } bool DecodeVideoFrame(bool& aKeyFrameSkip, int64_t aTimeThreshold) MOZ_OVERRIDE { if (!GetVideoReader()) { MSE_DEBUG("%p DecodeVideoFrame called with no video reader", this); MOZ_ASSERT(mPendingDecoders.IsEmpty()); return false; } bool rv = GetVideoReader()->DecodeVideoFrame(aKeyFrameSkip, aTimeThreshold); nsAutoTArray video; GetVideoReader()->VideoQueue().GetElementsAfter(-1, &video); for (uint32_t i = 0; i < video.Length(); ++i) { VideoQueue().Push(video[i]); } GetVideoReader()->VideoQueue().Empty(); if (rv) { return true; } MSE_DEBUG("%p MSR::DecodeVF %d (%p) returned false (readers=%u)", this, mActiveVideoDecoder, mDecoders[mActiveVideoDecoder].get(), mDecoders.Length()); if (SwitchVideoReaders(aTimeThreshold)) { rv = GetVideoReader()->DecodeVideoFrame(aKeyFrameSkip, aTimeThreshold); nsAutoTArray video; GetVideoReader()->VideoQueue().GetElementsAfter(-1, &video); for (uint32_t i = 0; i < video.Length(); ++i) { VideoQueue().Push(video[i]); } GetVideoReader()->VideoQueue().Empty(); } return rv; } bool HasVideo() MOZ_OVERRIDE { return mInfo.HasVideo(); } bool HasAudio() MOZ_OVERRIDE { return mInfo.HasAudio(); } nsresult ReadMetadata(MediaInfo* aInfo, MetadataTags** aTags) MOZ_OVERRIDE; nsresult Seek(int64_t aTime, int64_t aStartTime, int64_t aEndTime, int64_t aCurrentTime) MOZ_OVERRIDE { return NS_ERROR_NOT_IMPLEMENTED; } nsresult GetBuffered(dom::TimeRanges* aBuffered, int64_t aStartTime) MOZ_OVERRIDE { for (uint32_t i = 0; i < mDecoders.Length(); ++i) { nsRefPtr r = new dom::TimeRanges(); mDecoders[i]->GetBuffered(r); aBuffered->Add(r->GetStartTime(), r->GetEndTime()); } aBuffered->Normalize(); return NS_OK; } already_AddRefed CreateSubDecoder(const nsACString& aType, MediaSourceDecoder* aParentDecoder); void CallDecoderInitialization(); private: bool SwitchVideoReaders(int64_t aTimeThreshold) { MOZ_ASSERT(mActiveVideoDecoder != -1); // XXX: We switch when the first reader is depleted, but it might be // better to switch as soon as the next reader is ready to decode and // has data for the current media time. ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor()); GetVideoReader()->SetIdle(); WaitForPendingDecoders(); for (uint32_t i = mActiveVideoDecoder + 1; i < mDecoders.Length(); ++i) { if (!mDecoders[i]->GetReader()->GetMediaInfo().HasVideo()) { continue; } mActiveVideoDecoder = i; MSE_DEBUG("%p MSR::DecodeVF switching to %d", this, mActiveVideoDecoder); GetVideoReader()->SetActive(); GetVideoReader()->DecodeToTarget(aTimeThreshold); return true; } return false; } MediaDecoderReader* GetAudioReader() { if (mActiveAudioDecoder == -1) { return nullptr; } return mDecoders[mActiveAudioDecoder]->GetReader(); } MediaDecoderReader* GetVideoReader() { if (mActiveVideoDecoder == -1) { return nullptr; } return mDecoders[mActiveVideoDecoder]->GetReader(); } bool EnsureWorkQueueInitialized(); nsresult EnqueueDecoderInitialization(); void WaitForPendingDecoders(); nsTArray> mPendingDecoders; nsTArray> mDecoders; int32_t mActiveVideoDecoder; int32_t mActiveAudioDecoder; nsCOMPtr mWorkQueue; }; class MediaSourceStateMachine : public MediaDecoderStateMachine { public: MediaSourceStateMachine(MediaDecoder* aDecoder, MediaDecoderReader* aReader, bool aRealTime = false) : MediaDecoderStateMachine(aDecoder, aReader, aRealTime) { } already_AddRefed CreateSubDecoder(const nsACString& aType, MediaSourceDecoder* aParentDecoder) { return static_cast(mReader.get())->CreateSubDecoder(aType, aParentDecoder); } void CallDecoderInitialization() { return static_cast(mReader.get())->CallDecoderInitialization(); } }; MediaSourceDecoder::MediaSourceDecoder(dom::HTMLMediaElement* aElement) : mMediaSource(nullptr) { Init(aElement); } MediaDecoder* MediaSourceDecoder::Clone() { // TODO: Sort out cloning. return nullptr; } MediaDecoderStateMachine* MediaSourceDecoder::CreateStateMachine() { return new MediaSourceStateMachine(this, new MediaSourceReader(this)); } nsresult MediaSourceDecoder::Load(nsIStreamListener**, MediaDecoder*) { MOZ_ASSERT(!mDecoderStateMachine); mDecoderStateMachine = CreateStateMachine(); if (!mDecoderStateMachine) { NS_WARNING("Failed to create state machine!"); return NS_ERROR_FAILURE; } return mDecoderStateMachine->Init(nullptr); } nsresult MediaSourceDecoder::GetSeekable(dom::TimeRanges* aSeekable) { double duration = mMediaSource->Duration(); if (IsNaN(duration)) { // Return empty range. } else if (duration > 0 && mozilla::IsInfinite(duration)) { nsRefPtr bufferedRanges = new dom::TimeRanges(); GetBuffered(bufferedRanges); aSeekable->Add(bufferedRanges->GetStartTime(), bufferedRanges->GetEndTime()); } else { aSeekable->Add(0, duration); } return NS_OK; } /*static*/ already_AddRefed MediaSourceDecoder::CreateResource() { return nsRefPtr(new MediaSourceResource()).forget(); } void MediaSourceDecoder::AttachMediaSource(dom::MediaSource* aMediaSource) { MOZ_ASSERT(!mMediaSource && !mDecoderStateMachine); mMediaSource = aMediaSource; } void MediaSourceDecoder::DetachMediaSource() { mMediaSource = nullptr; } already_AddRefed MediaSourceDecoder::CreateSubDecoder(const nsACString& aType) { return static_cast(mDecoderStateMachine.get())->CreateSubDecoder(aType, this); } void MediaSourceDecoder::CallDecoderInitialization() { return static_cast(mDecoderStateMachine.get())->CallDecoderInitialization(); } bool MediaSourceReader::EnsureWorkQueueInitialized() { // TODO: Use a global threadpool rather than a thread per MediaSource. MOZ_ASSERT(NS_IsMainThread()); if (!mWorkQueue && NS_FAILED(NS_NewNamedThread("MediaSource", getter_AddRefs(mWorkQueue), nullptr, MEDIA_THREAD_STACK_SIZE))) { return false; } return true; } nsresult MediaSourceReader::EnqueueDecoderInitialization() { if (!EnsureWorkQueueInitialized()) { return NS_ERROR_FAILURE; } return mWorkQueue->Dispatch(NS_NewRunnableMethod(static_cast(mDecoder), &MediaSourceDecoder::CallDecoderInitialization), NS_DISPATCH_NORMAL); } void MediaSourceReader::CallDecoderInitialization() { ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor()); for (uint32_t i = 0; i < mPendingDecoders.Length(); ++i) { nsRefPtr decoder = mPendingDecoders[i]; MediaDecoderReader* reader = decoder->GetReader(); MSE_DEBUG("%p: Initializating subdecoder %p reader %p", this, decoder.get(), reader); reader->SetActive(); MediaInfo mi; nsAutoPtr tags; // TODO: Handle metadata. nsresult rv; { ReentrantMonitorAutoExit exitMon(mDecoder->GetReentrantMonitor()); rv = reader->ReadMetadata(&mi, getter_Transfers(tags)); } reader->SetIdle(); if (NS_FAILED(rv)) { // XXX: Need to signal error back to owning SourceBuffer. MSE_DEBUG("%p: Reader %p failed to initialize, rv=%x", this, reader, rv); continue; } bool active = false; if (mi.HasVideo() || mi.HasAudio()) { MSE_DEBUG("%p: Reader %p has video=%d audio=%d", this, reader, mi.HasVideo(), mi.HasAudio()); active = true; } if (active) { mDecoders.AppendElement(decoder); } else { MSE_DEBUG("%p: Reader %p not activated", this, reader); } } mPendingDecoders.Clear(); mDecoder->NotifyWaitingForResourcesStatusChanged(); mon.NotifyAll(); } void MediaSourceReader::WaitForPendingDecoders() { ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor()); while (!mPendingDecoders.IsEmpty()) { mon.Wait(); } } already_AddRefed MediaSourceReader::CreateSubDecoder(const nsACString& aType, MediaSourceDecoder* aParentDecoder) { // XXX: Why/when is mDecoder null here, since it should be equal to aParentDecoder?! nsRefPtr decoder = new SubBufferDecoder(new SourceBufferResource(nullptr, aType), aParentDecoder); nsAutoPtr reader(DecoderTraits::CreateReader(aType, decoder)); if (!reader) { return nullptr; } reader->Init(nullptr); ReentrantMonitorAutoEnter mon(aParentDecoder->GetReentrantMonitor()); MSE_DEBUG("Registered subdecoder %p subreader %p", decoder.get(), reader.get()); decoder->SetReader(reader.forget()); mPendingDecoders.AppendElement(decoder); EnqueueDecoderInitialization(); mDecoder->NotifyWaitingForResourcesStatusChanged(); return decoder.forget(); } nsresult MediaSourceReader::ReadMetadata(MediaInfo* aInfo, MetadataTags** aTags) { mDecoder->SetMediaSeekable(true); mDecoder->SetTransportSeekable(false); MSE_DEBUG("%p: MSR::ReadMetadata pending=%u", this, mPendingDecoders.Length()); WaitForPendingDecoders(); MSE_DEBUG("%p: MSR::ReadMetadata decoders=%u", this, mDecoders.Length()); // XXX: Make subdecoder setup async, so that use cases like bug 989888 can // work. This will require teaching the state machine about dynamic track // changes (and multiple tracks). // Shorter term, make this block until we've got at least one video track // and lie about having an audio track, then resample/remix as necessary // to match any audio track added later to fit the format we lied about // now. For now we just configure what we've got and cross our fingers. int64_t maxDuration = -1; for (uint32_t i = 0; i < mDecoders.Length(); ++i) { MediaDecoderReader* reader = mDecoders[i]->GetReader(); reader->SetActive(); // XXX check where this should be called MediaInfo mi = reader->GetMediaInfo(); if (mi.HasVideo() && !mInfo.HasVideo()) { MOZ_ASSERT(mActiveVideoDecoder == -1); mActiveVideoDecoder = i; mInfo.mVideo = mi.mVideo; maxDuration = std::max(maxDuration, mDecoders[i]->GetMediaDuration()); MSE_DEBUG("%p: MSR::ReadMetadata video decoder=%u maxDuration=%lld", this, i, maxDuration); } if (mi.HasAudio() && !mInfo.HasAudio()) { MOZ_ASSERT(mActiveAudioDecoder == -1); mActiveAudioDecoder = i; mInfo.mAudio = mi.mAudio; maxDuration = std::max(maxDuration, mDecoders[i]->GetMediaDuration()); MSE_DEBUG("%p: MSR::ReadMetadata audio decoder=%u maxDuration=%lld", this, i, maxDuration); } } *aInfo = mInfo; if (maxDuration != -1) { ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor()); mDecoder->SetMediaDuration(maxDuration); } return NS_OK; } } // namespace mozilla