diff --git a/content/media/webaudio/AudioBufferSourceNode.cpp b/content/media/webaudio/AudioBufferSourceNode.cpp index dc9f2f2d953..1d8aa78a692 100644 --- a/content/media/webaudio/AudioBufferSourceNode.cpp +++ b/content/media/webaudio/AudioBufferSourceNode.cpp @@ -10,13 +10,24 @@ #include "AudioNodeEngine.h" #include "AudioNodeStream.h" #include "AudioDestinationNode.h" +#include "PannerNode.h" #include "speex/speex_resampler.h" namespace mozilla { namespace dom { -NS_IMPL_CYCLE_COLLECTION_INHERITED_2(AudioBufferSourceNode, AudioNode, - mBuffer, mPlaybackRate) +NS_IMPL_CYCLE_COLLECTION_UNLINK_BEGIN_INHERITED(AudioBufferSourceNode, AudioNode) + NS_IMPL_CYCLE_COLLECTION_UNLINK(mBuffer) + NS_IMPL_CYCLE_COLLECTION_UNLINK(mPlaybackRate) + if (tmp->Context()) { + tmp->Context()->UnregisterAudioBufferSourceNode(tmp); + } +NS_IMPL_CYCLE_COLLECTION_UNLINK_END + +NS_IMPL_CYCLE_COLLECTION_TRAVERSE_BEGIN_INHERITED(AudioBufferSourceNode, AudioNode) + NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mBuffer) + NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mPlaybackRate) +NS_IMPL_CYCLE_COLLECTION_TRAVERSE_END NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION_INHERITED(AudioBufferSourceNode) NS_INTERFACE_MAP_END_INHERITING(AudioNode) @@ -33,6 +44,7 @@ public: mOffset(0), mDuration(0), mLoopStart(0), mLoopEnd(0), mSampleRate(0), mPosition(0), mChannels(0), mPlaybackRate(1.0f), + mDopplerShift(1.0f), mDestination(static_cast(aDestination->Stream())), mPlaybackRateTimeline(1.0f), mLoop(false) {} @@ -56,7 +68,8 @@ public: LOOP, LOOPSTART, LOOPEND, - PLAYBACKRATE + PLAYBACKRATE, + DOPPLERSHIFT }; virtual void SetTimelineParameter(uint32_t aIndex, const dom::AudioParamTimeline& aValue) { @@ -87,6 +100,16 @@ public: NS_ERROR("Bad AudioBufferSourceNodeEngine StreamTimeParameter"); } } + virtual void SetDoubleParameter(uint32_t aIndex, double aParam) + { + switch (aIndex) { + case DOPPLERSHIFT: + mDopplerShift = aParam; + break; + default: + NS_ERROR("Bad AudioBufferSourceNodeEngine double parameter."); + }; + } virtual void SetInt32Parameter(uint32_t aIndex, int32_t aParam) { switch (aIndex) { @@ -115,7 +138,7 @@ public: if (!mResampler) { mChannels = aChannels; mResampler = speex_resampler_init(mChannels, mSampleRate, - IdealAudioRate(), + ComputeFinalOutSampleRate(), SPEEX_RESAMPLER_QUALITY_DEFAULT, nullptr); } @@ -165,9 +188,7 @@ public: uint32_t aAvailableInInputBuffer, uint32_t& aFramesRead, uint32_t& aFramesWritten) { - // Compute the sample rate we want to resample to. - double finalSampleRate = mSampleRate / mPlaybackRate; - double finalPlaybackRate = finalSampleRate / IdealAudioRate(); + double finalPlaybackRate = static_cast(mSampleRate) / ComputeFinalOutSampleRate(); uint32_t availableInOuputBuffer = WEBAUDIO_BLOCK_SIZE - aBufferOffset; uint32_t inputSamples, outputSamples; @@ -247,9 +268,7 @@ public: uint32_t numFrames = std::min(std::min(WEBAUDIO_BLOCK_SIZE - *aOffsetWithinBlock, aBufferMax - aBufferOffset), uint32_t(mStop - *aCurrentPosition)); - if (numFrames == WEBAUDIO_BLOCK_SIZE && - mSampleRate == IdealAudioRate() && - mPlaybackRate == 1.0f) { + if (numFrames == WEBAUDIO_BLOCK_SIZE && !ShouldResample()) { BorrowFromInputBuffer(aOutput, aChannels, aBufferOffset); *aOffsetWithinBlock += numFrames; *aCurrentPosition += numFrames; @@ -259,7 +278,7 @@ public: MOZ_ASSERT(*aOffsetWithinBlock == 0); AllocateAudioBlock(aChannels, aOutput); } - if (mSampleRate == IdealAudioRate() && mPlaybackRate == 1.0f) { + if (!ShouldResample()) { CopyFromInputBuffer(aOutput, aChannels, aBufferOffset, *aOffsetWithinBlock, numFrames); *aOffsetWithinBlock += numFrames; *aCurrentPosition += numFrames; @@ -285,6 +304,37 @@ public: return mStart + mPosition; } + int32_t ComputeFinalOutSampleRate() const + { + return static_cast(IdealAudioRate() / (mPlaybackRate * mDopplerShift)); + } + + bool ShouldResample() const + { + return !(mPlaybackRate == 1.0 && + mDopplerShift == 1.0 && + mSampleRate == IdealAudioRate()); + } + + void UpdateSampleRateIfNeeded(AudioNodeStream* aStream) + { + if (mPlaybackRateTimeline.HasSimpleValue()) { + mPlaybackRate = mPlaybackRateTimeline.GetValue(); + } else { + mPlaybackRate = mPlaybackRateTimeline.GetValueAtTime(aStream->GetCurrentPosition()); + } + + uint32_t currentOutSampleRate, currentInSampleRate; + if (ShouldResample()) { + SpeexResamplerState* resampler = Resampler(mChannels); + speex_resampler_get_rate(resampler, ¤tInSampleRate, ¤tOutSampleRate); + uint32_t finalSampleRate = ComputeFinalOutSampleRate(); + if (currentOutSampleRate != finalSampleRate) { + speex_resampler_set_rate(resampler, currentInSampleRate, finalSampleRate); + } + } + } + virtual void ProduceAudioBlock(AudioNodeStream* aStream, const AudioChunk& aInput, AudioChunk* aOutput, @@ -302,16 +352,7 @@ public: // WebKit treats the playbackRate as a k-rate parameter in their code, // despite the spec saying that it should be an a-rate parameter. We treat // it as k-rate. Spec bug: https://www.w3.org/Bugs/Public/show_bug.cgi?id=21592 - float newPlaybackRate; - if (mPlaybackRateTimeline.HasSimpleValue()) { - newPlaybackRate = mPlaybackRateTimeline.GetValue(); - } else { - newPlaybackRate = mPlaybackRateTimeline.GetValueAtTime(aStream->GetCurrentPosition()); - } - if (newPlaybackRate != mPlaybackRate) { - mPlaybackRate = newPlaybackRate; - speex_resampler_set_rate(Resampler(mChannels), mSampleRate, mSampleRate / mPlaybackRate); - } + UpdateSampleRateIfNeeded(aStream); uint32_t written = 0; TrackTicks currentPosition = GetPosition(aStream); @@ -362,6 +403,7 @@ public: uint32_t mPosition; uint32_t mChannels; float mPlaybackRate; + float mDopplerShift; AudioNodeStream* mDestination; AudioParamTimeline mPlaybackRateTimeline; bool mLoop; @@ -374,6 +416,7 @@ AudioBufferSourceNode::AudioBufferSourceNode(AudioContext* aContext) , mLoop(false) , mStartCalled(false) , mPlaybackRate(new AudioParam(this, SendPlaybackRateToStream, 1.0f)) + , mPannerNode(nullptr) { SetProduceOwnOutput(true); mStream = aContext->Graph()->CreateAudioNodeStream( @@ -384,6 +427,10 @@ AudioBufferSourceNode::AudioBufferSourceNode(AudioContext* aContext) AudioBufferSourceNode::~AudioBufferSourceNode() { + // + if (Context()) { + Context()->UnregisterAudioBufferSourceNode(this); + } DestroyMediaStream(); } @@ -492,5 +539,11 @@ AudioBufferSourceNode::SendPlaybackRateToStream(AudioNode* aNode) SendTimelineParameterToStream(This, AudioBufferSourceNodeEngine::PLAYBACKRATE, *This->mPlaybackRate); } +void +AudioBufferSourceNode::SendDopplerShiftToStream(double aDopplerShift) +{ + SendDoubleParameterToStream(AudioBufferSourceNodeEngine::DOPPLERSHIFT, aDopplerShift); +} + } } diff --git a/content/media/webaudio/AudioBufferSourceNode.h b/content/media/webaudio/AudioBufferSourceNode.h index 5d7a0c20d8c..18c041de7f6 100644 --- a/content/media/webaudio/AudioBufferSourceNode.h +++ b/content/media/webaudio/AudioBufferSourceNode.h @@ -9,6 +9,7 @@ #include "AudioNode.h" #include "AudioBuffer.h" +#include "AudioParam.h" #include "mozilla/dom/BindingUtils.h" namespace mozilla { @@ -36,6 +37,18 @@ public: { return 0; } + virtual AudioBufferSourceNode* AsAudioBufferSourceNode() MOZ_OVERRIDE + { + return this; + } + + void UnregisterPannerNode() { + mPannerNode = nullptr; + } + + void RegisterPannerNode(PannerNode* aPannerNode) { + mPannerNode = aPannerNode; + } void JSBindingFinalized() { @@ -107,6 +120,7 @@ public: { mLoopEnd = aEnd; } + void SendDopplerShiftToStream(double aDopplerShift); virtual void NotifyMainThreadStateChanged() MOZ_OVERRIDE; @@ -118,6 +132,7 @@ private: bool mLoop; bool mStartCalled; nsRefPtr mPlaybackRate; + PannerNode* mPannerNode; }; } diff --git a/content/media/webaudio/AudioContext.cpp b/content/media/webaudio/AudioContext.cpp index 18e8983d5e4..09cbc57cae2 100644 --- a/content/media/webaudio/AudioContext.cpp +++ b/content/media/webaudio/AudioContext.cpp @@ -71,6 +71,7 @@ AudioContext::CreateBufferSource() { nsRefPtr bufferNode = new AudioBufferSourceNode(this); + mAudioBufferSourceNodes.AppendElement(bufferNode); return bufferNode.forget(); } @@ -128,6 +129,7 @@ already_AddRefed AudioContext::CreatePanner() { nsRefPtr pannerNode = new PannerNode(this); + mPannerNodes.AppendElement(pannerNode); return pannerNode.forget(); } @@ -187,6 +189,29 @@ AudioContext::RemoveFromDecodeQueue(WebAudioDecodeJob* aDecodeJob) mDecodeJobs.RemoveElement(aDecodeJob); } +void +AudioContext::UnregisterAudioBufferSourceNode(AudioBufferSourceNode* aNode) +{ + mAudioBufferSourceNodes.RemoveElement(aNode); +} + +void +AudioContext::UnregisterPannerNode(PannerNode* aNode) +{ + mPannerNodes.RemoveElement(aNode); +} + +void +AudioContext::UpdatePannerSource() +{ + for (unsigned i = 0; i < mAudioBufferSourceNodes.Length(); i++) { + mAudioBufferSourceNodes[i]->UnregisterPannerNode(); + } + for (unsigned i = 0; i < mPannerNodes.Length(); i++) { + mPannerNodes[i]->FindConnectedSources(); + } +} + MediaStreamGraph* AudioContext::Graph() const { diff --git a/content/media/webaudio/AudioContext.h b/content/media/webaudio/AudioContext.h index 8b2c7ba9de4..bfd848bcd14 100644 --- a/content/media/webaudio/AudioContext.h +++ b/content/media/webaudio/AudioContext.h @@ -138,6 +138,9 @@ public: MediaStreamGraph* Graph() const; MediaStream* DestinationStream() const; + void UnregisterAudioBufferSourceNode(AudioBufferSourceNode* aNode); + void UnregisterPannerNode(PannerNode* aNode); + void UpdatePannerSource(); private: void RemoveFromDecodeQueue(WebAudioDecodeJob* aDecodeJob); @@ -150,6 +153,10 @@ private: nsRefPtr mListener; MediaBufferDecoder mDecoder; nsTArray > mDecodeJobs; + // Two arrays containing all the PannerNodes and AudioBufferSourceNodes, + // to compute the doppler shift. Those are weak pointers. + nsTArray mPannerNodes; + nsTArray mAudioBufferSourceNodes; }; } diff --git a/content/media/webaudio/AudioListener.cpp b/content/media/webaudio/AudioListener.cpp index 11b96a47fda..d4f965bcecd 100644 --- a/content/media/webaudio/AudioListener.cpp +++ b/content/media/webaudio/AudioListener.cpp @@ -49,6 +49,7 @@ AudioListener::RegisterPannerNode(PannerNode* aPannerNode) aPannerNode->SendThreeDPointParameterToStream(PannerNode::LISTENER_VELOCITY, mVelocity); aPannerNode->SendDoubleParameterToStream(PannerNode::LISTENER_DOPPLER_FACTOR, mDopplerFactor); aPannerNode->SendDoubleParameterToStream(PannerNode::LISTENER_SPEED_OF_SOUND, mSpeedOfSound); + UpdatePannersVelocity(); } void @@ -71,6 +72,15 @@ AudioListener::SendThreeDPointParameterToStream(uint32_t aIndex, const ThreeDPoi } } +void AudioListener::UpdatePannersVelocity() +{ + for (uint32_t i = 0; i < mPanners.Length(); ++i) { + if (mPanners[i]) { + mPanners[i]->SendDopplerToSourcesIfNeeded(); + } + } +} + } } diff --git a/content/media/webaudio/AudioListener.h b/content/media/webaudio/AudioListener.h index cf0c1323cc3..dad80bbb599 100644 --- a/content/media/webaudio/AudioListener.h +++ b/content/media/webaudio/AudioListener.h @@ -80,6 +80,11 @@ public: SendThreeDPointParameterToStream(PannerNode::LISTENER_POSITION, mPosition); } + const ThreeDPoint& Position() const + { + return mPosition; + } + void SetOrientation(double aX, double aY, double aZ, double aXUp, double aYUp, double aZUp) { @@ -101,6 +106,11 @@ public: SendThreeDPointParameterToStream(PannerNode::LISTENER_UPVECTOR, mUpVector); } + const ThreeDPoint& Velocity() const + { + return mVelocity; + } + void SetVelocity(double aX, double aY, double aZ) { if (WebAudioUtils::FuzzyEqual(mVelocity.x, aX) && @@ -112,6 +122,7 @@ public: mVelocity.y = aY; mVelocity.z = aZ; SendThreeDPointParameterToStream(PannerNode::LISTENER_VELOCITY, mVelocity); + UpdatePannersVelocity(); } void RegisterPannerNode(PannerNode* aPannerNode); @@ -119,6 +130,7 @@ public: private: void SendDoubleParameterToStream(uint32_t aIndex, double aValue); void SendThreeDPointParameterToStream(uint32_t aIndex, const ThreeDPoint& aValue); + void UpdatePannersVelocity(); private: friend class PannerNode; diff --git a/content/media/webaudio/AudioNode.cpp b/content/media/webaudio/AudioNode.cpp index 55de21ef689..67b79ad7eb1 100644 --- a/content/media/webaudio/AudioNode.cpp +++ b/content/media/webaudio/AudioNode.cpp @@ -173,6 +173,9 @@ AudioNode::Connect(AudioNode& aDestination, uint32_t aOutput, input->mStreamPort = ps->AllocateInputPort(mStream, MediaInputPort::FLAG_BLOCK_INPUT); } + + // This connection may have connected a panner and a source. + Context()->UpdatePannerSource(); } void @@ -239,6 +242,9 @@ AudioNode::Disconnect(uint32_t aOutput, ErrorResult& aRv) for (uint32_t i = 0; i < outputsToUpdate.Length(); ++i) { outputsToUpdate[i]->UpdateOutputEnded(); } + + // This disconnection may have disconnected a panner and a source. + Context()->UpdatePannerSource(); } } diff --git a/content/media/webaudio/AudioNode.h b/content/media/webaudio/AudioNode.h index 54ca135a672..8041fc2f4e6 100644 --- a/content/media/webaudio/AudioNode.h +++ b/content/media/webaudio/AudioNode.h @@ -80,6 +80,10 @@ public: UpdateOutputEnded(); } + virtual AudioBufferSourceNode* AsAudioBufferSourceNode() { + return nullptr; + } + AudioContext* GetParentObject() const { return mContext; @@ -135,6 +139,11 @@ public: } } + const nsTArray& InputNodes() const + { + return mInputNodes; + } + protected: static void Callback(AudioNode* aNode) { /* not implemented */ } diff --git a/content/media/webaudio/PannerNode.cpp b/content/media/webaudio/PannerNode.cpp index 6492dbb842c..bb785b8f812 100644 --- a/content/media/webaudio/PannerNode.cpp +++ b/content/media/webaudio/PannerNode.cpp @@ -8,6 +8,7 @@ #include "AudioNodeEngine.h" #include "AudioNodeStream.h" #include "AudioListener.h" +#include "AudioBufferSourceNode.h" namespace mozilla { namespace dom { @@ -179,6 +180,7 @@ PannerNode::PannerNode(AudioContext* aContext) PannerNode::~PannerNode() { + Context()->UnregisterPannerNode(this); DestroyMediaStream(); } @@ -416,6 +418,96 @@ PannerNodeEngine::ComputeConeGain() return gain; } +float +PannerNode::ComputeDopplerShift() +{ + double dopplerShift = 1.0; // Initialize to default value + + AudioListener* listener = Context()->Listener(); + + if (listener->DopplerFactor() > 0) { + // Don't bother if both source and listener have no velocity. + if (!mVelocity.IsZero() || !listener->Velocity().IsZero()) { + // Calculate the source to listener vector. + ThreeDPoint sourceToListener = mPosition - listener->Velocity(); + + double sourceListenerMagnitude = sourceToListener.Magnitude(); + + double listenerProjection = sourceToListener.DotProduct(listener->Velocity()) / sourceListenerMagnitude; + double sourceProjection = sourceToListener.DotProduct(mVelocity) / sourceListenerMagnitude; + + listenerProjection = -listenerProjection; + sourceProjection = -sourceProjection; + + double scaledSpeedOfSound = listener->DopplerFactor() / listener->DopplerFactor(); + listenerProjection = min(listenerProjection, scaledSpeedOfSound); + sourceProjection = min(sourceProjection, scaledSpeedOfSound); + + dopplerShift = ((listener->SpeedOfSound() - listener->DopplerFactor() * listenerProjection) / (listener->SpeedOfSound() - listener->DopplerFactor() * sourceProjection)); + + WebAudioUtils::FixNaN(dopplerShift); // Avoid illegal values + + // Limit the pitch shifting to 4 octaves up and 3 octaves down. + dopplerShift = min(dopplerShift, 16.); + dopplerShift = max(dopplerShift, 0.125); + } + } + + return dopplerShift; +} + +void +PannerNode::FindConnectedSources() +{ + mSources.Clear(); + std::set cycleSet; + FindConnectedSources(this, mSources, cycleSet); + for (unsigned i = 0; i < mSources.Length(); i++) { + mSources[i]->RegisterPannerNode(this); + } +} + +void +PannerNode::FindConnectedSources(AudioNode* aNode, + nsTArray& aSources, + std::set& aNodesSeen) +{ + if (!aNode) { + return; + } + + const nsTArray& inputNodes = aNode->InputNodes(); + + for(unsigned i = 0; i < inputNodes.Length(); i++) { + // Return if we find a node that we have seen already. + if (aNodesSeen.find(inputNodes[i].mInputNode) != aNodesSeen.end()) { + return; + } + aNodesSeen.insert(inputNodes[i].mInputNode); + // Recurse + FindConnectedSources(inputNodes[i].mInputNode, aSources, aNodesSeen); + + // Check if this node is an AudioBufferSourceNode + AudioBufferSourceNode* node = inputNodes[i].mInputNode->AsAudioBufferSourceNode(); + if (node) { + aSources.AppendElement(node); + } + } +} + +void +PannerNode::SendDopplerToSourcesIfNeeded() +{ + // Don't bother sending the doppler shift if both the source and the listener + // are not moving, because the doppler shift is going to be 1.0. + if (!(Context()->Listener()->Velocity().IsZero() && mVelocity.IsZero())) { + for(uint32_t i = 0; i < mSources.Length(); i++) { + mSources[i]->SendDopplerShiftToStream(ComputeDopplerShift()); + } + } +} + + } } diff --git a/content/media/webaudio/PannerNode.h b/content/media/webaudio/PannerNode.h index 1335350421c..b2960ad6a92 100644 --- a/content/media/webaudio/PannerNode.h +++ b/content/media/webaudio/PannerNode.h @@ -15,11 +15,13 @@ #include "ThreeDPoint.h" #include "mozilla/WeakPtr.h" #include "WebAudioUtils.h" +#include namespace mozilla { namespace dom { class AudioContext; +class AudioBufferSourceNode; class PannerNode : public AudioNode, public SupportsWeakPtr @@ -92,6 +94,7 @@ public: mVelocity.y = aY; mVelocity.z = aZ; SendThreeDPointParameterToStream(VELOCITY, mVelocity); + SendDopplerToSourcesIfNeeded(); } double RefDistance() const @@ -172,6 +175,11 @@ public: SendDoubleParameterToStream(CONE_OUTER_GAIN, mConeOuterGain); } + float ComputeDopplerShift(); + void SendDopplerToSourcesIfNeeded(); + void FindConnectedSources(); + void FindConnectedSources(AudioNode* aNode, nsTArray& aSources, std::set& aSeenNodes); + private: friend class AudioListener; friend class PannerNodeEngine; @@ -207,6 +215,10 @@ private: double mConeInnerAngle; double mConeOuterAngle; double mConeOuterGain; + + // An array of all the AudioBufferSourceNode connected directly or indirectly + // to this AudioPannerNode. + nsTArray mSources; }; } diff --git a/content/media/webaudio/WebAudioUtils.h b/content/media/webaudio/WebAudioUtils.h index f47c912c5cd..e9c42355c3e 100644 --- a/content/media/webaudio/WebAudioUtils.h +++ b/content/media/webaudio/WebAudioUtils.h @@ -66,6 +66,13 @@ struct WebAudioUtils { { return std::pow(10.0f, 0.05f * aDecibel); } + + static void FixNaN(double& aDouble) + { + if (MOZ_DOUBLE_IS_NaN(aDouble) || MOZ_DOUBLE_IS_INFINITE(aDouble)) { + aDouble = 0.0; + } + } }; }