Files
UnrealEngineUWP/Engine/Source/Runtime/MediaAssets/Private/Assets/MediaSoundComponent.cpp
Chris Adams 9e3125c0a2 adding back surround sound capability to mediasoundcomponent
#jira none

[CL 17587074 by Chris Adams in ue5-main branch]
2021-09-21 15:55:30 -04:00

611 lines
16 KiB
C++

// Copyright Epic Games, Inc. All Rights Reserved.
#include "MediaSoundComponent.h"
#include "MediaAssetsPrivate.h"
#include "Components/BillboardComponent.h"
#include "Engine/Texture2D.h"
#include "IMediaAudioSample.h"
#include "IMediaClock.h"
#include "IMediaClockSink.h"
#include "IMediaModule.h"
#include "IMediaPlayer.h"
#include "MediaAudioResampler.h"
#include "Misc/ScopeLock.h"
#include "Sound/AudioSettings.h"
#include "UObject/UObjectGlobals.h"
#include "ProfilingDebugging/CsvProfiler.h"
#include "MediaPlayer.h"
#include "MediaPlayerFacade.h"
DECLARE_FLOAT_COUNTER_STAT(TEXT("MediaUtils MediaSoundComponent Sync"), STAT_MediaUtils_MediaSoundComponentSync, STATGROUP_Media);
DECLARE_FLOAT_COUNTER_STAT(TEXT("MediaUtils MediaSoundComponent SampleTime"), STAT_MediaUtils_MediaSoundComponentSampleTime, STATGROUP_Media);
DECLARE_DWORD_COUNTER_STAT(TEXT("MediaUtils MediaSoundComponent Queued"), STAT_Media_SoundCompQueued, STATGROUP_Media);
CSV_DECLARE_CATEGORY_MODULE_EXTERN(MEDIA_API, MediaStreaming);
/**
* Clock sink for UMediaSoundComponent.
*/
class FMediaSoundComponentClockSink
: public IMediaClockSink
{
public:
FMediaSoundComponentClockSink(UMediaSoundComponent& InOwner)
: Owner(&InOwner)
{ }
virtual ~FMediaSoundComponentClockSink() { }
public:
virtual void TickInput(FTimespan DeltaTime, FTimespan Timecode) override
{
if (UMediaSoundComponent* OwnerPtr = Owner.Get())
{
OwnerPtr->UpdatePlayer();
}
}
private:
TWeakObjectPtr<UMediaSoundComponent> Owner;
};
static const int32 MaxAudioInputSamples = 8; // accept at most these many samples into our input queue
/* UMediaSoundComponent structors
*****************************************************************************/
UMediaSoundComponent::UMediaSoundComponent(const FObjectInitializer& ObjectInitializer)
: Super(ObjectInitializer)
, Channels(EMediaSoundChannels::Stereo)
, DynamicRateAdjustment(false)
, RateAdjustmentFactor(0.00000001f)
, RateAdjustmentRange(FFloatRange(0.995f, 1.005f))
, CachedRate(0.0f)
, CachedTime(FTimespan::Zero())
, RateAdjustment(1.0f)
, Resampler(new FMediaAudioResampler)
, LastPlaySampleTime(FTimespan::MinValue())
, EnvelopeFollowerAttackTime(10)
, EnvelopeFollowerReleaseTime(100)
, CurrentEnvelopeValue(0.0f)
, bSpectralAnalysisEnabled(false)
, bEnvelopeFollowingEnabled(false)
{
PrimaryComponentTick.bCanEverTick = true;
bAutoActivate = true;
#if PLATFORM_MAC
PreferredBufferLength = 4 * 1024; // increase buffer callback size on macOS to prevent underruns
#endif
#if WITH_EDITORONLY_DATA
bVisualizeComponent = true;
#endif
}
UMediaSoundComponent::~UMediaSoundComponent()
{
delete Resampler;
RemoveClockSink();
}
/* UMediaSoundComponent interface
*****************************************************************************/
bool UMediaSoundComponent::BP_GetAttenuationSettingsToApply(FSoundAttenuationSettings& OutAttenuationSettings)
{
const FSoundAttenuationSettings* SelectedAttenuationSettings = GetSelectedAttenuationSettings();
if (SelectedAttenuationSettings == nullptr)
{
return false;
}
OutAttenuationSettings = *SelectedAttenuationSettings;
return true;
}
UMediaPlayer* UMediaSoundComponent::GetMediaPlayer() const
{
return CurrentPlayer.Get();
}
void UMediaSoundComponent::SetMediaPlayer(UMediaPlayer* NewMediaPlayer)
{
CurrentPlayer = NewMediaPlayer;
}
#if WITH_EDITOR
void UMediaSoundComponent::SetDefaultMediaPlayer(UMediaPlayer* NewMediaPlayer)
{
MediaPlayer = NewMediaPlayer;
CurrentPlayer = MediaPlayer;
}
#endif
void UMediaSoundComponent::UpdatePlayer()
{
UMediaPlayer* CurrentPlayerPtr = CurrentPlayer.Get();
if (CurrentPlayerPtr == nullptr)
{
CachedRate = 0.0f;
CachedTime = FTimespan::Zero();
FScopeLock Lock(&CriticalSection);
SampleQueue.Reset();
return;
}
// create a new sample queue if the player changed
TSharedRef<FMediaPlayerFacade, ESPMode::ThreadSafe> PlayerFacade = CurrentPlayerPtr->GetPlayerFacade();
// We have some audio decoders which are running with a limited amount of pre-allocated audio sample packets.
// When the audio packets are not consumed in the UMediaSoundComponent::OnGenerateAudio method below, these packets are not
// returned to the decoder which then cannot produce more audio samples.
//
// The UMediaSoundComponent::OnGenerateAudio is only called when our parent USynthComponent it active and
// this is conrolled by USynthComponent::Start() and USynthComponent::Stop(). We are tracking a state change here.
if (PlayerFacade != CurrentPlayerFacade)
{
if (IsActive())
{
const auto NewSampleQueue = MakeShared<FMediaAudioSampleQueue, ESPMode::ThreadSafe>(MaxAudioInputSamples);
PlayerFacade->AddAudioSampleSink(NewSampleQueue);
{
FScopeLock Lock(&CriticalSection);
SampleQueue = NewSampleQueue;
}
CurrentPlayerFacade = PlayerFacade;
}
}
else
{
// Here, we have a CurrentPlayerFacade set which means are also have a valid FMediaAudioSampleQueue set
// We need to check for deactivation as it seems there is not callback scheduled when USynthComponent::Stop() is called.
if(!IsActive())
{
FScopeLock Lock(&CriticalSection);
SampleQueue.Reset();
CurrentPlayerFacade.Reset();
}
}
// caching play rate and time for audio thread (eventual consistency is sufficient)
CachedRate = PlayerFacade->GetRate();
CachedTime = PlayerFacade->GetTime();
PlayerFacade->SetLastAudioRenderedSampleTime(LastPlaySampleTime.Load());
}
/* TAttenuatedComponentVisualizer interface
*****************************************************************************/
void UMediaSoundComponent::CollectAttenuationShapesForVisualization(TMultiMap<EAttenuationShape::Type, FBaseAttenuationSettings::AttenuationShapeDetails>& ShapeDetailsMap) const
{
const FSoundAttenuationSettings* SelectedAttenuationSettings = GetSelectedAttenuationSettings();
if (SelectedAttenuationSettings != nullptr)
{
SelectedAttenuationSettings->CollectAttenuationShapesForVisualization(ShapeDetailsMap);
}
}
/* UActorComponent interface
*****************************************************************************/
void UMediaSoundComponent::OnRegister()
{
Super::OnRegister();
#if WITH_EDITORONLY_DATA
if (SpriteComponent != nullptr)
{
SpriteComponent->SpriteInfo.Category = TEXT("Sounds");
SpriteComponent->SpriteInfo.DisplayName = NSLOCTEXT("SpriteCategory", "Sounds", "Sounds");
if (bAutoActivate)
{
SpriteComponent->SetSprite(LoadObject<UTexture2D>(nullptr, TEXT("/Engine/EditorResources/AudioIcons/S_AudioComponent_AutoActivate.S_AudioComponent_AutoActivate")));
}
else
{
SpriteComponent->SetSprite(LoadObject<UTexture2D>(nullptr, TEXT("/Engine/EditorResources/AudioIcons/S_AudioComponent.S_AudioComponent")));
}
}
#endif
}
void UMediaSoundComponent::OnUnregister()
{
{
FScopeLock Lock(&CriticalSection);
SampleQueue.Reset();
}
CurrentPlayerFacade.Reset();
Super::OnUnregister();
}
void UMediaSoundComponent::TickComponent(float DeltaTime, enum ELevelTick TickType, FActorComponentTickFunction *ThisTickFunction)
{
Super::TickComponent(DeltaTime, TickType, ThisTickFunction);
UpdatePlayer();
}
/* USceneComponent interface
*****************************************************************************/
void UMediaSoundComponent::Activate(bool bReset)
{
if (bReset || ShouldActivate())
{
SetComponentTickEnabled(true);
}
Super::Activate(bReset);
}
void UMediaSoundComponent::Deactivate()
{
if (!ShouldActivate())
{
SetComponentTickEnabled(false);
{
FScopeLock Lock(&CriticalSection);
SampleQueue.Reset();
}
CurrentPlayerFacade.Reset();
}
Super::Deactivate();
}
/* UObject interface
*****************************************************************************/
void UMediaSoundComponent::PostLoad()
{
Super::PostLoad();
CurrentPlayer = MediaPlayer;
}
#if WITH_EDITOR
void UMediaSoundComponent::PostEditChangeProperty(FPropertyChangedEvent& PropertyChangedEvent)
{
static const FName MediaPlayerName = GET_MEMBER_NAME_CHECKED(UMediaSoundComponent, MediaPlayer);
FProperty* PropertyThatChanged = PropertyChangedEvent.Property;
if (PropertyThatChanged != nullptr)
{
const FName PropertyName = PropertyThatChanged->GetFName();
if (PropertyName == MediaPlayerName)
{
CurrentPlayer = MediaPlayer;
}
}
Super::PostEditChangeProperty(PropertyChangedEvent);
}
#endif //WITH_EDITOR
/* USynthComponent interface
*****************************************************************************/
bool UMediaSoundComponent::Init(int32& SampleRate)
{
Super::Init(SampleRate);
// Initialize the settings for the spectrum analyzer
SpectrumAnalyzer.Init(SampleRate);
if (Channels == EMediaSoundChannels::Mono)
{
NumChannels = 1;
}
else if (Channels == EMediaSoundChannels::Stereo)
{
NumChannels = 2;
}
else
{
NumChannels = 8;
}
// increase buffer callback size for media decoding. Media doesn't need fast response time so can decode more per callback.
//PreferredBufferLength = NumChannels * 8196;
Resampler->Initialize(NumChannels, SampleRate);
Audio::FEnvelopeFollowerInitParams EnvelopeInitParams;
EnvelopeInitParams.SampleRate = SampleRate;
EnvelopeInitParams.NumChannels = 1; //EnvelopeFollower uses mixed down mono buffer
EnvelopeFollower.Init(EnvelopeInitParams);
return true;
}
int32 UMediaSoundComponent::OnGenerateAudio(float* OutAudio, int32 NumSamples)
{
CSV_SCOPED_TIMING_STAT(MediaStreaming, UMediaSoundComponent_OnGenerateAudio);
int32 InitialSyncOffset = 0;
TSharedPtr<FMediaAudioSampleQueue, ESPMode::ThreadSafe> PinnedSampleQueue;
{
FScopeLock Lock(&CriticalSection);
PinnedSampleQueue = SampleQueue;
}
// We have an input queue and are actively playing?
if (PinnedSampleQueue.IsValid() && (CachedRate != 0.0f))
{
const float Rate = CachedRate.Load();
const FTimespan Time = CachedTime.Load();
{
const uint32 FramesRequested = uint32(NumSamples / NumChannels);
uint32 JumpFrame = MAX_uint32;
FMediaTimeStamp OutTime = FMediaTimeStamp(FTimespan::Zero());
uint32 FramesWritten = Resampler->Generate(OutAudio, OutTime, FramesRequested, Rate, Time, *PinnedSampleQueue, JumpFrame);
if (FramesWritten == 0)
{
return 0; // no samples available
}
// Fill in any gap left as we didn't have enough data
if (FramesWritten < FramesRequested)
{
memset(OutAudio + FramesWritten * NumChannels, 0, (NumSamples - FramesWritten * NumChannels) * sizeof(float));
}
// Update audio time
LastPlaySampleTime = OutTime.Time;
PinnedSampleQueue->SetAudioTime(FMediaTimeStampSample(OutTime, FPlatformTime::Seconds()));
SET_FLOAT_STAT(STAT_MediaUtils_MediaSoundComponentSampleTime, OutTime.Time.GetTotalSeconds());
SET_DWORD_STAT(STAT_Media_SoundCompQueued, PinnedSampleQueue->Num());
}
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
if (bSpectralAnalysisEnabled || bEnvelopeFollowingEnabled)
{
float* BufferToUseForAnalysis = nullptr;
int32 NumFrames = NumSamples;
if (NumChannels == 2)
{
NumFrames = NumSamples / 2;
// Use the scratch buffer to sum the audio to mono
AudioScratchBuffer.Reset();
AudioScratchBuffer.AddUninitialized(NumFrames);
BufferToUseForAnalysis = AudioScratchBuffer.GetData();
int32 SampleIndex = 0;
for (int32 FrameIndex = 0; FrameIndex < NumFrames; ++FrameIndex, SampleIndex += NumChannels)
{
BufferToUseForAnalysis[FrameIndex] = 0.5f * (OutAudio[SampleIndex] + OutAudio[SampleIndex + 1]);
}
}
else
{
BufferToUseForAnalysis = OutAudio;
}
if (bSpectralAnalysisEnabled)
{
SpectrumAnalyzer.PushAudio(BufferToUseForAnalysis, NumFrames);
SpectrumAnalyzer.PerformAsyncAnalysisIfPossible(true);
}
{
FScopeLock ScopeLock(&EnvelopeFollowerCriticalSection);
if (bEnvelopeFollowingEnabled)
{
if (bEnvelopeFollowerSettingsChanged)
{
EnvelopeFollower.SetAttackTime((float)EnvelopeFollowerAttackTime);
EnvelopeFollower.SetReleaseTime((float)EnvelopeFollowerReleaseTime);
bEnvelopeFollowerSettingsChanged = false;
}
EnvelopeFollower.ProcessAudio(BufferToUseForAnalysis, NumFrames);
const TArray<float>& EnvelopeValues = EnvelopeFollower.GetEnvelopeValues();
if (ensure(EnvelopeValues.Num() > 0))
{
CurrentEnvelopeValue = FMath::Clamp(EnvelopeValues[0], 0.f, 1.f);
}
else
{
CurrentEnvelopeValue = 0.f;
}
}
}
}
}
else
{
Resampler->Flush();
LastPlaySampleTime = FTimespan::MinValue();
}
return NumSamples;
}
void UMediaSoundComponent::SetEnableSpectralAnalysis(bool bInSpectralAnalysisEnabled)
{
bSpectralAnalysisEnabled = bInSpectralAnalysisEnabled;
}
void UMediaSoundComponent::SetSpectralAnalysisSettings(TArray<float> InFrequenciesToAnalyze, EMediaSoundComponentFFTSize InFFTSize)
{
Audio::FSpectrumAnalyzerSettings::EFFTSize SpectrumAnalyzerSize;
switch (InFFTSize)
{
case EMediaSoundComponentFFTSize::Min_64:
SpectrumAnalyzerSize = Audio::FSpectrumAnalyzerSettings::EFFTSize::Min_64;
break;
case EMediaSoundComponentFFTSize::Small_256:
SpectrumAnalyzerSize = Audio::FSpectrumAnalyzerSettings::EFFTSize::Small_256;
break;
default:
case EMediaSoundComponentFFTSize::Medium_512:
SpectrumAnalyzerSize = Audio::FSpectrumAnalyzerSettings::EFFTSize::Medium_512;
break;
case EMediaSoundComponentFFTSize::Large_1024:
SpectrumAnalyzerSize = Audio::FSpectrumAnalyzerSettings::EFFTSize::Large_1024;
break;
}
SpectrumAnalyzerSettings.FFTSize = SpectrumAnalyzerSize;
SpectrumAnalyzer.SetSettings(SpectrumAnalyzerSettings);
FrequenciesToAnalyze = InFrequenciesToAnalyze;
}
TArray<FMediaSoundComponentSpectralData> UMediaSoundComponent::GetSpectralData()
{
if (bSpectralAnalysisEnabled)
{
// Locks analyzer output buffer during access to frequency data.
Audio::FAsyncSpectrumAnalyzerScopeLock AnalyzerBufferLock(&SpectrumAnalyzer);
TArray<FMediaSoundComponentSpectralData> SpectralData;
for (float Frequency : FrequenciesToAnalyze)
{
FMediaSoundComponentSpectralData Data;
Data.FrequencyHz = Frequency;
Data.Magnitude = SpectrumAnalyzer.GetMagnitudeForFrequency(Frequency);
SpectralData.Add(Data);
}
return SpectralData;
}
// Empty array if spectrum analysis is not implemented
return TArray<FMediaSoundComponentSpectralData>();
}
TArray<FMediaSoundComponentSpectralData> UMediaSoundComponent::GetNormalizedSpectralData()
{
if (bSpectralAnalysisEnabled)
{
// Locks analyzer output buffer during access to frequency data.
Audio::FAsyncSpectrumAnalyzerScopeLock AnalyzerBufferLock(&SpectrumAnalyzer);
TArray<FMediaSoundComponentSpectralData> SpectralData;
for (float Frequency : FrequenciesToAnalyze)
{
FMediaSoundComponentSpectralData Data;
Data.FrequencyHz = Frequency;
Data.Magnitude = SpectrumAnalyzer.GetNormalizedMagnitudeForFrequency(Frequency);
SpectralData.Add(Data);
}
return SpectralData;
}
// Empty array if spectrum analysis is not implemented
return TArray<FMediaSoundComponentSpectralData>();
}
void UMediaSoundComponent::SetEnableEnvelopeFollowing(bool bInEnvelopeFollowing)
{
FScopeLock ScopeLock(&EnvelopeFollowerCriticalSection);
bEnvelopeFollowingEnabled = bInEnvelopeFollowing;
CurrentEnvelopeValue = 0.0f;
}
void UMediaSoundComponent::SetEnvelopeFollowingsettings(int32 AttackTimeMsec, int32 ReleaseTimeMsec)
{
FScopeLock ScopeLock(&EnvelopeFollowerCriticalSection);
EnvelopeFollowerAttackTime = AttackTimeMsec;
EnvelopeFollowerReleaseTime = ReleaseTimeMsec;
bEnvelopeFollowerSettingsChanged = true;
}
float UMediaSoundComponent::GetEnvelopeValue() const
{
return CurrentEnvelopeValue;
}
void UMediaSoundComponent::AddClockSink()
{
if (!ClockSink.IsValid())
{
IMediaModule* MediaModule = FModuleManager::LoadModulePtr<IMediaModule>("Media");
if (MediaModule != nullptr)
{
ClockSink = MakeShared<FMediaSoundComponentClockSink, ESPMode::ThreadSafe>(*this);
MediaModule->GetClock().AddSink(ClockSink.ToSharedRef());
}
}
}
void UMediaSoundComponent::RemoveClockSink()
{
if (ClockSink.IsValid())
{
IMediaModule* MediaModule = FModuleManager::LoadModulePtr<IMediaModule>("Media");
if (MediaModule != nullptr)
{
MediaModule->GetClock().RemoveSink(ClockSink.ToSharedRef());
}
ClockSink.Reset();
}
}
/* UMediaSoundComponent implementation
*****************************************************************************/
const FSoundAttenuationSettings* UMediaSoundComponent::GetSelectedAttenuationSettings() const
{
if (bOverrideAttenuation)
{
return &AttenuationOverrides;
}
if (AttenuationSettings != nullptr)
{
return &AttenuationSettings->Attenuation;
}
return nullptr;
}