izzy/UnrealEngineUWP
0
0
Fork 0
mirror of https://github.com/izzy2lost/UnrealEngineUWP.git synced 2026-03-26 18:15:20 -07:00
Code Issues Packages Projects Releases Wiki Activity

MLDeformer: Updated NearestNeighborModel to use new NNERuntimeBasicCpu plugin

Browse Source 
#rb zhenglin.geng

[CL 27519060 by daniel holden in ue5-main branch]
This commit is contained in:
daniel holden
2023-08-31 08:37:04 -04:00
parent 71ee593230
commit 89a66144e6
10 changed files with 301 additions and 243 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
Engine/Build/Commit.gitdeps.xml
View File

@@ -58019,7 +58019,7 @@
<File Name="Engine/Plugins/Animation/MLDeformer/NearestNeighborModel/Content/Python/Lib/Win64/site-packages/threadpoolctl.py" Hash="9b2d0a519b75e333bc68377faa183509c99ce2cc" />
<File Name="Engine/Plugins/Animation/MLDeformer/NearestNeighborModel/Content/Python/Lib/Win64/site-packages/threadpoolctl.tps" Hash="d55f1b210bbd8dcef18784f0fbf7bf43da06f6cf" />
<File Name="Engine/Plugins/Animation/MLDeformer/NearestNeighborModel/Content/Python/init_unreal.py" Hash="451d918efcb3d31b954d80ffbc30095471adb1f4" />
<File Name="Engine/Plugins/Animation/MLDeformer/NearestNeighborModel/Content/Python/nearestneighbormodel.py" Hash="db62afa423afc465accf41b27422b89cc7b7b081" />
<File Name="Engine/Plugins/Animation/MLDeformer/NearestNeighborModel/Content/Python/nearestneighbormodel.py" Hash="942bb781b6339b7d900dc4f5c33a895a5e4a8171" />
<File Name="Engine/Plugins/Animation/MLDeformer/NearestNeighborModel/Content/Python/serialization_helpers.py" Hash="9c2e840e42fa0dc25a6a3620daae0db4e0ebb609" />
<File Name="Engine/Plugins/Animation/MLDeformer/NeuralMorphModel/Content/Python/init_unreal.py" Hash="8796940dcf60193a99f1ae5eb6fb5d3bfb199bb4" />
<File Name="Engine/Plugins/Animation/MLDeformer/NeuralMorphModel/Content/Python/neuralmorphmodel_global.py" Hash="abb2fe1493f6faced7e4c80b523b7b397662037a" />
@@ -138370,6 +138370,7 @@
<Blob Hash="942a3602a51504a20a2318a600041d7e373bf143" Size="17818" PackHash="b1da960e7aafb63435ae3d92e8da3afaf73b89db" PackOffset="1333491" />
<Blob Hash="942b3b9d0622198c092e5ffe78ddfbb2e87a7f52" Size="1201" PackHash="b5b52b49e9daa1407edb66997fba1a5ae4a59ec2" PackOffset="400261" />
<Blob Hash="942b721faaac007495b401d118fd0c05e167454b" Size="3660" PackHash="75c9c9b6f21a206e3c2a9a8909da0184bd922394" PackOffset="1891392" />
<Blob Hash="942bb781b6339b7d900dc4f5c33a895a5e4a8171" Size="33239" PackHash="296860e02e1dec06a00c023227089e80b9bd385f" PackOffset="8" />
<Blob Hash="942c1fe04265dd199ec6ee865ca00e699c8509d5" Size="6094" PackHash="7382296627f9c2913b7c88bf79c4c2c3797ece61" PackOffset="96457" />
<Blob Hash="9431139225ec2b13f01f7b931f113299df726cb8" Size="2220" PackHash="aad47491345afd7f479e103b43284661ff7f5a5f" PackOffset="526883" />
<Blob Hash="943183039850c9ddeeed3804a7b1c72f9d4923d7" Size="69212" PackHash="fad444039daee1cbffa6fe09d063bb1d308855c2" PackOffset="169164" />
@@ -156065,7 +156066,6 @@
<Blob Hash="db60eee63e0101813ff851464f3fe2030b3ca9ae" Size="1491" PackHash="9cb792def01303214753d500cad00013ff6cd329" PackOffset="8" />
<Blob Hash="db61dc25875d08ae9ff04c10969b9f60b9dc04ce" Size="9716" PackHash="f3b7b85a14b6fb2f9b5dcde637e36e6d9e8b76dc" PackOffset="871981" />
<Blob Hash="db6269bccad6d91d8ff4bb688208f696b7521141" Size="7083" PackHash="242d288ffca7ff8b7dc9fced709a65c132904c1a" PackOffset="183030" />
<Blob Hash="db62afa423afc465accf41b27422b89cc7b7b081" Size="30965" PackHash="61ec1d9f03649c423d3d73e8858551a0aa40c7d4" PackOffset="8" />
<Blob Hash="db62f685c7df7b9350338fd03907549fbc843f38" Size="37504" PackHash="f5604e39f610a032a42ed9756304612fa5fe1f71" PackOffset="2039712" />
<Blob Hash="db63dbcdd5a96f904ae453793cb18fff0f5560dd" Size="17617" PackHash="ee8790bbadc0ca7a2cafd07479b2837fcc323c64" PackOffset="1211877" />
<Blob Hash="db6510a06de1d0b60b54e401dca484fe712a10fb" Size="101304" PackHash="5979696a63692ebd07536e3d75a524331d8fa8ab" PackOffset="767713" />
@@ -166329,6 +166329,7 @@
<Pack Hash="295342d9c72f6eb2a2d2f08cef133437a6435736" Size="1521945" CompressedSize="1309043" RemotePath="UnrealEngine-25328963" />
<Pack Hash="2961ec184bd3fef316db483a79854d2e503eec8e" Size="10404872" CompressedSize="4571956" RemotePath="UnrealEngine-25357016" />
<Pack Hash="2963971c58277e48df7767e0436d0f2b4e1308c2" Size="1803047" CompressedSize="165351" RemotePath="UnrealEngine-25357016" />
<Pack Hash="296860e02e1dec06a00c023227089e80b9bd385f" Size="33247" CompressedSize="6951" RemotePath="UnrealEngine-27519060" />
<Pack Hash="296ba0bbc11a26ab341baa6385edb13279e36cff" Size="575332" CompressedSize="39048" RemotePath="UnrealEngine-25328963" />
<Pack Hash="297841ade675ba1cfe7b484b3a27879529efe6f4" Size="12394684" CompressedSize="2992955" RemotePath="UnrealEngine-25328963" />
<Pack Hash="2981952302ae2d5c766054b24d77765d40ddd2f2" Size="39686" CompressedSize="37152" RemotePath="UnrealEngine-25328963" />
@@ -167988,7 +167989,6 @@
<Pack Hash="61cfae0046ba3973c3940f2f89d6694b1dc5bc9a" Size="2097016" CompressedSize="1626250" RemotePath="UnrealEngine-25357016" />
<Pack Hash="61d8f86541c6f1e261da25389c8568967186b630" Size="623624" CompressedSize="311845" RemotePath="UnrealEngine-27139403" />
<Pack Hash="61e6ac664b2d7208ac46c2ea17028541001e90c6" Size="117479" CompressedSize="99652" RemotePath="UnrealEngine-25328963" />
<Pack Hash="61ec1d9f03649c423d3d73e8858551a0aa40c7d4" Size="30973" CompressedSize="6485" RemotePath="UnrealEngine-25609828" />
<Pack Hash="61f3de9e7119eca2749e7b544d763606a093799b" Size="1654948" CompressedSize="417539" RemotePath="UnrealEngine-25328963" />
<Pack Hash="6215cd6c044a2be766f8af730aa256c44cc21b8e" Size="318309" CompressedSize="34534" RemotePath="UnrealEngine-26888638" />
<Pack Hash="6216683a711ed2705cbdc9b3b3baacc1aba54484" Size="45566" CompressedSize="9730" RemotePath="UnrealEngine-25843287" />

4
Engine/Plugins/Animation/MLDeformer/NearestNeighborModel/NearestNeighborModel.uplugin
View File

@@ -46,6 +46,10 @@
{
"Name": "MLDeformerFramework",
"Enabled": true
},
{
"Name": "NNERuntimeBasicCpu",
"Enabled": true
}
]
}

2
Engine/Plugins/Animation/MLDeformer/NearestNeighborModel/Source/NearestNeighborModel/NearestNeighborModel.Build.cs
View File

@@ -43,6 +43,8 @@ public class NearestNeighborModel : ModuleRules
"RenderCore",
"RHI",
"MLDeformerFramework",
"NNE",
"NNERuntimeBasicCpu",
}
);

35
Engine/Plugins/Animation/MLDeformer/NearestNeighborModel/Source/NearestNeighborModel/Private/NearestNeighborModel.cpp
View File

@@ -437,11 +437,7 @@ FString UNearestNeighborModel::GetModelDir() const
bool UNearestNeighborModel::ShouldUseOptimizedNetwork() const
{
#if NEARESTNEIGHBORMODEL_USE_ISPC
return UE::NearestNeighborModel::bNearestNeighborModelUseOptimizedNetwork;
#else
return false;
#endif
}
void UNearestNeighborModel::SetUseOptimizedNetwork(bool bInUseOptimizedNetwork)
@@ -535,6 +531,37 @@ bool UNearestNeighborModel::LoadOptimizedNetwork(const FString& OnnxPath)
return false;
}
bool UNearestNeighborModel::LoadOptimizedNetworkFromFile(const FString& Filename)
{
const FString FullPath = FPaths::ConvertRelativePathToFull(Filename);
if (FPaths::FileExists(FullPath))
{
UE_LOG(LogNearestNeighborModel, Display, TEXT("Loading Network file '%s'..."), *FullPath);
// When we create a new UNearestNeighborOptimizedNetwork object we need to set the input
// and output sizes as it cannot get them from the model on load.
UNearestNeighborOptimizedNetwork* Result = NewObject<UNearestNeighborOptimizedNetwork>(this);
Result->SetNumInputs(InputDim);
Result->SetNumOutputs(OutputDim);
const bool bSuccess = Result->Load(FullPath);
if (bSuccess)
{
// Clear optimized network to avoid error messages in SetOptimizedNetwork.
OptimizedNetwork = nullptr;
SetOptimizedNetwork(Result);
return true;
}
}
else
{
UE_LOG(LogNearestNeighborModel, Error, TEXT("Network file '%s' does not exist!"), *FullPath);
}
return false;
}
int32 UNearestNeighborModel::GetOptimizedNetworkNumOutputs() const
{
return OptimizedNetwork ? OptimizedNetwork->GetNumOutputs() : 0;

289
Engine/Plugins/Animation/MLDeformer/NearestNeighborModel/Source/NearestNeighborModel/Private/NearestNeighborOptimizedNetwork.cpp
View File

@@ -1,68 +1,191 @@
// Copyright Epic Games, Inc. All Rights Reserved.
#include "NearestNeighborOptimizedNetwork.h"
#if NEARESTNEIGHBORMODEL_USE_ISPC
#include "NearestNeighborOptimizedNetwork.ispc.generated.h"
#endif
//--------------------------------------------------------------------------
// UNearestNeighborNetworkLayer
//--------------------------------------------------------------------------
void UNearestNeighborNetworkLayer::AddParameter(const TArray<float>& Values, const TArray<int32>& Shape)
{
Parameters.Add({Values, Shape});
}
void UNearestNeighborNetworkLayer::Run(const float* RESTRICT InputBuffer, float* RESTRICT OutputBuffer) const
{
}
#include "Misc/FileHelper.h"
void UNearestNeighborNetworkLayer_Gemm_Prelu::Run(const float* RESTRICT InputBuffer, float* RESTRICT OutputBuffer) const
{
const float* Gemm_Weights = Parameters[0].Values.GetData();
const float* Gemm_Bias = Parameters[1].Values.GetData();
const float PRelu_Slope = Parameters[2].Values[0];
#if NEARESTNEIGHBORMODEL_USE_ISPC
ispc::Gemm_PRelu(OutputBuffer, InputBuffer, Gemm_Weights, Gemm_Bias, PRelu_Slope, NumInputs, NumOutputs);
#endif
}
void UNearestNeighborNetworkLayer_Gemm::Run(const float* RESTRICT InputBuffer, float* RESTRICT OutputBuffer) const
{
const float* Gemm_Weights = Parameters[0].Values.GetData();
const float* Gemm_Bias = Parameters[1].Values.GetData();
#if NEARESTNEIGHBORMODEL_USE_ISPC
ispc::Gemm(OutputBuffer, InputBuffer, Gemm_Weights, Gemm_Bias, NumInputs, NumOutputs);
#endif
}
#include "NNE.h"
#include "NNERuntime.h"
#include "NNERuntimeCPU.h"
#include "NNEModelData.h"
#include "NNERuntimeBasicCpu.h"
//--------------------------------------------------------------------------
// UNearestNeighborOptimizedNetwork
//--------------------------------------------------------------------------
namespace UE::NearestNeighborModel::Private
{
PRAGMA_DISABLE_DEPRECATION_WARNINGS
/** Creates the FileData from the legacy network format and clears it. */
static inline void CreateFileDataAndClearLayers(TArray<uint8>& OutFileData, TArray<TObjectPtr<UNearestNeighborNetworkLayer>>& Layers)
{
UE::NNE::RuntimeBasic::FSequentialModelBuilder Builder;
// Allocate data for the PReLuAlpha values as these need to be kept around until we run "Builder.WriteAndReset"
TArray<TArray<float>> PreluAlphaData;
for (UNearestNeighborNetworkLayer* Layer : Layers)
{
if (UNearestNeighborNetworkLayer_Gemm_Prelu* GemmPreluLayer = Cast<UNearestNeighborNetworkLayer_Gemm_Prelu>(Layer))
{
Builder.AddLinear(
GemmPreluLayer->NumInputs,
GemmPreluLayer->NumOutputs,
GemmPreluLayer->Parameters[0].Values,
GemmPreluLayer->Parameters[1].Values);
PreluAlphaData.AddDefaulted();
PreluAlphaData.Last().Init(GemmPreluLayer->Parameters[2].Values[0], GemmPreluLayer->NumOutputs);
Builder.AddPReLU(PreluAlphaData.Last());
}
else if (UNearestNeighborNetworkLayer_Gemm* GemmLayer = Cast<UNearestNeighborNetworkLayer_Gemm>(Layer))
{
Builder.AddLinear(
GemmLayer->NumInputs,
GemmLayer->NumOutputs,
GemmLayer->Parameters[0].Values,
GemmLayer->Parameters[1].Values);
}
else
{
checkf(false, TEXT("Unknown Layer Type"));
}
}
Layers.Empty();
OutFileData.SetNumUninitialized(Builder.GetWriteByteNum());
Builder.WriteAndReset(OutFileData);
}
PRAGMA_ENABLE_DEPRECATION_WARNINGS
}
PRAGMA_DISABLE_DEPRECATION_WARNINGS
const TCHAR* UNearestNeighborOptimizedNetwork::RuntimeName = TEXT("NNERuntimeBasicCpu");
void UNearestNeighborOptimizedNetwork::PostLoad()
{
Super::PostLoad();
if (Layers_DEPRECATED.Num() > 0)
{
// NumInputs and NumOutputs are not used in the legacy format so we need to load them here.
NumInputs = Layers_DEPRECATED[0]->NumInputs;
NumOutputs = Layers_DEPRECATED.Last()->NumOutputs;
TArray<uint8> FileData;
UE::NearestNeighborModel::Private::CreateFileDataAndClearLayers(FileData, Layers_DEPRECATED);
if (!ModelData)
{
ModelData = NewObject<UNNEModelData>(this);
}
ModelData->Init(TEXT("ubnne"), FileData);
// The ModelData object stores a copy of the FileData so we manually clear this copy to avoid
// having multiple copies of the network in memory at once during loading.
FileData.Empty();
}
// Create in-memory representation of network
TWeakInterfacePtr<INNERuntimeCPU> RuntimeCPU = UE::NNE::GetRuntime<INNERuntimeCPU>(RuntimeName);
if (ensureMsgf(RuntimeCPU.IsValid(), TEXT("Could not find requested NNE Runtime")))
{
if (ModelData)
{
Model = RuntimeCPU->CreateModelCPU(ModelData);
}
}
// If we are not in the editor then we clear the FileData and FileType since these will be
// using additional memory if we are loading from the legacy format.
#if !WITH_EDITOR
ModelData->ClearFileDataAndFileType();
#endif
}
PRAGMA_ENABLE_DEPRECATION_WARNINGS
void UNearestNeighborOptimizedNetwork::Empty()
{
Layers.Empty();
if (ModelData)
{
ModelData->ConditionalBeginDestroy();
ModelData = nullptr;
Model.Reset();
}
}
bool UNearestNeighborOptimizedNetwork::IsEmpty() const
{
return Layers.IsEmpty();
}
int32 UNearestNeighborOptimizedNetwork::GetNumInputs() const
{
return !Layers.IsEmpty() ? Layers[0]->NumInputs : 0;
}
int32 UNearestNeighborOptimizedNetwork::GetNumOutputs() const
{
return !Layers.IsEmpty() ? Layers[Layers.Num()-1]->NumOutputs : 0;
return Model == nullptr;
}
bool UNearestNeighborOptimizedNetwork::Load(const FString& Filename)
{
Empty();
TArray<uint8> FileData;
if (FFileHelper::LoadFileToArray(FileData, *Filename))
{
if (!ModelData)
{
ModelData = NewObject<UNNEModelData>(this);
}
ModelData->Init(TEXT("ubnne"), FileData);
// Clear FileData to avoid multiple copies in memory at once
FileData.Empty();
TWeakInterfacePtr<INNERuntimeCPU> RuntimeCPU = UE::NNE::GetRuntime<INNERuntimeCPU>(RuntimeName);
if (ensureMsgf(RuntimeCPU.IsValid(), TEXT("Could not find requested NNE Runtime")))
{
if (ModelData)
{
Model = RuntimeCPU->CreateModelCPU(ModelData);
}
}
}
else
{
return false;
}
return true;
}
int32 UNearestNeighborOptimizedNetwork::GetNumInputs() const
{
return NumInputs;
}
int32 UNearestNeighborOptimizedNetwork::GetNumOutputs() const
{
return NumOutputs;
}
void UNearestNeighborOptimizedNetwork::SetNumInputs(int32 InNumInputs)
{
NumInputs = InNumInputs;
}
void UNearestNeighborOptimizedNetwork::SetNumOutputs(int32 InNumOutputs)
{
NumOutputs = InNumOutputs;
}
UNearestNeighborOptimizedNetworkInstance* UNearestNeighborOptimizedNetwork::CreateInstance()
{
UNearestNeighborOptimizedNetworkInstance* Instance = NewObject<UNearestNeighborOptimizedNetworkInstance>(this);
@@ -70,41 +193,20 @@ UNearestNeighborOptimizedNetworkInstance* UNearestNeighborOptimizedNetwork::Crea
return Instance;
}
const int32 UNearestNeighborOptimizedNetwork::GetNumLayers() const
UE::NNE::IModelCPU* UNearestNeighborOptimizedNetwork::GetModel() const
{
return Layers.Num();
}
UNearestNeighborNetworkLayer* UNearestNeighborOptimizedNetwork::GetLayer(int32 Index) const
{
return Layers[Index].Get();
}
UNearestNeighborNetworkLayer* UNearestNeighborOptimizedNetwork::AddLayer(const int32 LayerType)
{
UNearestNeighborNetworkLayer* Layer = nullptr;
switch ((ENearestNeighborNetworkLayerType)LayerType)
{
case ENearestNeighborNetworkLayerType::Gemm_Prelu:
Layer = NewObject<UNearestNeighborNetworkLayer_Gemm_Prelu>(this);
break;
case ENearestNeighborNetworkLayerType::Gemm:
Layer = NewObject<UNearestNeighborNetworkLayer_Gemm>(this);
break;
default:
break;
}
if (Layer)
{
Layers.Add(Layer);
}
return Layer;
return Model.Get();
}
//--------------------------------------------------------------------------
// UNearestNeighborOptimizedNetworkInstance
//--------------------------------------------------------------------------
UNearestNeighborOptimizedNetworkInstance::UNearestNeighborOptimizedNetworkInstance() = default;
UNearestNeighborOptimizedNetworkInstance::UNearestNeighborOptimizedNetworkInstance(const FObjectInitializer& ObjectInitializer) : Super(ObjectInitializer) {}
UNearestNeighborOptimizedNetworkInstance::UNearestNeighborOptimizedNetworkInstance(FVTableHelper& Helper) : Super(Helper) {}
UNearestNeighborOptimizedNetworkInstance::~UNearestNeighborOptimizedNetworkInstance() = default;
TArrayView<float> UNearestNeighborOptimizedNetworkInstance::GetInputs()
{
return TArrayView<float>(Inputs.GetData(), Inputs.Num());
@@ -136,50 +238,21 @@ void UNearestNeighborOptimizedNetworkInstance::Init(UNearestNeighborOptimizedNet
Inputs.SetNumZeroed(Network->GetNumInputs());
Outputs.SetNumZeroed(Network->GetNumOutputs());
int32 MaxNumUnits = 0;
const int32 NumLayers = Network->GetNumLayers();
for (int32 LayerIndex = 0; LayerIndex < NumLayers; ++LayerIndex)
if (Network->GetModel())
{
const UNearestNeighborNetworkLayer* CurLayer = Network->GetLayer(LayerIndex);
const int32 NumInputUnits = CurLayer->NumInputs;
const int32 NumOutputUnits = CurLayer->NumOutputs;
MaxNumUnits = FMath::Max3<int32>(NumInputUnits, NumOutputUnits, MaxNumUnits);
Instance = Network->GetModel()->CreateModelInstanceCPU();
Instance->SetInputTensorShapes({ UE::NNE::FTensorShape::Make({ 1, (uint32)Inputs.Num() }) });
}
TempInputArray.SetNumZeroed(MaxNumUnits);
TempOutputArray.SetNumZeroed(MaxNumUnits);
}
void UNearestNeighborOptimizedNetworkInstance::Run()
{
TRACE_CPUPROFILER_EVENT_SCOPE(UNearestNeighborOptimizedNetwork::Run)
TRACE_CPUPROFILER_EVENT_SCOPE(UNearestNeighborOptimizedNetwork::Run);
// Setup the buffer pointers.
FRunSettings RunSettings;
RunSettings.TempInputBuffer = TempInputArray.GetData();
RunSettings.TempOutputBuffer = TempOutputArray.GetData();
RunSettings.InputBuffer = Inputs.GetData();
RunSettings.OutputBuffer = Outputs.GetData();
// Run the network.
const int32 NumLayers = Network->GetNumLayers();
for (int32 LayerIndex = 0; LayerIndex < NumLayers; ++LayerIndex)
if (Instance)
{
const UNearestNeighborNetworkLayer* CurLayer = Network->GetLayer(LayerIndex);
if (LayerIndex == 0)
{
const float* const RESTRICT NetworkInputs = RunSettings.InputBuffer;
CurLayer->Run(RunSettings.InputBuffer, RunSettings.TempInputBuffer);
}
else if (LayerIndex == NumLayers - 1)
{
CurLayer->Run(RunSettings.TempInputBuffer, RunSettings.OutputBuffer);
}
else
{
CurLayer->Run(RunSettings.TempInputBuffer, RunSettings.TempOutputBuffer);
Swap(RunSettings.TempInputBuffer, RunSettings.TempOutputBuffer);
}
Instance->RunSync(
{ { (void*)Inputs.GetData(), Inputs.Num() * sizeof(float) } },
{ { (void*)Outputs.GetData(), Outputs.Num() * sizeof(float) } });
}
}

130
Engine/Plugins/Animation/MLDeformer/NearestNeighborModel/Source/NearestNeighborModel/Private/NearestNeighborOptimizedNetwork.h
View File

@@ -5,15 +5,18 @@
#include "UObject/Object.h"
#include "NearestNeighborOptimizedNetwork.generated.h"
namespace UE::NNE
{
class IModelCPU;
class IModelInstanceCPU;
}
class UNNEModelData;
class UNearestNeighborOptimizedNetworkInstance;
#define NEARESTNEIGHBORMODEL_USE_ISPC INTEL_ISPC
#if !defined(NEARESTNEIGHBORMODEL_USE_ISPC)
#define NEARESTNEIGHBORMODEL_USE_ISPC 0
#endif
USTRUCT()
struct FNearestNeighborNetworkParameter
struct UE_DEPRECATED(5.4, "Nearest Neighbor Network Parameter no longer used for inference.") FNearestNeighborNetworkParameter
{
GENERATED_BODY()
@@ -24,62 +27,43 @@ struct FNearestNeighborNetworkParameter
TArray<int32> Shape;
};
UENUM(BlueprintType)
enum class ENearestNeighborNetworkLayerType : uint8
{
None,
Gemm_Prelu,
Gemm,
};
/** A general network layer that contains a list of parameters. The Run() method should be implemented by child classes */
UCLASS()
class UNearestNeighborNetworkLayer
class UE_DEPRECATED(5.4, "Nearest Neighbor Network Layer no longer used for inference.") UNearestNeighborNetworkLayer
: public UObject
{
GENERATED_BODY()
public:
/** The weight matrix number of inputs (rows). */
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "MLDeformer")
int32 NumInputs = 0;
/** The weight matrix number of outputs (columns). */
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "MLDeformer")
int32 NumOutputs = 0;
// /** The parameters of the layer */
PRAGMA_DISABLE_DEPRECATION_WARNINGS
UPROPERTY()
TArray<FNearestNeighborNetworkParameter> Parameters;
UFUNCTION(BlueprintCallable, Category = "MLDeformer")
void AddParameter(const TArray<float>& Values, const TArray<int32>& Shape);
virtual void Run(const float* RESTRICT InputBuffer, float* RESTRICT OutputBuffer) const;
PRAGMA_ENABLE_DEPRECATION_WARNINGS
};
/** Gemm: GEneral Matrix Multiplication: Wx + b */
PRAGMA_DISABLE_DEPRECATION_WARNINGS
UCLASS()
class UNearestNeighborNetworkLayer_Gemm_Prelu
: public UNearestNeighborNetworkLayer
class UE_DEPRECATED(5.4, "Nearest Neighbor Network Layer no longer used for inference.") UNearestNeighborNetworkLayer_Gemm_Prelu : public UNearestNeighborNetworkLayer
{
GENERATED_BODY()
public:
virtual void Run(const float* RESTRICT InputBuffer, float* RESTRICT OutputBuffer) const override;
};
UCLASS()
class UNearestNeighborNetworkLayer_Gemm
: public UNearestNeighborNetworkLayer
class UE_DEPRECATED(5.4, "Nearest Neighbor Network Layer no longer used for inference.") UNearestNeighborNetworkLayer_Gemm : public UNearestNeighborNetworkLayer
{
GENERATED_BODY()
public:
virtual void Run(const float* RESTRICT InputBuffer, float* RESTRICT OutputBuffer) const override;
};
PRAGMA_ENABLE_DEPRECATION_WARNINGS
/**
* The specialized neural network for the MLDeformerModel.
* This class is used to do inference at runtime at a higher performance than using UNeuralNetwork.
@@ -91,6 +75,9 @@ class UNearestNeighborOptimizedNetwork
GENERATED_BODY()
public:
virtual void PostLoad() override;
/** Clear the network, getting rid of all layers. */
UFUNCTION(BlueprintCallable, Category = "MLDeformer")
virtual void Empty();
@@ -129,31 +116,51 @@ public:
*/
virtual int32 GetNumOutputs() const;
/**
* Sets the number of inputs, which is the number of floats the network takes as input.
* @param InNumInputs The number of input floats to the network.
*/
virtual void SetNumInputs(int32 InNumInputs);
/**
* Get the number of network layers.
* @return The number of network layer.
* Sets the number of outputs, which is the number of floats the network will output.
* @param InNumOutputs The number of floats that the network outputs.
*/
const int32 GetNumLayers() const;
virtual void SetNumOutputs(int32 InNumOutputs);
/**
* Get a given network layer.
* @return A pointer to the layer, which will contain the parameters.
* Get the Model.
* @return A pointer to the Model.
*/
UNearestNeighborNetworkLayer* GetLayer(int32 Index) const;
/**
* Add a network layer.
* @return A pointer to the layer, which will contain the parameters.
*/
UFUNCTION(BlueprintCallable, Category = "MLDeformer")
UNearestNeighborNetworkLayer* AddLayer(const int32 LayerType);
UE::NNE::IModelCPU* GetModel() const;
protected:
/** The network layers */
/** The Model Data used for network */
UPROPERTY()
TArray<TObjectPtr<UNearestNeighborNetworkLayer>> Layers;
TObjectPtr<UNNEModelData> ModelData;
/** The Neural Network used. */
TSharedPtr<UE::NNE::IModelCPU> Model;
UPROPERTY()
int32 NumInputs = 0;
UPROPERTY()
int32 NumOutputs = 0;
PRAGMA_DISABLE_DEPRECATION_WARNINGS
UE_DEPRECATED(5.4, "Layers Object no longer available.")
UPROPERTY(meta = (DeprecatedProperty, DeprecationMessage = "Network format changed."))
TArray<TObjectPtr<UNearestNeighborNetworkLayer>> Layers_DEPRECATED;
PRAGMA_ENABLE_DEPRECATION_WARNINGS
private:
static const TCHAR* RuntimeName;
};
/**
@@ -170,6 +177,13 @@ class UNearestNeighborOptimizedNetworkInstance
GENERATED_BODY()
public:
// These are required because of the use of TUniquePtr
UNearestNeighborOptimizedNetworkInstance();
UNearestNeighborOptimizedNetworkInstance(const FObjectInitializer& ObjectInitializer);
UNearestNeighborOptimizedNetworkInstance(FVTableHelper& Helper);
virtual ~UNearestNeighborOptimizedNetworkInstance();
/**
* Get the network input buffer.
* @return The array view of floats that represents the network inputs.
@@ -198,13 +212,6 @@ public:
virtual void Run();
protected:
struct FRunSettings
{
float* TempInputBuffer;
float* TempOutputBuffer;
const float* InputBuffer;
float* OutputBuffer;
};
/**
* Initialize this instance using a given network object.
@@ -220,11 +227,8 @@ protected:
/** The output values. */
TArray<float> Outputs;
/** A pre-allocated temp buffer for inputs. */
TArray<float> TempInputArray;
/** A pre-allocated temp buffer for outputs. */
TArray<float> TempOutputArray;
/** The instance data for this Network Instance. */
TSharedPtr<UE::NNE::IModelInstanceCPU> Instance;
/** The neural network this is an instance of. */
UPROPERTY(Transient)

59
Engine/Plugins/Animation/MLDeformer/NearestNeighborModel/Source/NearestNeighborModel/Private/NearestNeighborOptimizedNetwork.ispc
View File

@@ -1,59 +0,0 @@
// Copyright Epic Games, Inc. All Rights Reserved.
export void Gemm_PRelu(
uniform float Outputs[],
const uniform float Inputs[],
const uniform float Gemm_Weights[],
const uniform float Gemm_Biases[],
const uniform float PRelu_Alpha,
const uniform int NumInputs,
const uniform int NumOutputs)
{
foreach (OutputIndex = 0 ... NumOutputs)
{
Outputs[OutputIndex] = Gemm_Biases[OutputIndex];
}
for (uniform int InputIndex = 0; InputIndex < NumInputs; ++InputIndex)
{
uniform float X = Inputs[InputIndex];
foreach (OutputIndex = 0 ... NumOutputs)
{
Outputs[OutputIndex] += X * Gemm_Weights[InputIndex * NumOutputs + OutputIndex];
}
}
foreach (OutputIndex = 0 ... NumOutputs)
{
const float Output = Outputs[OutputIndex];
Outputs[OutputIndex] = select(Output > 0.0f, Output, Output * PRelu_Alpha);
}
}
export void Gemm(
uniform float Outputs[],
const uniform float Inputs[],
const uniform float Gemm_Weights[],
const uniform float Gemm_Biases[],
const uniform int NumInputs,
const uniform int NumOutputs)
{
foreach (OutputIndex = 0 ... NumOutputs)
{
Outputs[OutputIndex] = Gemm_Biases[OutputIndex];
}
for (uniform int InputIndex = 0; InputIndex < NumInputs; ++InputIndex)
{
uniform float X = Inputs[InputIndex];
foreach (OutputIndex = 0 ... NumOutputs)
{
Outputs[OutputIndex] += X * Gemm_Weights[InputIndex * NumOutputs + OutputIndex];
}
}
}

13
Engine/Plugins/Animation/MLDeformer/NearestNeighborModel/Source/NearestNeighborModel/Public/NearestNeighborModel.h
View File

@@ -261,8 +261,12 @@ private:
UNearestNeighborOptimizedNetwork* GetOptimizedNetwork() const { return OptimizedNetwork.Get(); }
int32 GetOptimizedNetworkNumOutputs() const;
void SetOptimizedNetwork(UNearestNeighborOptimizedNetwork* InOptimizedNetwork);
UE_DEPRECATED(5.4, "Onnx file no longer used for training network. Use LoadOptimizedNetworkFromFile instead.")
bool LoadOptimizedNetwork(const FString& OnnxPath);
bool LoadOptimizedNetworkFromFile(const FString& Filename);
#if WITH_EDITORONLY_DATA
TObjectPtr<UAnimSequence> GetNearestNeighborSkeletons(int32 PartId);
const TObjectPtr<UAnimSequence> GetNearestNeighborSkeletons(int32 PartId) const;
@@ -347,16 +351,17 @@ private:
#endif
protected:
#if WITH_EDITORONLY_DATA
UPROPERTY(VisibleAnywhere, BlueprintReadOnly, Category = "Training Settings")
int32 InputDim = 0;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Training Settings")
TArray<int32> HiddenLayerDims;
UPROPERTY(VisibleAnywhere, BlueprintReadOnly, Category = "Training Settings")
int32 OutputDim = 0;
#if WITH_EDITORONLY_DATA
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Training Settings")
TArray<int32> HiddenLayerDims;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Training Settings", meta = (ClampMin = "1"))
int32 NumEpochs = 10000;

4
Engine/Plugins/Animation/MLDeformer/NearestNeighborModel/Source/NearestNeighborModelEditor/Private/NearestNeighborEditorModel.cpp
View File

@@ -99,10 +99,10 @@ namespace UE::NearestNeighborModel
UNearestNeighborModel* NearestNeighborModel = GetNearestNeighborModel();
if (NearestNeighborModel)
{
const FString OnnxFile = GetTrainedNetworkOnnxFile();
const FString File = GetNearestNeighborModel()->GetModelDir() / TEXT("NearestNeighborModel.ubnne");
if (NearestNeighborModel->ShouldUseOptimizedNetwork())
{
const bool bSuccess = NearestNeighborModel->LoadOptimizedNetwork(OnnxFile);
const bool bSuccess = NearestNeighborModel->LoadOptimizedNetworkFromFile(File);
if (bSuccess)
{
UNearestNeighborModelInstance* ModelInstance = static_cast<UNearestNeighborModelInstance*>(GetTestMLDeformerModelInstance());

2
Engine/Plugins/Animation/MLDeformer/NearestNeighborModel/Source/NearestNeighborModelEditor/Public/NearestNeighborEditorModel.h
View File

@@ -48,6 +48,8 @@ namespace UE::NearestNeighborModel
virtual bool LoadTrainedNetwork() const override;
virtual void OnPropertyChanged(FPropertyChangedEvent& PropertyChangedEvent) override;
virtual void OnPostTraining(ETrainingResult TrainingResult, bool bUsePartiallyTrainedWhenAborted) override;
UE_DEPRECATED(5.4, "Onnx file no longer used for training network.")
virtual FString GetTrainedNetworkOnnxFile() const override;
virtual int32 GetNumTrainingFrames() const override;
// ~END FMLDeformerGeomCacheEditorModel overrides.