izzy/UnrealEngineUWP
0
0
Fork 0
mirror of https://github.com/izzy2lost/UnrealEngineUWP.git synced 2026-03-26 18:15:20 -07:00
Code Issues Packages Projects Releases Wiki Activity
Files
5.5
UnrealEngineUWP/Engine/Plugins/Mutable/Source/MutableTools/Private/MuT/ASTOpConstantResource.cpp
pere rifa 9681e550ad [Mutable] Add a SourceIDs to identify and group Roms and use the SourceDataDescriptor to propagate them. 
Set unique row IDs to table rows to generate deterministic SourceIDs.
Bump current supported version.

#jira UE-224812
#rnx
#rb genis.sole, jordi.rovira

[CL 36750126 by pere rifa in 5.5 branch]
2024-10-01 18:10:37 -04:00

788 lines
20 KiB
C++
Raw Permalink Blame History

// Copyright Epic Games, Inc. All Rights Reserved.
#include "MuT/ASTOpConstantResource.h"
#include "MuT/CompilerPrivate.h"
#include "MuR/Layout.h"
#include "MuR/Mesh.h"
#include "MuR/ModelPrivate.h"
#include "MuR/ImagePrivate.h"
#include "MuR/MutableMath.h"
#include "MuR/PhysicsBody.h"
#include "MuR/Serialisation.h"
#include "MuR/Skeleton.h"
#include "MuR/Types.h"
#include "Containers/Array.h"
#include "HAL/PlatformMath.h"
#include "HAL/PlatformFileManager.h"
#include "Hash/CityHash.h"
#include "Misc/AssertionMacros.h"
#include "GenericPlatform/GenericPlatformFile.h"
#include "Compression/OodleDataCompression.h"
#include <inttypes.h> // Required for 64-bit printf macros
namespace mu
{
/** Proxy class for a temporary resource while compiling.
* The resource may be stored in different ways:
* - as is, in memory with its own pointer.
* - in a compressed buffer
* - saved to a disk file compressed or uncompressed.
*/
template<class R>
class MUTABLETOOLS_API ResourceProxyTempFile : public ResourceProxy<R>
{
private:
/** Actual resource to store. If the pointer is valid, it wasn't worth dumping to disk or compressing. */
Ptr<const R> Resource;
/** Temp filename used if it was necessary. */
FString FileName;
/** Size of the resource in memory. */
uint32 UncompressedSize = 0;
/** Size of the saved file. It may be the size of the resource in memory, or its compressed size. */
uint32 FileSize = 0;
/** Valid if the resource was compressed and stored in memory instead of dumped to disk. */
TArray<uint8> CompressedBuffer;
/** Shared context with cache settings and stats. */
FProxyFileContext& Options;
/** Prevent concurrent access to a signel resource. */
FCriticalSection Mutex;
public:
ResourceProxyTempFile(const R* InResource, FProxyFileContext& InOptions)
: Options(InOptions)
{
if (!InResource)
{
return;
}
IPlatformFile& PlatformFile = FPlatformFileManager::Get().GetPlatformFile();
OutputMemoryStream stream(128*1024);
OutputArchive arch(&stream);
R::Serialise(InResource, arch);
UncompressedSize = stream.GetBufferSize();
if (stream.GetBufferSize() <= Options.MinProxyFileSize)
{
// Not worth compressing or caching to disk
Resource = InResource;
}
else
{
// Compress
int64 CompressedSize = 0;
constexpr bool bEnableCompression = true;
if (bEnableCompression)
{
int64 CompressedBufferSize = FOodleDataCompression::CompressedBufferSizeNeeded(stream.GetBufferSize());
CompressedBufferSize = FMath::Max(CompressedBufferSize, int64(stream.GetBufferSize() / 2));
CompressedBuffer.SetNumUninitialized(CompressedBufferSize);
CompressedSize = FOodleDataCompression::CompressParallel(
CompressedBuffer.GetData(), CompressedBufferSize,
stream.GetBuffer(), stream.GetBufferSize(),
FOodleDataCompression::ECompressor::Kraken,
FOodleDataCompression::ECompressionLevel::SuperFast,
true // CompressIndependentChunks
);
}
bool bCompressed = CompressedSize != 0;
if (bCompressed && uint64(CompressedSize) <= Options.MinProxyFileSize)
{
// Keep the compressed data, and don't store to file
CompressedBuffer.SetNum(CompressedSize,EAllowShrinking::Yes);
}
else
{
// Save
FString Prefix = FPlatformProcess::UserTempDir();
uint32 PID = FPlatformProcess::GetCurrentProcessId();
Prefix += FString::Printf(TEXT("mut.temp.%u"), PID);
FString FinalTempPath;
IFileHandle* ResourceFile = nullptr;
uint64 AttemptCount = 0;
while (!ResourceFile && AttemptCount < Options.MaxFileCreateAttempts)
{
uint64 ThisThreadFileIndex = Options.CurrentFileIndex.load();
while (!Options.CurrentFileIndex.compare_exchange_strong(ThisThreadFileIndex, ThisThreadFileIndex + 1));
FinalTempPath = Prefix + FString::Printf(TEXT(".%.16" PRIx64), ThisThreadFileIndex);
ResourceFile = PlatformFile.OpenWrite(*FinalTempPath);
++AttemptCount;
}
if (!ResourceFile)
{
UE_LOG(LogMutableCore, Error, TEXT("Failed to create temporary file. Disk full?"));
check(false);
}
if (bCompressed)
{
FileSize = CompressedSize;
ResourceFile->Write(CompressedBuffer.GetData(), FileSize);
}
else
{
FileSize = UncompressedSize;
ResourceFile->Write((const uint8*)stream.GetBuffer(), FileSize);
}
CompressedBuffer.SetNum(0, EAllowShrinking::Yes);
delete ResourceFile;
FileName = FinalTempPath;
Options.FilesWritten++;
Options.BytesWritten += FileSize;
}
}
}
~ResourceProxyTempFile()
{
FScopeLock Lock(&Mutex);
if (!FileName.IsEmpty())
{
// Delete temp file
FPlatformFileManager::Get().GetPlatformFile().DeleteFile(*FileName);
FileName.Empty();
}
}
Ptr<const R> Get() override
{
FScopeLock Lock(&Mutex);
Ptr<const R> Result;
if (Resource)
{
// Cached as is
Result = Resource;
}
else if (!CompressedBuffer.Num() && !FileName.IsEmpty())
{
IFileHandle* resourceFile = FPlatformFileManager::Get().GetPlatformFile().OpenRead(*FileName);
check(resourceFile);
CompressedBuffer.SetNumUninitialized(FileSize);
resourceFile->Read(CompressedBuffer.GetData(), FileSize);
delete resourceFile;
bool bCompressed = FileSize != UncompressedSize;
if (!bCompressed)
{
InputMemoryStream stream(CompressedBuffer.GetData(), FileSize);
InputArchive arch(&stream);
Result = R::StaticUnserialise(arch);
CompressedBuffer.SetNum(0, EAllowShrinking::Yes);
}
Options.FilesRead++;
Options.BytesRead += FileSize;
}
if (CompressedBuffer.Num())
{
// Cached compressed
TArray<uint8> UncompressedBuf;
UncompressedBuf.SetNumUninitialized(UncompressedSize);
bool bSuccess = FOodleDataCompression::DecompressParallel(
UncompressedBuf.GetData(), UncompressedSize,
CompressedBuffer.GetData(), CompressedBuffer.Num());
check(bSuccess);
if (bSuccess)
{
InputMemoryStream stream(UncompressedBuf.GetData(), UncompressedSize);
InputArchive arch(&stream);
Result = R::StaticUnserialise(arch);
}
if (!FileName.IsEmpty())
{
CompressedBuffer.SetNum(0, EAllowShrinking::Yes);
}
}
return Result;
}
};
//-------------------------------------------------------------------------------------------------
//-------------------------------------------------------------------------------------------------
//-------------------------------------------------------------------------------------------------
void ASTOpConstantResource::ForEachChild(const TFunctionRef<void(ASTChild&)>)
{
}
//-------------------------------------------------------------------------------------------------
bool ASTOpConstantResource::IsEqual(const ASTOp& OtherUntyped) const
{
if (OtherUntyped.GetOpType()==GetOpType())
{
const ASTOpConstantResource* Other = static_cast<const ASTOpConstantResource*>(&OtherUntyped);
return Type == Other->Type && ValueHash == Other->ValueHash &&
LoadedValue == Other->LoadedValue && Proxy == Other->Proxy
&& SourceDataDescriptor == Other->SourceDataDescriptor;
}
return false;
}
//-------------------------------------------------------------------------------------------------
mu::Ptr<ASTOp> ASTOpConstantResource::Clone(MapChildFuncRef) const
{
Ptr<ASTOpConstantResource> n = new ASTOpConstantResource();
n->Type = Type;
n->Proxy = Proxy;
n->LoadedValue = LoadedValue;
n->ValueHash = ValueHash;
n->SourceDataDescriptor = SourceDataDescriptor;
return n;
}
//-------------------------------------------------------------------------------------------------
uint64 ASTOpConstantResource::Hash() const
{
uint64 res = std::hash<uint64>()(uint64(Type));
hash_combine(res, ValueHash);
return res;
}
namespace
{
/** Adds a constant image data to a program and returns its constant index. */
int32 AddConstantImage(FProgram& Program, const Ptr<const Image>& pImage, FLinkerOptions& Options)
{
MUTABLE_CPUPROFILER_SCOPE(AddConstantImage);
check(pImage->GetSizeX() * pImage->GetSizeY() > 0);
// Mips to store
int32 MipsToStore = 1;
int32 FirstLODIndexIndex = Program.ConstantImageLODIndices.Num();
FImageOperator& ImOp = Options.ImageOperator;
Ptr<const Image> pMip;
if (!Options.bSeparateImageMips)
{
pMip = pImage;
}
else
{
// We may want the full mipmaps for fragments of images, regardless of the resident mip size, for intermediate operations.
// \TODO: Calculate the mip ranges that makes sense to store.
int32 MaxMipmaps = Image::GetMipmapCount(pImage->GetSizeX(), pImage->GetSizeY());
MipsToStore = MaxMipmaps;
// Some images cannot be resized or mipmaped
bool bCannotBeScaled = pImage->m_flags & Image::IF_CANNOT_BE_SCALED;
if (bCannotBeScaled)
{
// Store only the mips that we have already calculated. We assume we have calculated them correctly.
MipsToStore = pImage->GetLODCount();
}
if (pImage->GetLODCount() == 1)
{
pMip = pImage;
}
else
{
pMip = ImOp.ExtractMip(pImage.get(), 0);
}
}
// Temporary uncompressed version of the image, if we need to generate the mips and the source is compressed.
Ptr<const Image> UncompressedMip;
EImageFormat UncompressedFormat = GetUncompressedFormat( pMip->GetFormat() );
for (int32 Mip = 0; Mip < MipsToStore; ++Mip)
{
check(pMip->GetFormat() == pImage->GetFormat());
// Ensure unique at mip level
int32 MipIndex = -1;
// Use a map-based deduplication only if we are splitting mips.
if (Options.bSeparateImageMips)
{
MUTABLE_CPUPROFILER_SCOPE(Deduplicate);
const int32* IndexPtr = Options.ImageConstantMipMap.Find(pMip);
if (IndexPtr)
{
MipIndex = *IndexPtr;
}
}
if (MipIndex<0)
{
MipIndex = Program.ConstantImageLODs.Add(TPair<int32, Ptr<const Image>>(-1, pMip));
Options.ImageConstantMipMap.Add(pMip, MipIndex);
}
Program.ConstantImageLODIndices.Add(uint32(MipIndex));
// Generate next mip if necessary
if (Mip + 1 < MipsToStore)
{
Ptr<Image> NewMip;
if (Mip+1 < pImage->GetLODCount())
{
// Extract directly from source image
NewMip = ImOp.ExtractMip(pImage.get(), Mip + 1);
}
else
{
// Generate from the last mip.
if (UncompressedFormat!=pMip->GetFormat())
{
int32 Quality = 4; // TODO
if (!UncompressedMip)
{
UncompressedMip = ImOp.ImagePixelFormat(Quality, pMip.get(), UncompressedFormat);
}
UncompressedMip = ImOp.ExtractMip(UncompressedMip.get(), 1);
NewMip = ImOp.ImagePixelFormat(Quality, UncompressedMip.get(), pMip->GetFormat());
}
else
{
NewMip = ImOp.ExtractMip(pMip.get(), 1);
}
}
check(NewMip);
pMip = NewMip;
}
}
FImageLODRange LODRange;
LODRange.FirstIndex = FirstLODIndexIndex;
LODRange.LODCount = MipsToStore;
LODRange.ImageFormat = pImage->GetFormat();
LODRange.ImageSizeX = pImage->GetSizeX();
LODRange.ImageSizeY = pImage->GetSizeY();
int32 ImageIndex = Program.ConstantImages.Add(LODRange);
return ImageIndex;
}
}
//-------------------------------------------------------------------------------------------------
void ASTOpConstantResource::Link(FProgram& program, FLinkerOptions* Options)
{
MUTABLE_CPUPROFILER_SCOPE(ASTOpConstantResource_Link);
if (!linkedAddress && !bLinkedAndNull)
{
if (Type == OP_TYPE::ME_CONSTANT)
{
OP::MeshConstantArgs args;
FMemory::Memset(&args, 0, sizeof(args));
Ptr<Mesh> MeshData = static_cast<const Mesh*>(GetValue().get())->Clone();
check(MeshData);
args.skeleton = -1;
if (Ptr<const Skeleton> pSkeleton = MeshData->GetSkeleton())
{
args.skeleton = program.AddConstant(pSkeleton.get());
MeshData->SetSkeleton(nullptr);
}
args.physicsBody = -1;
if (Ptr<const PhysicsBody> pPhysicsBody = MeshData->GetPhysicsBody())
{
args.physicsBody = program.AddConstant(pPhysicsBody.get());
MeshData->SetPhysicsBody(nullptr);
}
// Use a map-based deduplication
mu::Ptr<const mu::Mesh> Key = MeshData;
const int32* IndexPtr = Options->MeshConstantMap.Find(Key);
if (!IndexPtr)
{
args.value = program.AddConstant(MeshData.get());
int32 DataDescIndex = Options->AdditionalData.SourceMeshPerConstant.Add(SourceDataDescriptor);
check(DataDescIndex == args.value);
Options->MeshConstantMap.Add(MeshData, int32(args.value));
}
else
{
args.value = *IndexPtr;
}
linkedAddress = (OP::ADDRESS)program.m_opAddress.Num();
program.m_opAddress.Add((uint32_t)program.m_byteCode.Num());
AppendCode(program.m_byteCode, Type);
AppendCode(program.m_byteCode, args);
}
else
{
OP::ResourceConstantArgs args;
FMemory::Memset(&args, 0, sizeof(args));
bool bValidData = true;
switch (Type)
{
case OP_TYPE::IM_CONSTANT:
{
Ptr<const Image> pTyped = static_cast<const Image*>(GetValue().get());
check(pTyped);
if (pTyped->GetSizeX() * pTyped->GetSizeY() == 0)
{
// It's an empty or degenerated image, return a null operation.
bValidData = false;
}
else
{
args.value = AddConstantImage( program, pTyped, *Options);
int32 DataDescIndex = Options->AdditionalData.SourceImagePerConstant.Add(SourceDataDescriptor);
check(DataDescIndex == args.value);
}
break;
}
case OP_TYPE::LA_CONSTANT:
{
Ptr<const Layout> pTyped = static_cast<const Layout*>(GetValue().get());
check(pTyped);
args.value = program.AddConstant(pTyped);
break;
}
default:
check(false);
}
if (bValidData)
{
linkedAddress = (OP::ADDRESS)program.m_opAddress.Num();
program.m_opAddress.Add((uint32)program.m_byteCode.Num());
AppendCode(program.m_byteCode, Type);
AppendCode(program.m_byteCode, args);
}
else
{
// Null op
linkedAddress = 0;
bLinkedAndNull = true;
}
}
// Clear stored value to reduce memory usage.
LoadedValue = nullptr;
Proxy = nullptr;
}
}
//-------------------------------------------------------------------------------------------------
FImageDesc ASTOpConstantResource::GetImageDesc(bool, class FGetImageDescContext*) const
{
FImageDesc Result;
if (Type == OP_TYPE::IM_CONSTANT)
{
// TODO: cache to avoid disk loading
Ptr<const Image> ConstImage = static_cast<const Image*>(GetValue().get());
Result.m_format = ConstImage->GetFormat();
Result.m_lods = ConstImage->GetLODCount();
Result.m_size = ConstImage->GetSize();
}
else
{
check(false);
}
return Result;
}
//-------------------------------------------------------------------------------------------------
void ASTOpConstantResource::GetBlockLayoutSize(uint64 BlockId, int32* BlockX, int32* BlockY, FBlockLayoutSizeCache*)
{
switch (Type)
{
case OP_TYPE::LA_CONSTANT:
{
Ptr<const Layout> pLayout = static_cast<const Layout*>(GetValue().get());
check(pLayout);
if (pLayout)
{
int relId = pLayout->FindBlock(BlockId);
if (relId >= 0)
{
*BlockX = pLayout->Blocks[relId].Size[0];
*BlockY = pLayout->Blocks[relId].Size[1];
}
else
{
*BlockX = 0;
*BlockY = 0;
}
}
break;
}
default:
check(false);
}
}
//-------------------------------------------------------------------------------------------------
void ASTOpConstantResource::GetLayoutBlockSize(int32* pBlockX, int32* pBlockY)
{
switch (Type)
{
case OP_TYPE::IM_CONSTANT:
{
// We didn't find any layout.
*pBlockX = 0;
*pBlockY = 0;
break;
}
default:
checkf(false, TEXT("Instruction not supported"));
}
}
//-------------------------------------------------------------------------------------------------
bool ASTOpConstantResource::GetNonBlackRect(FImageRect& maskUsage) const
{
if (Type == OP_TYPE::IM_CONSTANT)
{
// TODO: cache
Ptr<const Image> pMask = static_cast<const Image*>(GetValue().get());
pMask->GetNonBlackRect(maskUsage);
return true;
}
return false;
}
//-------------------------------------------------------------------------------------------------
bool ASTOpConstantResource::IsImagePlainConstant(FVector4f& colour) const
{
bool res = false;
switch (Type)
{
case OP_TYPE::IM_CONSTANT:
{
Ptr<const Image> pImage = static_cast<const Image*>(GetValue().get());
if (pImage->GetSizeX() <= 0 || pImage->GetSizeY() <= 0)
{
res = true;
colour = FVector4f(0.0f,0.0f,0.0f,1.0f);
}
else if (pImage->m_flags & Image::IF_IS_PLAIN_COLOUR_VALID)
{
if (pImage->m_flags & Image::IF_IS_PLAIN_COLOUR)
{
res = true;
colour = pImage->Sample(FVector2f(0, 0));
}
else
{
res = false;
}
}
else
{
if (pImage->IsPlainColour(colour))
{
res = true;
pImage->m_flags |= Image::IF_IS_PLAIN_COLOUR;
}
pImage->m_flags |= Image::IF_IS_PLAIN_COLOUR_VALID;
}
break;
}
default:
break;
}
return res;
}
//-------------------------------------------------------------------------------------------------
ASTOpConstantResource::~ASTOpConstantResource()
{
}
//-------------------------------------------------------------------------------------------------
uint64 ASTOpConstantResource::GetValueHash() const
{
return ValueHash;
}
//-------------------------------------------------------------------------------------------------
mu::Ptr<const RefCounted> ASTOpConstantResource::GetValue() const
{
if (LoadedValue)
{
return LoadedValue;
}
else
{
switch (Type)
{
case OP_TYPE::IM_CONSTANT:
{
Ptr<ResourceProxy<Image>> typedProxy = static_cast<ResourceProxy<Image>*>(Proxy.get());
Ptr<const Image> r = typedProxy->Get();
return r;
}
default:
check(false);
break;
}
}
return nullptr;
}
//-------------------------------------------------------------------------------------------------
void ASTOpConstantResource::SetValue(const Ptr<const RefCounted>& v, FProxyFileContext* DiskCacheContext)
{
MUTABLE_CPUPROFILER_SCOPE(ASTOpConstantResource_SetValue);
switch (Type)
{
case OP_TYPE::IM_CONSTANT:
{
Ptr<const Image> r = static_cast<const Image*>(v.get());
OutputHashStream stream;
{
MUTABLE_CPUPROFILER_SCOPE(Serialize);
OutputArchive arch(&stream);
Image::Serialise(r.get(), arch);
}
ValueHash = stream.GetHash();
if (DiskCacheContext)
{
Proxy = new ResourceProxyTempFile<Image>(r.get(), *DiskCacheContext);
}
else
{
LoadedValue = r;
}
break;
}
case OP_TYPE::ME_CONSTANT:
{
Ptr<const Mesh> r = static_cast<const Mesh*>(v.get());
OutputHashStream stream;
{
MUTABLE_CPUPROFILER_SCOPE(Serialize);
OutputArchive arch(&stream);
Mesh::Serialise(r.get(), arch);
}
ValueHash = stream.GetHash();
LoadedValue = v;
break;
}
case OP_TYPE::LA_CONSTANT:
{
Ptr<const Layout> r = static_cast<const Layout*>(v.get());
OutputHashStream stream;
{
MUTABLE_CPUPROFILER_SCOPE(Serialize);
OutputArchive arch(&stream);
Layout::Serialise(r.get(), arch);
}
ValueHash = stream.GetHash();
LoadedValue = v;
break;
}
default:
LoadedValue = v;
break;
}
}
mu::Ptr<ImageSizeExpression> ASTOpConstantResource::GetImageSizeExpression() const
{
if (Type==OP_TYPE::IM_CONSTANT)
{
Ptr<ImageSizeExpression> pRes = new ImageSizeExpression;
pRes->type = ImageSizeExpression::ISET_CONSTANT;
Ptr<const Image> pConst = static_cast<const Image*>(GetValue().get());
pRes->size = pConst->GetSize();
return pRes;
}
return nullptr;
}
FSourceDataDescriptor ASTOpConstantResource::GetSourceDataDescriptor(FGetSourceDataDescriptorContext*) const
{
return SourceDataDescriptor;
}
}
Reference in New Issue View Git Blame Copy Permalink