izzy/UnrealEngineUWP
0
0
Fork 0
mirror of https://github.com/izzy2lost/UnrealEngineUWP.git synced 2026-03-26 18:15:20 -07:00
Code Issues Packages Projects Releases Wiki Activity

Change output of DDC2 texture build function to better organize structured data and bulk data into separate payloads.

Browse Source 
Remove older prototype DDC2 texture build code path and route the commandline switch it used to be synonymous with the newer path.
Fixed a bug where DDC2 produced texture mips didn't clear the "PagedToDerivedData" flag when they were streamed in causing the code to then go get the DDC1 texture payload.
Fixed a bug where the mip info provider wasn't accounting for the fact that DDC2 mip payloads aren't prefixed with a MipSize.

#rb devin.doucette
#preflight 613a369fbf5a71000167c318

#ROBOMERGE-AUTHOR: zousar.shaker
#ROBOMERGE-SOURCE: CL 17474647 in //UE5/Main/...
#ROBOMERGE-BOT: STARSHIP (Main -> Release-Engine-Test) (v870-17433530)

[CL 17474683 by zousar shaker in ue5-release-engine-test branch]
This commit is contained in:
zousar shaker
2021-09-09 14:17:19 -04:00
parent 7d8d668bf9
commit a443e34580
8 changed files with 247 additions and 357 deletions
Download Patch File Download Diff File Expand all files Collapse all files

145
Engine/Source/Developer/TextureBuild/Private/TextureBuildFunction.cpp
View File

@@ -15,6 +15,7 @@
#include "Modules/ModuleManager.h"
#include "PixelFormat.h"
#include "Serialization/CompactBinary.h"
#include "Serialization/CompactBinaryWriter.h"
#include "Serialization/FileRegions.h"
#include "Serialization/MemoryWriter.h"
#include "TextureCompressorModule.h"
@@ -26,7 +27,7 @@ DEFINE_LOG_CATEGORY_STATIC(LogTextureBuildFunction, Log, All);
// MUST have a corresponding change to this version. Individual texture formats have a version to
// change that is specific to the format. A merge conflict affecting the version MUST be resolved
// by generating a new version.
static const FGuid TextureDerivedDataVersion(TEXT("392f5367-8112-4305-8cef-ae7897844779"));
static const FGuid TextureDerivedDataVersion(TEXT("4af5c0e4-46cf-466b-a6f7-7b2b3dec8b10"));
#ifndef CASE_ENUM_TO_TEXT
#define CASE_ENUM_TO_TEXT(txt) case txt: return TEXT(#txt);
@@ -151,10 +152,9 @@ static FTextureBuildSettings ReadBuildSettingsFromCompactBinary(const FCbObjectV
return BuildSettings;
}
static void ReadOutputSettingsFromCompactBinary(const FCbObjectView& Object, int32& NumInlineMips, FString& MipKeyPrefix)
static void ReadOutputSettingsFromCompactBinary(const FCbObjectView& Object, int32& NumInlineMips)
{
NumInlineMips = Object["NumInlineMips"].AsInt32();
MipKeyPrefix = FUTF8ToTCHAR(Object["MipKeyPrefix"].AsString());
}
static ERawImageFormat::Type ComputeRawImageFormat(ETextureSourceFormat SourceFormat)
@@ -326,8 +326,7 @@ void FTextureBuildFunction::Build(UE::DerivedData::FBuildContext& Context) const
}
int32 NumInlineMips;
FString MipKeyPrefix;
ReadOutputSettingsFromCompactBinary(Settings["Output"].AsObjectView(), NumInlineMips, MipKeyPrefix);
ReadOutputSettingsFromCompactBinary(Settings["Output"].AsObjectView(), NumInlineMips);
TArray<FImage> SourceMips;
if (!TryReadTextureSourceFromCompactBinary(Settings["Source"], Context, SourceMips))
@@ -365,108 +364,82 @@ void FTextureBuildFunction::Build(UE::DerivedData::FBuildContext& Context) const
const bool bForceAllMipsToBeInlined = BuildSettings.bCubemap || (BuildSettings.bVolume && !BuildSettings.bStreamable) || (BuildSettings.bTextureArray && !BuildSettings.bStreamable);
const int32 FirstInlineMip = bForceAllMipsToBeInlined ? 0 : FMath::Max(0, MipCount - FMath::Max(NumInlineMips, (int32)NumMipsInTail));
// Write out texture platform data and non-streaming mip tail as a single payload
// This is meant to match the behavior in SerializePlatformData in TextureDerivedData.cpp.
// TODO: The texture header/footer information here should not be serialized from the worker but could be left to the editor to fill in.
// Doing it here causes more code duplication than I would like.
{
bool bHasOptData = (NumMipsInTail != 0) || (ExtData != 0);
TArray<FSharedBuffer> OrderedBuffers;
TArray<uint8> TextureHeaderArray;
FMemoryWriter TextureHeaderWriter(TextureHeaderArray, /*bIsPersistent=*/ true);
TextureHeaderWriter << CompressedMips[0].SizeX;
TextureHeaderWriter << CompressedMips[0].SizeY;
FCbWriter DescriptionWriter;
DescriptionWriter.BeginObject();
DescriptionWriter.BeginArray("Size"_ASV);
DescriptionWriter.AddInteger(CompressedMips[0].SizeX);
DescriptionWriter.AddInteger(CompressedMips[0].SizeY);
const int32 NumSlices = (BuildSettings.bVolume || BuildSettings.bTextureArray) ? CompressedMips[0].SizeZ : BuildSettings.bCubemap ? 6 : 1;
static constexpr uint32 BitMask_CubeMap = 1u << 31u;
static constexpr uint32 BitMask_HasOptData = 1u << 30u;
static constexpr uint32 BitMask_NumSlices = BitMask_HasOptData - 1u;
uint32 PackedData = (NumSlices & BitMask_NumSlices) | (BuildSettings.bCubemap ? BitMask_CubeMap : 0) | (bHasOptData ? BitMask_HasOptData : 0);
TextureHeaderWriter << PackedData;
DescriptionWriter.AddInteger(NumSlices);
DescriptionWriter.EndArray();
FString PixelFormatString(GetPixelFormatString((EPixelFormat)CompressedMips[0].PixelFormat));
TextureHeaderWriter << PixelFormatString;
DescriptionWriter.AddString("PixelFormat"_ASV, GetPixelFormatString((EPixelFormat)CompressedMips[0].PixelFormat));
DescriptionWriter.AddBool("bCubeMap"_ASV, BuildSettings.bCubemap);
DescriptionWriter.AddInteger("ExtData"_ASV, ExtData);
DescriptionWriter.AddInteger("NumMips"_ASV, MipCount);
DescriptionWriter.AddInteger("NumStreamingMips"_ASV, FirstInlineMip);
DescriptionWriter.AddInteger("NumMipsInTail"_ASV, NumMipsInTail);
if (bHasOptData)
{
TextureHeaderWriter << ExtData;
TextureHeaderWriter << NumMipsInTail;
}
TextureHeaderWriter << MipCount;
OrderedBuffers.Add(FSharedBuffer::MakeView(TextureHeaderArray.GetData(), TextureHeaderArray.Num()));
int64 CurrentOffset = TextureHeaderArray.Num();
DescriptionWriter.BeginArray("Mips"_ASV);
int64 MipPayloadOffset = 0;
for (int32 MipIndex = 0; MipIndex < MipCount; ++MipIndex)
{
bool bIsInlineMip = MipIndex >= FirstInlineMip;
FCompressedImage2D& CompressedMip = CompressedMips[MipIndex];
TArray<uint8> MipHeader;
FMemoryWriter MipHeaderWriter(MipHeader, /*bIsPersistent=*/ true);
bool bCooked = false;
MipHeaderWriter << bCooked;
DescriptionWriter.BeginObject();
DescriptionWriter.BeginArray("Size"_ASV);
DescriptionWriter.AddInteger(CompressedMip.SizeX);
DescriptionWriter.AddInteger(CompressedMip.SizeY);
DescriptionWriter.AddInteger(CompressedMip.SizeZ);
DescriptionWriter.EndArray();
uint32 BulkDataFlags = 0;
MipHeaderWriter << BulkDataFlags;
const bool bIsInlineMip = MipIndex >= FirstInlineMip;
int32 BulkDataElementCount = bIsInlineMip ? CompressedMip.RawData.Num() : 0;
MipHeaderWriter << BulkDataElementCount;
EFileRegionType FileRegion = FFileRegion::SelectType(EPixelFormat(CompressedMip.PixelFormat));
DescriptionWriter.AddInteger("FileRegion"_ASV, static_cast<int32>(FileRegion));
if (bIsInlineMip)
{
DescriptionWriter.AddInteger("PayloadOffset"_ASV, MipPayloadOffset);
}
DescriptionWriter.AddInteger("NumBytes"_ASV, CompressedMip.RawData.Num());
int32 BulkDataBytesOnDisk = bIsInlineMip ? CompressedMip.RawData.Num() : 0;
MipHeaderWriter << BulkDataBytesOnDisk;
int64 BulkDataOffset = CurrentOffset + MipHeader.Num() + sizeof(int64);
MipHeaderWriter << BulkDataOffset;
CurrentOffset += MipHeader.Num();
OrderedBuffers.Add(MakeSharedBufferFromArray(MoveTemp(MipHeader)));
DescriptionWriter.EndObject();
if (bIsInlineMip)
{
OrderedBuffers.Add(FSharedBuffer::MakeView(CompressedMip.RawData.GetData(), CompressedMip.RawData.Num()));
CurrentOffset += CompressedMip.RawData.Num();
MipPayloadOffset += CompressedMip.RawData.Num();
}
TArray<uint8> MipFooter;
FMemoryWriter MipFooterWriter(MipFooter, /*bIsPersistent=*/ true);
MipFooterWriter << CompressedMip.SizeX;
MipFooterWriter << CompressedMip.SizeY;
MipFooterWriter << CompressedMip.SizeZ;
EFileRegionType FileRegion = FFileRegion::SelectType(EPixelFormat(CompressedMip.PixelFormat));
MipFooterWriter << FileRegion;
bool bExistsInDerivedData = !bIsInlineMip;
MipFooterWriter << bExistsInDerivedData;
CurrentOffset += MipFooter.Num();
OrderedBuffers.Add(MakeSharedBufferFromArray(MoveTemp(MipFooter)));
}
DescriptionWriter.EndArray();
// Bool serialized as 32-bit int is the footer
uint32 IsVirtual = 0;
OrderedBuffers.Add(FSharedBuffer::MakeView(&IsVirtual, sizeof(IsVirtual)));
FCompositeBuffer CompositeResult(OrderedBuffers);
Context.AddPayload(UE::DerivedData::FPayloadId::FromName(TEXT("Texture"_SV)), CompositeResult);
DescriptionWriter.EndObject();
FCbObject DescriptionObject = DescriptionWriter.Save().AsObject();
Context.AddPayload(UE::DerivedData::FPayloadId::FromName("Description"_ASV), DescriptionObject);
}
// Write out streaming mips as payloads.
const int32 WritableMipCount = MipCount - ((NumMipsInTail > 0) ? ((int32)NumMipsInTail - 1) : 0);
for (int32 MipIndex = 0; MipIndex < WritableMipCount; ++MipIndex)
// Streaming mips
for (int32 MipIndex = 0; MipIndex < FirstInlineMip; ++MipIndex)
{
if (MipIndex < FirstInlineMip)
{
FCompressedImage2D& CompressedMip = CompressedMips[MipIndex];
TStringBuilder<32> PayloadName;
PayloadName << "Mip" << MipIndex;
TAnsiStringBuilder<16> MipName;
MipName << "Mip"_ASV << MipIndex;
FSharedBuffer MipData = MakeSharedBufferFromArray(MoveTemp(CompressedMip.RawData));
Context.AddPayload(UE::DerivedData::FPayloadId::FromName(*PayloadName), MipData);
}
FSharedBuffer MipData = MakeSharedBufferFromArray(MoveTemp(CompressedMips[MipIndex].RawData));
Context.AddPayload(UE::DerivedData::FPayloadId::FromName(MipName), MipData);
}
// Mip tail
TArray<FSharedBuffer> MipTailComponents;
for (int32 MipIndex = FirstInlineMip; MipIndex < MipCount; ++MipIndex)
{
FSharedBuffer MipData = MakeSharedBufferFromArray(MoveTemp(CompressedMips[MipIndex].RawData));
MipTailComponents.Add(MipData);
}
FCompositeBuffer MipTail(MipTailComponents);
if (MipTail.GetSize() > 0)
{
Context.AddPayload(UE::DerivedData::FPayloadId::FromName("MipTail"_ASV), MipTail);
}
}

12
Engine/Source/Runtime/Engine/Private/Streaming/Texture2DMipDataProvider_DDC.cpp
View File

@@ -123,11 +123,8 @@ void FTexture2DMipDataProvider_DDC::Init(const FTextureUpdateContext& Context, c
}
}
bool FTexture2DMipDataProvider_DDC::SerializeMipInfo(const FTextureUpdateContext& Context, FArchive& Ar, int32 MipIndex, const FTextureMipInfo& OutMipInfo)
bool FTexture2DMipDataProvider_DDC::SerializeMipInfo(const FTextureUpdateContext& Context, FArchive& Ar, int32 MipIndex, int32 MipSize, const FTextureMipInfo& OutMipInfo)
{
int32 MipSize = 0;
Ar << MipSize;
const uint32 DepthOrArraySize = FMath::Max<uint32>(OutMipInfo.ArraySize, OutMipInfo.SizeZ);
if (MipSize == OutMipInfo.DataSize)
{
@@ -187,7 +184,10 @@ int32 FTexture2DMipDataProvider_DDC::GetMips(
// The result must be read from a memory reader!
FMemoryReader Ar(DerivedMipData, true);
if (SerializeMipInfo(Context, Ar, MipIndex, MipInfo))
int32 MipSize = 0;
Ar << MipSize;
check(MipSize == (DerivedMipData.Num() - sizeof(int32)));
if (SerializeMipInfo(Context, Ar, MipIndex, MipSize, MipInfo))
{
bSuccess = true;
}
@@ -210,7 +210,7 @@ int32 FTexture2DMipDataProvider_DDC::GetMips(
{
// The result must be read from a memory reader!
FMemoryReaderView Ar(DDCBuffers[MipIndex], true);
if (SerializeMipInfo(Context, Ar, MipIndex, MipInfos[MipIndex]))
if (SerializeMipInfo(Context, Ar, MipIndex, DDCBuffers[MipIndex].GetSize(), MipInfos[MipIndex]))
{
bSuccess = true;
}

2
Engine/Source/Runtime/Engine/Private/Streaming/Texture2DMipDataProvider_DDC.h
View File

@@ -40,7 +40,7 @@ protected:
// Helper to route incomplete DDC requests to the FAbandonedDDCHandleManager.
void ReleaseDDCResources();
bool SerializeMipInfo(const FTextureUpdateContext& Context, FArchive& Ar, int32 MipIndex, const FTextureMipInfo& OutMipInfo);
bool SerializeMipInfo(const FTextureUpdateContext& Context, FArchive& Ar, int32 MipIndex, int32 MipSize, const FTextureMipInfo& OutMipInfo);
private:

3
Engine/Source/Runtime/Engine/Private/TextureDerivedData.cpp
View File

@@ -894,7 +894,7 @@ void FTexturePlatformData::CancelCache()
bool FTexturePlatformData::IsUsingNewDerivedData()
{
static const bool bUseNewDerivedData = FParse::Param(FCommandLine::Get(), TEXT("DDC2AsyncTextureBuilds"));
static const bool bUseNewDerivedData = FParse::Param(FCommandLine::Get(), TEXT("DDC2AsyncTextureBuilds")) || FParse::Param(FCommandLine::Get(), TEXT("DDC2TextureBuilds"));
return bUseNewDerivedData;
}
@@ -1056,6 +1056,7 @@ bool FTexturePlatformData::TryInlineMipData(int32 FirstMipToLoad, UTexture* Text
void* MipData = Mip.BulkData.Realloc(int64(MipBuffer.GetSize()));
FMemory::Memcpy(MipData, MipBuffer.GetData(), MipBuffer.GetSize());
Mip.BulkData.Unlock();
Mip.SetPagedToDerivedData(false);
}))
{
return false;

17
Engine/Source/Runtime/Engine/Private/TextureDerivedDataBuildUtils.cpp
View File

@@ -158,22 +158,11 @@ static void WriteBuildSettings(FCbWriter& Writer, const FTextureBuildSettings& B
Writer.EndObject();
}
static void WriteOutputSettings(FCbWriter& Writer, int32 NumInlineMips, const FString& KeySuffix)
static void WriteOutputSettings(FCbWriter& Writer, int32 NumInlineMips)
{
Writer.BeginObject();
Writer.AddInteger("NumInlineMips", NumInlineMips);
FString MipDerivedDataKey;
FTexture2DMipMap DummyMip;
DummyMip.SizeX = 0;
DummyMip.SizeY = 0;
GetTextureDerivedMipKey(0, DummyMip, KeySuffix, MipDerivedDataKey);
int32 PrefixEndIndex = MipDerivedDataKey.Find(TEXT("_MIP0_"), ESearchCase::CaseSensitive);
check(PrefixEndIndex != -1);
MipDerivedDataKey.LeftInline(PrefixEndIndex);
check(!MipDerivedDataKey.IsEmpty());
Writer.AddString("MipKeyPrefix",*MipDerivedDataKey);
Writer.EndObject();
}
@@ -237,7 +226,7 @@ bool TryFindTextureBuildFunction(FStringBuilderBase& OutFunctionName, const FTex
return UE::DerivedData::GetBuild().GetFunctionRegistry().FindFunctionVersion(FunctionName).IsValid();
}
FCbObject SaveTextureBuildSettings(const FString& KeySuffix, const UTexture& Texture, const FTextureBuildSettings& BuildSettings, int32 LayerIndex, int32 NumInlineMips)
FCbObject SaveTextureBuildSettings(const UTexture& Texture, const FTextureBuildSettings& BuildSettings, int32 LayerIndex, int32 NumInlineMips)
{
const ITextureFormat* TextureFormat = nullptr;
if (ITextureFormatManagerModule* TFM = GetTextureFormatManager())
@@ -265,7 +254,7 @@ FCbObject SaveTextureBuildSettings(const FString& KeySuffix, const UTexture& Tex
WriteBuildSettings(Writer, BuildSettings, TextureFormat);
Writer.SetName("Output");
WriteOutputSettings(Writer, NumInlineMips, KeySuffix);
WriteOutputSettings(Writer, NumInlineMips);
Writer.SetName("Source");
WriteSource(Writer, Texture, LayerIndex);

2
Engine/Source/Runtime/Engine/Private/TextureDerivedDataBuildUtils.h
View File

@@ -13,6 +13,6 @@ class UTexture;
struct FTextureBuildSettings;
bool TryFindTextureBuildFunction(FStringBuilderBase& OutFunctionName, const FTextureBuildSettings& BuildSettings);
FCbObject SaveTextureBuildSettings(const FString& KeySuffix, const UTexture& Texture, const FTextureBuildSettings& BuildSettings, int32 LayerIndex, int32 NumInlineMips);
FCbObject SaveTextureBuildSettings(const UTexture& Texture, const FTextureBuildSettings& BuildSettings, int32 LayerIndex, int32 NumInlineMips);
#endif // WITH_EDITOR

419
Engine/Source/Runtime/Engine/Private/TextureDerivedDataTask.cpp
View File

File diff suppressed because it is too large Load Diff

4
Engine/Source/Runtime/Engine/Private/TextureDerivedDataTask.h
View File

@@ -124,8 +124,6 @@ class FTextureCacheDerivedDataWorker : public FNonAbandonableTask
FTextureSourceData TextureData;
/** Source mip images of the composite texture (e.g. normal map for compute roughness). Not necessarily in RGBA32F, usually only top mip as other mips need to be generated first */
FTextureSourceData CompositeTextureData;
/** DDC2 build function name to use to build this texture (if DDC2 is enabled and the target texture type has a DDC2 build function, empty otherwise) */
FString BuildFunctionName;
/** Texture cache flags. */
ETextureCacheFlags CacheFlags;
/** Have many bytes were loaded from DDC or built (for telemetry) */
@@ -140,8 +138,6 @@ class FTextureCacheDerivedDataWorker : public FNonAbandonableTask
/** Build the texture. This function is safe to call from any thread. */
void BuildTexture(bool bReplaceExistingDDC = false);
void ConsumeBuildFunctionOutput(const UE::DerivedData::FBuildOutput& BuildOutput, const FString& TexturePath, bool bReplaceExistingDDC);
public:
/** Initialization constructor. */