You've already forked UnrealEngineUWP
mirror of
https://github.com/izzy2lost/UnrealEngineUWP.git
synced 2026-03-26 18:15:20 -07:00
109bb67a95
#rb matt.peters #preflight 61f1997573238441cb66c9ea #ROBOMERGE-AUTHOR: zousar.shaker #ROBOMERGE-SOURCE: CL 18740488 in //UE5/Release-5.0/... via CL 18740829 via CL 18742806 #ROBOMERGE-BOT: UE5 (Release-Engine-Test -> Main) (v903-18687472) [CL 18743180 by zousar shaker in ue5-main branch]
1181 lines
42 KiB
C++
1181 lines
42 KiB
C++
// Copyright Epic Games, Inc. All Rights Reserved.
|
|
|
|
#include "DerivedDataCache.h"
|
|
#include "DerivedDataCacheInterface.h"
|
|
|
|
#include "Algo/Accumulate.h"
|
|
#include "Algo/AllOf.h"
|
|
#include "Algo/BinarySearch.h"
|
|
#include "Algo/Sort.h"
|
|
#include "Async/AsyncWork.h"
|
|
#include "Async/TaskGraphInterfaces.h"
|
|
#include "Containers/Map.h"
|
|
#include "Containers/StringConv.h"
|
|
#include "DDCCleanup.h"
|
|
#include "DerivedDataBackendInterface.h"
|
|
#include "DerivedDataCache.h"
|
|
#include "DerivedDataCacheMaintainer.h"
|
|
#include "DerivedDataCachePrivate.h"
|
|
#include "DerivedDataCacheUsageStats.h"
|
|
#include "DerivedDataPluginInterface.h"
|
|
#include "DerivedDataRequestOwner.h"
|
|
#include "Features/IModularFeatures.h"
|
|
#include "HAL/ThreadSafeCounter.h"
|
|
#include "Misc/CommandLine.h"
|
|
#include "Misc/CoreMisc.h"
|
|
#include "Misc/ScopeLock.h"
|
|
#include "ProfilingDebugging/CookStats.h"
|
|
#include "Serialization/CompactBinary.h"
|
|
#include "Serialization/CompactBinaryWriter.h"
|
|
#include "Stats/Stats.h"
|
|
#include "Stats/StatsMisc.h"
|
|
#include "String/ParseTokens.h"
|
|
#include "ZenServerInterface.h"
|
|
#include <atomic>
|
|
|
|
DEFINE_STAT(STAT_DDC_NumGets);
|
|
DEFINE_STAT(STAT_DDC_NumPuts);
|
|
DEFINE_STAT(STAT_DDC_NumBuilds);
|
|
DEFINE_STAT(STAT_DDC_NumExist);
|
|
DEFINE_STAT(STAT_DDC_SyncGetTime);
|
|
DEFINE_STAT(STAT_DDC_ASyncWaitTime);
|
|
DEFINE_STAT(STAT_DDC_PutTime);
|
|
DEFINE_STAT(STAT_DDC_SyncBuildTime);
|
|
DEFINE_STAT(STAT_DDC_ExistTime);
|
|
|
|
//#define DDC_SCOPE_CYCLE_COUNTER(x) QUICK_SCOPE_CYCLE_COUNTER(STAT_ ## x)
|
|
#define DDC_SCOPE_CYCLE_COUNTER(x) TRACE_CPUPROFILER_EVENT_SCOPE(x);
|
|
|
|
#if ENABLE_COOK_STATS
|
|
#include "DerivedDataCacheUsageStats.h"
|
|
namespace UE::DerivedData::CookStats
|
|
{
|
|
// Use to prevent potential divide by zero issues
|
|
inline double SafeDivide(const int64 Numerator, const int64 Denominator)
|
|
{
|
|
return Denominator != 0 ? (double)Numerator / (double)Denominator : 0.0;
|
|
}
|
|
|
|
// AddCookStats cannot be a lambda because of false positives in static analysis.
|
|
// See https://developercommunity.visualstudio.com/content/problem/576913/c6244-regression-in-new-lambda-processorpermissive.html
|
|
static void AddCookStats(FCookStatsManager::AddStatFuncRef AddStat)
|
|
{
|
|
PRAGMA_DISABLE_DEPRECATION_WARNINGS;
|
|
TSharedRef<FDerivedDataCacheStatsNode> RootNode = GetDerivedDataCacheRef().GatherUsageStats();
|
|
PRAGMA_ENABLE_DEPRECATION_WARNINGS;
|
|
|
|
{
|
|
const FString StatName(TEXT("DDC.Usage"));
|
|
for (const auto& UsageStatPair : RootNode->ToLegacyUsageMap())
|
|
{
|
|
UsageStatPair.Value.LogStats(AddStat, StatName, UsageStatPair.Key);
|
|
}
|
|
}
|
|
|
|
TArray<TSharedRef<const FDerivedDataCacheStatsNode>> Nodes;
|
|
RootNode->ForEachDescendant([&Nodes](TSharedRef<const FDerivedDataCacheStatsNode> Node)
|
|
{
|
|
if (Node->Children.IsEmpty())
|
|
{
|
|
Nodes.Add(Node);
|
|
}
|
|
});
|
|
|
|
// Now lets add some summary data to that applies some crazy knowledge of how we set up our DDC. The goal
|
|
// is to print out the global hit rate, and the hit rate of the local and shared DDC.
|
|
// This is done by adding up the total get/miss calls the root node receives.
|
|
// Then we find the FileSystem nodes that correspond to the local and shared cache using some hacky logic to detect a "network drive".
|
|
// If the DDC graph ever contains more than one local or remote filesystem, this will only find one of them.
|
|
{
|
|
const TSharedRef<const FDerivedDataCacheStatsNode>* LocalNode = Nodes.FindByPredicate([](TSharedRef<const FDerivedDataCacheStatsNode> Node) { return Node->GetCacheType() == TEXT("File System") && Node->IsLocal(); });
|
|
const TSharedRef<const FDerivedDataCacheStatsNode>* SharedNode = Nodes.FindByPredicate([](TSharedRef<const FDerivedDataCacheStatsNode> Node) { return Node->GetCacheType() == TEXT("File System") && !Node->IsLocal(); });
|
|
const TSharedRef<const FDerivedDataCacheStatsNode>* CloudNode = Nodes.FindByPredicate([](TSharedRef<const FDerivedDataCacheStatsNode> Node) { return Node->GetCacheType() == TEXT("Horde Storage"); });
|
|
const TSharedRef<const FDerivedDataCacheStatsNode>* ZenLocalNode = Nodes.FindByPredicate([](TSharedRef<const FDerivedDataCacheStatsNode> Node) { return Node->GetCacheType() == TEXT("Zen") && Node->IsLocal(); });
|
|
const TSharedRef<const FDerivedDataCacheStatsNode>* ZenRemoteNode = Nodes.FindByPredicate([](TSharedRef<const FDerivedDataCacheStatsNode> Node) { return (Node->GetCacheType() == TEXT("Zen") || Node->GetCacheType() == TEXT("Horde")) && !Node->IsLocal(); });
|
|
|
|
const FDerivedDataCacheUsageStats& RootStats = RootNode->Stats.CreateConstIterator().Value();
|
|
const int64 TotalGetHits = RootStats.GetStats.GetAccumulatedValueAnyThread(FCookStats::CallStats::EHitOrMiss::Hit, FCookStats::CallStats::EStatType::Counter);
|
|
const int64 TotalGetMisses = RootStats.GetStats.GetAccumulatedValueAnyThread(FCookStats::CallStats::EHitOrMiss::Miss, FCookStats::CallStats::EStatType::Counter);
|
|
const int64 TotalGets = TotalGetHits + TotalGetMisses;
|
|
|
|
int64 LocalHits = 0;
|
|
if (LocalNode)
|
|
{
|
|
const FDerivedDataCacheUsageStats& Stats = (*LocalNode)->Stats.CreateConstIterator().Value();
|
|
LocalHits += Stats.GetStats.GetAccumulatedValueAnyThread(FCookStats::CallStats::EHitOrMiss::Hit, FCookStats::CallStats::EStatType::Counter);
|
|
}
|
|
if (ZenLocalNode)
|
|
{
|
|
const FDerivedDataCacheUsageStats& Stats = (*ZenLocalNode)->Stats.CreateConstIterator().Value();
|
|
LocalHits += Stats.GetStats.GetAccumulatedValueAnyThread(FCookStats::CallStats::EHitOrMiss::Hit, FCookStats::CallStats::EStatType::Counter);
|
|
}
|
|
int64 SharedHits = 0;
|
|
if (SharedNode)
|
|
{
|
|
// The shared DDC is only queried if the local one misses (or there isn't one). So it's hit rate is technically
|
|
const FDerivedDataCacheUsageStats& Stats = (*SharedNode)->Stats.CreateConstIterator().Value();
|
|
SharedHits += Stats.GetStats.GetAccumulatedValueAnyThread(FCookStats::CallStats::EHitOrMiss::Hit, FCookStats::CallStats::EStatType::Counter);
|
|
}
|
|
if (ZenRemoteNode)
|
|
{
|
|
const FDerivedDataCacheUsageStats& Stats = (*ZenRemoteNode)->Stats.CreateConstIterator().Value();
|
|
SharedHits += Stats.GetStats.GetAccumulatedValueAnyThread(FCookStats::CallStats::EHitOrMiss::Hit, FCookStats::CallStats::EStatType::Counter);
|
|
}
|
|
int64 CloudHits = 0;
|
|
if (CloudNode)
|
|
{
|
|
const FDerivedDataCacheUsageStats& Stats = (*CloudNode)->Stats.CreateConstIterator().Value();
|
|
CloudHits += Stats.GetStats.GetAccumulatedValueAnyThread(FCookStats::CallStats::EHitOrMiss::Hit, FCookStats::CallStats::EStatType::Counter);
|
|
}
|
|
|
|
const int64 TotalPutHits = RootStats.PutStats.GetAccumulatedValueAnyThread(FCookStats::CallStats::EHitOrMiss::Hit, FCookStats::CallStats::EStatType::Counter);
|
|
const int64 TotalPutMisses = RootStats.PutStats.GetAccumulatedValueAnyThread(FCookStats::CallStats::EHitOrMiss::Miss, FCookStats::CallStats::EStatType::Counter);
|
|
const int64 TotalPuts = TotalPutHits + TotalPutMisses;
|
|
|
|
AddStat(TEXT("DDC.Summary"), FCookStatsManager::CreateKeyValueArray(
|
|
TEXT("BackEnd"), FDerivedDataBackend::Get().GetGraphName(),
|
|
TEXT("HasLocalCache"), LocalNode || ZenLocalNode,
|
|
TEXT("HasSharedCache"), SharedNode || ZenRemoteNode,
|
|
TEXT("HasCloudCache"), !!CloudNode,
|
|
TEXT("HasZenCache"), ZenLocalNode || ZenRemoteNode,
|
|
TEXT("TotalGetHits"), TotalGetHits,
|
|
TEXT("TotalGets"), TotalGets,
|
|
TEXT("TotalGetHitPct"), SafeDivide(TotalGetHits, TotalGets),
|
|
TEXT("LocalGetHitPct"), SafeDivide(LocalHits, TotalGets),
|
|
TEXT("SharedGetHitPct"), SafeDivide(SharedHits, TotalGets),
|
|
TEXT("CloudGetHitPct"), SafeDivide(CloudHits, TotalGets),
|
|
TEXT("OtherGetHitPct"), SafeDivide((TotalGetHits - LocalHits - SharedHits - CloudHits), TotalGets),
|
|
TEXT("GetMissPct"), SafeDivide(TotalGetMisses, TotalGets),
|
|
TEXT("TotalPutHits"), TotalPutHits,
|
|
TEXT("TotalPuts"), TotalPuts,
|
|
TEXT("TotalPutHitPct"), SafeDivide(TotalPutHits, TotalPuts),
|
|
TEXT("PutMissPct"), SafeDivide(TotalPutMisses, TotalPuts)
|
|
));
|
|
}
|
|
}
|
|
|
|
FCookStatsManager::FAutoRegisterCallback RegisterCookStats(AddCookStats);
|
|
}
|
|
#endif
|
|
|
|
void GatherDerivedDataCacheResourceStats(TArray<FDerivedDataCacheResourceStat>& DDCResourceStats);
|
|
void GatherDerivedDataCacheSummaryStats(FDerivedDataCacheSummaryStats& DDCSummaryStats);
|
|
|
|
/** Whether we want to verify the DDC (pass in -VerifyDDC on the command line)*/
|
|
bool GVerifyDDC = false;
|
|
|
|
namespace UE::DerivedData::Private { class FCacheRecordPolicyShared; }
|
|
|
|
namespace UE::DerivedData
|
|
{
|
|
|
|
namespace Private::CachePolicy
|
|
{
|
|
// TODO: Implement these as Ansi String instead to maximize the most-optimal path, see ParseCachePolicyImpl
|
|
constexpr ANSICHAR DelimiterChar = ',';
|
|
constexpr FStringView Delimiter = TEXTVIEW(",");
|
|
constexpr FStringView None = TEXTVIEW("None");
|
|
constexpr FStringView QueryLocal = TEXTVIEW("QueryLocal");
|
|
constexpr FStringView QueryRemote = TEXTVIEW("QueryRemote");
|
|
constexpr FStringView Query = TEXTVIEW("Query");
|
|
constexpr FStringView StoreLocal = TEXTVIEW("StoreLocal");
|
|
constexpr FStringView StoreRemote = TEXTVIEW("StoreRemote");
|
|
constexpr FStringView Store = TEXTVIEW("Store");
|
|
constexpr FStringView SkipMeta = TEXTVIEW("SkipMeta");
|
|
constexpr FStringView SkipData = TEXTVIEW("SkipData");
|
|
constexpr FStringView PartialRecord = TEXTVIEW("PartialRecord");
|
|
constexpr FStringView KeepAlive = TEXTVIEW("KeepAlive");
|
|
constexpr FStringView Local = TEXTVIEW("Local");
|
|
constexpr FStringView Remote = TEXTVIEW("Remote");
|
|
constexpr FStringView Default = TEXTVIEW("Default");
|
|
constexpr FStringView Disable = TEXTVIEW("Disable");
|
|
|
|
const TMap<FStringView, ECachePolicy> TextToPolicy
|
|
{
|
|
{None, ECachePolicy::None},
|
|
{QueryLocal, ECachePolicy::QueryLocal},
|
|
{QueryRemote, ECachePolicy::QueryRemote},
|
|
{Query, ECachePolicy::Query},
|
|
{StoreLocal, ECachePolicy::StoreLocal},
|
|
{StoreRemote, ECachePolicy::StoreRemote},
|
|
{Store, ECachePolicy::Store},
|
|
{SkipMeta, ECachePolicy::SkipMeta},
|
|
{SkipData, ECachePolicy::SkipData},
|
|
{PartialRecord, ECachePolicy::PartialRecord},
|
|
{KeepAlive, ECachePolicy::KeepAlive},
|
|
{Local, ECachePolicy::Local},
|
|
{Remote, ECachePolicy::Remote},
|
|
{Default, ECachePolicy::Default},
|
|
{Disable, ECachePolicy::Disable}
|
|
};
|
|
|
|
using FPolicyTextPair = TPair<ECachePolicy, FStringView>;
|
|
const FPolicyTextPair FlagsToString[]
|
|
{
|
|
// Order of these Flags is important: we want the aliases before the atomic values,
|
|
// and the bigger aliases first, to reduce the number of tokens we add
|
|
{ ECachePolicy::Default, Default },
|
|
{ ECachePolicy::Remote, Remote },
|
|
{ ECachePolicy::Local, Local },
|
|
{ ECachePolicy::Store, Store },
|
|
{ ECachePolicy::Query, Query },
|
|
|
|
// Order of Atomics doesn't matter, so arbitrarily we list them in enum order
|
|
{ ECachePolicy::QueryLocal, QueryLocal },
|
|
{ ECachePolicy::QueryRemote, QueryRemote },
|
|
{ ECachePolicy::StoreLocal, StoreLocal },
|
|
{ ECachePolicy::StoreRemote, StoreRemote },
|
|
{ ECachePolicy::SkipMeta, SkipMeta },
|
|
{ ECachePolicy::SkipData, SkipData },
|
|
{ ECachePolicy::PartialRecord, PartialRecord },
|
|
{ ECachePolicy::KeepAlive, KeepAlive },
|
|
|
|
// None must come at the end of the array, to write out only if no others exist
|
|
{ ECachePolicy::None, None },
|
|
};
|
|
constexpr ECachePolicy KnownFlags = ECachePolicy::Default | ECachePolicy::SkipMeta | ECachePolicy::SkipData
|
|
| ECachePolicy::KeepAlive | ECachePolicy::PartialRecord;
|
|
|
|
} // namespace Private::CachePolicy
|
|
|
|
template <typename CharType>
|
|
TStringBuilderBase<CharType>& AppendToBuilderImpl(
|
|
TStringBuilderBase<CharType>& Builder, UE::DerivedData::ECachePolicy Policy)
|
|
{
|
|
// Remove any bits we don't recognize; write None if there are not any bits we recognize
|
|
Policy = Policy & Private::CachePolicy::KnownFlags;
|
|
for (const Private::CachePolicy::FPolicyTextPair& Pair : Private::CachePolicy::FlagsToString)
|
|
{
|
|
if (EnumHasAllFlags(Policy, Pair.Key))
|
|
{
|
|
EnumRemoveFlags(Policy, Pair.Key);
|
|
Builder << Pair.Value << Private::CachePolicy::DelimiterChar;
|
|
if (Policy == ECachePolicy::None)
|
|
{
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
Builder.RemoveSuffix(1); // Text will have been added by ECachePolicy::None if not by anything else
|
|
return Builder;
|
|
}
|
|
FAnsiStringBuilderBase& operator<<(FAnsiStringBuilderBase& Builder, UE::DerivedData::ECachePolicy Policy)
|
|
{
|
|
return AppendToBuilderImpl(Builder, Policy);
|
|
}
|
|
FUtf8StringBuilderBase& operator<<(FUtf8StringBuilderBase& Builder, UE::DerivedData::ECachePolicy Policy)
|
|
{
|
|
return AppendToBuilderImpl(Builder, Policy);
|
|
}
|
|
FWideStringBuilderBase& operator<<(FWideStringBuilderBase& Builder, UE::DerivedData::ECachePolicy Policy)
|
|
{
|
|
return AppendToBuilderImpl(Builder, Policy);
|
|
}
|
|
|
|
template <typename CharType>
|
|
ECachePolicy ParseCachePolicyImpl(TStringView<CharType> Text)
|
|
{
|
|
checkf(!Text.IsEmpty(), TEXT("Empty string is not valid input to ParseCachePolicy"));
|
|
|
|
ECachePolicy Result = ECachePolicy::None;
|
|
// TODO: Implement ParseTokens for FAnsiStringView so we can convert to Ansi instead of Wide
|
|
auto WideText = StringCast<TCHAR, 128>(Text.GetData(), Text.Len());
|
|
UE::String::ParseTokens(WideText, Private::CachePolicy::DelimiterChar, [&Result](FStringView Token) {
|
|
const ECachePolicy* TokenPolicy = Private::CachePolicy::TextToPolicy.Find(Token);
|
|
if (TokenPolicy)
|
|
{
|
|
Result |= *TokenPolicy;
|
|
}
|
|
});
|
|
|
|
return Result;
|
|
}
|
|
ECachePolicy ParseCachePolicy(FAnsiStringView Text)
|
|
{
|
|
return ParseCachePolicyImpl(Text);
|
|
}
|
|
ECachePolicy ParseCachePolicy(FUtf8StringView Text)
|
|
{
|
|
return ParseCachePolicyImpl(Text);
|
|
}
|
|
ECachePolicy ParseCachePolicy(FWideStringView Text)
|
|
{
|
|
return ParseCachePolicyImpl(Text);
|
|
}
|
|
|
|
|
|
class Private::FCacheRecordPolicyShared final : public Private::ICacheRecordPolicyShared
|
|
{
|
|
public:
|
|
inline void AddRef() const final
|
|
{
|
|
ReferenceCount.fetch_add(1, std::memory_order_relaxed);
|
|
}
|
|
|
|
inline void Release() const final
|
|
{
|
|
if (ReferenceCount.fetch_sub(1, std::memory_order_acq_rel) == 1)
|
|
{
|
|
delete this;
|
|
}
|
|
}
|
|
|
|
inline TConstArrayView<FCacheValuePolicy> GetValuePolicies() const final
|
|
{
|
|
return Values;
|
|
}
|
|
|
|
inline void AddValuePolicy(const FCacheValuePolicy& Policy) final
|
|
{
|
|
Values.Add(Policy);
|
|
}
|
|
|
|
inline void Build() final
|
|
{
|
|
Algo::SortBy(Values, &FCacheValuePolicy::Id);
|
|
}
|
|
|
|
private:
|
|
TArray<FCacheValuePolicy, TInlineAllocator<14>> Values;
|
|
mutable std::atomic<uint32> ReferenceCount{0};
|
|
};
|
|
|
|
ECachePolicy FCacheRecordPolicy::GetValuePolicy(const FValueId& Id) const
|
|
{
|
|
if (Shared)
|
|
{
|
|
if (TConstArrayView<FCacheValuePolicy> Values = Shared->GetValuePolicies(); !Values.IsEmpty())
|
|
{
|
|
if (int32 Index = Algo::BinarySearchBy(Values, Id, &FCacheValuePolicy::Id); Index != INDEX_NONE)
|
|
{
|
|
return Values[Index].Policy;
|
|
}
|
|
}
|
|
}
|
|
return DefaultValuePolicy;
|
|
}
|
|
|
|
FCacheRecordPolicy FCacheRecordPolicy::Transform(TFunctionRef<ECachePolicy (ECachePolicy)> Op) const
|
|
{
|
|
if (IsUniform())
|
|
{
|
|
return Op(RecordPolicy);
|
|
}
|
|
FCacheRecordPolicyBuilder Builder(Op(DefaultValuePolicy));
|
|
for (const FCacheValuePolicy& Value : GetValuePolicies())
|
|
{
|
|
Builder.AddValuePolicy({Value.Id, Op(Value.Policy)});
|
|
}
|
|
return Builder.Build();
|
|
}
|
|
|
|
void FCacheRecordPolicy::Save(FCbWriter& Writer) const
|
|
{
|
|
Writer.BeginObject();
|
|
{
|
|
// The RecordPolicy is calculated from the ValuePolicies and does not need to be saved separately.
|
|
Writer << "DefaultValuePolicy"_ASV << WriteToUtf8String<128>(GetDefaultValuePolicy());
|
|
if (!IsUniform())
|
|
{
|
|
// FCacheRecordPolicyBuilder guarantees IsUniform -> non-empty GetValuePolicies. Small size penalty here if not.
|
|
Writer.BeginArray("ValuePolicies"_ASV);
|
|
{
|
|
for (const FCacheValuePolicy& ValuePolicy : GetValuePolicies())
|
|
{
|
|
// FCacheRecordPolicyBuilder is responsible for ensuring that each ValuePolicy != DefaultValuePolicy
|
|
// If it lets any duplicates through we will incur a small serialization size penalty here
|
|
Writer.BeginObject();
|
|
Writer << "Id"_ASV << ValuePolicy.Id;
|
|
Writer << "Policy"_ASV << WriteToUtf8String<128>(ValuePolicy.Policy);
|
|
Writer.EndObject();
|
|
}
|
|
}
|
|
Writer.EndArray();
|
|
}
|
|
}
|
|
Writer.EndObject();
|
|
}
|
|
|
|
FCacheRecordPolicy FCacheRecordPolicy::Load(FCbObjectView Object, ECachePolicy DefaultPolicy)
|
|
{
|
|
FUtf8StringView PolicyText = Object["DefaultValuePolicy"_ASV].AsString();
|
|
ECachePolicy DefaultValuePolicy = !PolicyText.IsEmpty() ? ParseCachePolicy(PolicyText) : DefaultPolicy;
|
|
|
|
FCacheRecordPolicyBuilder Builder(DefaultValuePolicy);
|
|
for (FCbFieldView ValueObjectField : Object["ValuePolicies"_ASV])
|
|
{
|
|
FCbObjectView ValueObject = ValueObjectField.AsObjectView();
|
|
const FCbObjectId ValueId = ValueObject["Id"_ASV].AsObjectId();
|
|
PolicyText = ValueObject["Policy"_ASV].AsString();
|
|
ECachePolicy ValuePolicy = !PolicyText.IsEmpty() ? ParseCachePolicy(PolicyText) : DefaultValuePolicy;
|
|
// FCacheRecordPolicyBuilder should guarantee that FValueId(ValueId).IsValid and ValuePolicy != DefaultValuePolicy
|
|
// If it lets any through we will have unused data in the record we create.
|
|
Builder.AddValuePolicy(ValueId, ValuePolicy);
|
|
}
|
|
|
|
return Builder.Build();
|
|
}
|
|
|
|
void FCacheRecordPolicyBuilder::AddValuePolicy(const FCacheValuePolicy& Policy)
|
|
{
|
|
if (!Shared)
|
|
{
|
|
Shared = new Private::FCacheRecordPolicyShared;
|
|
}
|
|
Shared->AddValuePolicy(Policy);
|
|
}
|
|
|
|
FCacheRecordPolicy FCacheRecordPolicyBuilder::Build()
|
|
{
|
|
FCacheRecordPolicy Policy(BasePolicy);
|
|
if (Shared)
|
|
{
|
|
Shared->Build();
|
|
const auto PolicyOr = [](ECachePolicy A, ECachePolicy B) { return A | (B & ~ECachePolicy::SkipData); };
|
|
const TConstArrayView<FCacheValuePolicy> Values = Shared->GetValuePolicies();
|
|
Policy.RecordPolicy = Algo::TransformAccumulate(Values, &FCacheValuePolicy::Policy, BasePolicy, PolicyOr);
|
|
Policy.Shared = MoveTemp(Shared);
|
|
}
|
|
return Policy;
|
|
}
|
|
|
|
void ICacheStore::PutValue(
|
|
const TConstArrayView<FCachePutValueRequest> Requests,
|
|
IRequestOwner& Owner,
|
|
FOnCachePutValueComplete&& OnComplete)
|
|
{
|
|
if (OnComplete)
|
|
{
|
|
for (const FCachePutValueRequest& Request : Requests)
|
|
{
|
|
OnComplete({Request.Name, Request.Key, Request.UserData, EStatus::Error});
|
|
}
|
|
}
|
|
}
|
|
|
|
void ICacheStore::GetValue(
|
|
const TConstArrayView<FCacheGetValueRequest> Requests,
|
|
IRequestOwner& Owner,
|
|
FOnCacheGetValueComplete&& OnComplete)
|
|
{
|
|
if (OnComplete)
|
|
{
|
|
for (const FCacheGetValueRequest& Request : Requests)
|
|
{
|
|
OnComplete({Request.Name, Request.Key, {}, Request.UserData, EStatus::Error});
|
|
}
|
|
}
|
|
}
|
|
|
|
} // UE::DerivedData
|
|
|
|
namespace UE::DerivedData::Private
|
|
{
|
|
|
|
FQueuedThreadPool* GCacheThreadPool;
|
|
|
|
/**
|
|
* Implementation of the derived data cache
|
|
* This API is fully threadsafe
|
|
**/
|
|
class FDerivedDataCache final
|
|
: public FDerivedDataCacheInterface
|
|
, public ICache
|
|
, public ICacheStoreMaintainer
|
|
, public IDDCCleanup
|
|
{
|
|
|
|
/**
|
|
* Async worker that checks the cache backend and if that fails, calls the deriver to build the data and then puts the results to the cache
|
|
**/
|
|
friend class FBuildAsyncWorker;
|
|
class FBuildAsyncWorker : public FNonAbandonableTask
|
|
{
|
|
public:
|
|
enum EWorkerState : uint32
|
|
{
|
|
WorkerStateNone = 0,
|
|
WorkerStateRunning = 1 << 0,
|
|
WorkerStateFinished = 1 << 1,
|
|
WorkerStateDestroyed = 1 << 2,
|
|
};
|
|
|
|
/**
|
|
* Constructor for async task
|
|
* @param InDataDeriver plugin to produce cache key and in the event of a miss, return the data.
|
|
* @param InCacheKey Complete cache key for this data.
|
|
**/
|
|
FBuildAsyncWorker(FDerivedDataPluginInterface* InDataDeriver, const TCHAR* InCacheKey, FStringView InDebugContext, bool bInSynchronousForStats)
|
|
: bSuccess(false)
|
|
, bSynchronousForStats(bInSynchronousForStats)
|
|
, bDataWasBuilt(false)
|
|
, DataDeriver(InDataDeriver)
|
|
, CacheKey(InCacheKey)
|
|
, DebugContext(InDebugContext)
|
|
{
|
|
}
|
|
|
|
virtual ~FBuildAsyncWorker()
|
|
{
|
|
// Record that the task is destroyed and check that it was not running or destroyed previously.
|
|
{
|
|
const uint32 PreviousState = WorkerState.fetch_or(WorkerStateDestroyed, std::memory_order_relaxed);
|
|
checkf(!(PreviousState & WorkerStateRunning), TEXT("Destroying DDC worker that is still running! Key: %s"), *CacheKey);
|
|
checkf(!(PreviousState & WorkerStateDestroyed), TEXT("Destroying DDC worker that has been destroyed previously! Key: %s"), *CacheKey);
|
|
}
|
|
}
|
|
|
|
/** Async worker that checks the cache backend and if that fails, calls the deriver to build the data and then puts the results to the cache **/
|
|
void DoWork()
|
|
{
|
|
// Record that the task is running and check that it was not running, finished, or destroyed previously.
|
|
{
|
|
const uint32 PreviousState = WorkerState.fetch_or(WorkerStateRunning, std::memory_order_relaxed);
|
|
checkf(!(PreviousState & WorkerStateRunning), TEXT("Starting DDC worker that is already running! Key: %s"), *CacheKey);
|
|
checkf(!(PreviousState & WorkerStateFinished), TEXT("Starting DDC worker that is already finished! Key: %s"), *CacheKey);
|
|
checkf(!(PreviousState & WorkerStateDestroyed), TEXT("Starting DDC worker that has been destroyed! Key: %s"), *CacheKey);
|
|
}
|
|
|
|
TRACE_CPUPROFILER_EVENT_SCOPE(DDC_DoWork);
|
|
|
|
const int32 NumBeforeDDC = Data.Num();
|
|
bool bGetResult;
|
|
{
|
|
TRACE_CPUPROFILER_EVENT_SCOPE(DDC_Get);
|
|
|
|
INC_DWORD_STAT(STAT_DDC_NumGets);
|
|
STAT(double ThisTime = 0);
|
|
{
|
|
SCOPE_SECONDS_COUNTER(ThisTime);
|
|
FLegacyCacheGetRequest LegacyRequest;
|
|
LegacyRequest.Name = DebugContext;
|
|
LegacyRequest.Key = FLegacyCacheKey(CacheKey, FDerivedDataBackend::Get().GetMaxKeyLength());
|
|
FRequestOwner BlockingOwner(EPriority::Blocking);
|
|
FDerivedDataBackend::Get().GetRoot().LegacyGet({LegacyRequest}, BlockingOwner,
|
|
[this, &bGetResult](FLegacyCacheGetResponse&& Response)
|
|
{
|
|
bGetResult = Response.Status == EStatus::Ok && Response.Value.GetSize() < MAX_int32;
|
|
if (bGetResult)
|
|
{
|
|
Data = MakeArrayView(static_cast<const uint8*>(Response.Value.GetData()), int32(Response.Value.GetSize()));
|
|
}
|
|
});
|
|
BlockingOwner.Wait();
|
|
}
|
|
INC_FLOAT_STAT_BY(STAT_DDC_SyncGetTime, bSynchronousForStats ? (float)ThisTime : 0.0f);
|
|
}
|
|
if (bGetResult)
|
|
{
|
|
|
|
if(GVerifyDDC && DataDeriver && DataDeriver->IsDeterministic())
|
|
{
|
|
TArray<uint8> CmpData;
|
|
DataDeriver->Build(CmpData);
|
|
const int32 NumInDDC = Data.Num() - NumBeforeDDC;
|
|
const int32 NumGenerated = CmpData.Num();
|
|
|
|
bool bMatchesInSize = NumGenerated == NumInDDC;
|
|
bool bDifferentMemory = true;
|
|
int32 DifferentOffset = 0;
|
|
if (bMatchesInSize)
|
|
{
|
|
bDifferentMemory = false;
|
|
for (int32 i = 0; i < NumGenerated; i++)
|
|
{
|
|
if (CmpData[i] != Data[i])
|
|
{
|
|
bDifferentMemory = true;
|
|
DifferentOffset = i;
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
if(!bMatchesInSize || bDifferentMemory)
|
|
{
|
|
FString ErrMsg = FString::Printf(TEXT("There is a mismatch between the DDC data and the generated data for plugin (%s) for asset (%s). BytesInDDC:%d, BytesGenerated:%d, bDifferentMemory:%d, offset:%d"), DataDeriver->GetPluginName(), *DataDeriver->GetDebugContextString(), NumInDDC, NumGenerated, bDifferentMemory, DifferentOffset);
|
|
ensureMsgf(false, TEXT("%s"), *ErrMsg);
|
|
UE_LOG(LogDerivedDataCache, Error, TEXT("%s"), *ErrMsg );
|
|
}
|
|
|
|
}
|
|
|
|
check(Data.Num());
|
|
bSuccess = true;
|
|
delete DataDeriver;
|
|
DataDeriver = NULL;
|
|
}
|
|
else if (DataDeriver)
|
|
{
|
|
{
|
|
TRACE_CPUPROFILER_EVENT_SCOPE(DDC_Build);
|
|
|
|
INC_DWORD_STAT(STAT_DDC_NumBuilds);
|
|
STAT(double ThisTime = 0);
|
|
{
|
|
SCOPE_SECONDS_COUNTER(ThisTime);
|
|
bSuccess = DataDeriver->Build(Data);
|
|
bDataWasBuilt = true;
|
|
}
|
|
INC_FLOAT_STAT_BY(STAT_DDC_SyncBuildTime, bSynchronousForStats ? (float)ThisTime : 0.0f);
|
|
}
|
|
delete DataDeriver;
|
|
DataDeriver = NULL;
|
|
if (bSuccess)
|
|
{
|
|
check(Data.Num());
|
|
|
|
TRACE_CPUPROFILER_EVENT_SCOPE(DDC_Put);
|
|
|
|
INC_DWORD_STAT(STAT_DDC_NumPuts);
|
|
STAT(double ThisTime = 0);
|
|
{
|
|
SCOPE_SECONDS_COUNTER(ThisTime);
|
|
FLegacyCachePutRequest LegacyRequest;
|
|
LegacyRequest.Name = DebugContext;
|
|
LegacyRequest.Key = FLegacyCacheKey(CacheKey, FDerivedDataBackend::Get().GetMaxKeyLength());
|
|
LegacyRequest.Value = FCompositeBuffer(FSharedBuffer::Clone(MakeMemoryView(Data)));
|
|
FRequestOwner BlockingOwner(EPriority::Blocking);
|
|
FDerivedDataBackend::Get().GetRoot().LegacyPut({LegacyRequest}, BlockingOwner, [](auto&&){});
|
|
BlockingOwner.Wait();
|
|
}
|
|
INC_FLOAT_STAT_BY(STAT_DDC_PutTime, bSynchronousForStats ? (float)ThisTime : 0.0f);
|
|
}
|
|
}
|
|
if (!bSuccess)
|
|
{
|
|
Data.Empty();
|
|
}
|
|
FDerivedDataBackend::Get().AddToAsyncCompletionCounter(-1);
|
|
|
|
// Record that the task is finished and check that it was running and not finished or destroyed previously.
|
|
{
|
|
const uint32 PreviousState = WorkerState.fetch_xor(WorkerStateRunning | WorkerStateFinished, std::memory_order_relaxed);
|
|
checkf((PreviousState & WorkerStateRunning), TEXT("Finishing DDC worker that was not running! Key: %s"), *CacheKey);
|
|
checkf(!(PreviousState & WorkerStateFinished), TEXT("Finishing DDC worker that is already finished! Key: %s"), *CacheKey);
|
|
checkf(!(PreviousState & WorkerStateDestroyed), TEXT("Finishing DDC worker that has been destroyed! Key: %s"), *CacheKey);
|
|
}
|
|
}
|
|
|
|
FORCEINLINE TStatId GetStatId() const
|
|
{
|
|
RETURN_QUICK_DECLARE_CYCLE_STAT(FBuildAsyncWorker, STATGROUP_ThreadPoolAsyncTasks);
|
|
}
|
|
|
|
std::atomic<uint32> WorkerState{WorkerStateNone};
|
|
/** true in the case of a cache hit, otherwise the result of the deriver build call **/
|
|
bool bSuccess;
|
|
/** true if we should record the timing **/
|
|
bool bSynchronousForStats;
|
|
/** true if we had to build the data */
|
|
bool bDataWasBuilt;
|
|
/** Data dervier we are operating on **/
|
|
FDerivedDataPluginInterface* DataDeriver;
|
|
/** Cache key associated with this build **/
|
|
FString CacheKey;
|
|
/** Context from the caller */
|
|
FSharedString DebugContext;
|
|
/** Data to return to caller, later **/
|
|
TArray<uint8> Data;
|
|
};
|
|
|
|
public:
|
|
|
|
/** Constructor, called once to cereate a singleton **/
|
|
FDerivedDataCache()
|
|
: CurrentHandle(19248) // we will skip some potential handles to catch errors
|
|
{
|
|
if (FPlatformProcess::SupportsMultithreading())
|
|
{
|
|
GCacheThreadPool = FQueuedThreadPool::Allocate();
|
|
const int32 ThreadCount = FPlatformMisc::NumberOfIOWorkerThreadsToSpawn();
|
|
verify(GCacheThreadPool->Create(ThreadCount, 96 * 1024, TPri_AboveNormal, TEXT("DDC IO ThreadPool")));
|
|
}
|
|
|
|
FDerivedDataBackend::Get(); // we need to make sure this starts before we allow us to start
|
|
|
|
CacheStoreMaintainers = IModularFeatures::Get().GetModularFeatureImplementations<ICacheStoreMaintainer>(FeatureName);
|
|
|
|
GVerifyDDC = FParse::Param(FCommandLine::Get(), TEXT("VerifyDDC"));
|
|
|
|
UE_CLOG(GVerifyDDC, LogDerivedDataCache, Display, TEXT("Items retrieved from the DDC will be verified (-VerifyDDC)"));
|
|
}
|
|
|
|
/** Destructor, flushes all sync tasks **/
|
|
~FDerivedDataCache()
|
|
{
|
|
WaitForQuiescence(true);
|
|
FScopeLock ScopeLock(&SynchronizationObject);
|
|
for (TMap<uint32,FAsyncTask<FBuildAsyncWorker>*>::TIterator It(PendingTasks); It; ++It)
|
|
{
|
|
It.Value()->EnsureCompletion();
|
|
delete It.Value();
|
|
}
|
|
PendingTasks.Empty();
|
|
}
|
|
|
|
virtual bool GetSynchronous(FDerivedDataPluginInterface* DataDeriver, TArray<uint8>& OutData, bool* bDataWasBuilt = nullptr) override
|
|
{
|
|
DDC_SCOPE_CYCLE_COUNTER(DDC_GetSynchronous);
|
|
check(DataDeriver);
|
|
FString CacheKey = FDerivedDataCache::BuildCacheKey(DataDeriver);
|
|
UE_LOG(LogDerivedDataCache, VeryVerbose, TEXT("GetSynchronous %s from '%s'"), *CacheKey, *DataDeriver->GetDebugContextString());
|
|
FAsyncTask<FBuildAsyncWorker> PendingTask(DataDeriver, *CacheKey, DataDeriver->GetDebugContextString(), true);
|
|
AddToAsyncCompletionCounter(1);
|
|
PendingTask.StartSynchronousTask();
|
|
OutData = PendingTask.GetTask().Data;
|
|
if (bDataWasBuilt)
|
|
{
|
|
*bDataWasBuilt = PendingTask.GetTask().bDataWasBuilt;
|
|
}
|
|
return PendingTask.GetTask().bSuccess;
|
|
}
|
|
|
|
virtual uint32 GetAsynchronous(FDerivedDataPluginInterface* DataDeriver) override
|
|
{
|
|
DDC_SCOPE_CYCLE_COUNTER(DDC_GetAsynchronous);
|
|
FScopeLock ScopeLock(&SynchronizationObject);
|
|
const uint32 Handle = NextHandle();
|
|
FString CacheKey = FDerivedDataCache::BuildCacheKey(DataDeriver);
|
|
UE_LOG(LogDerivedDataCache, VeryVerbose, TEXT("GetAsynchronous %s from '%s', Handle %d"), *CacheKey, *DataDeriver->GetDebugContextString(), Handle);
|
|
const bool bSync = !DataDeriver->IsBuildThreadsafe();
|
|
FAsyncTask<FBuildAsyncWorker>* AsyncTask = new FAsyncTask<FBuildAsyncWorker>(DataDeriver, *CacheKey, DataDeriver->GetDebugContextString(), bSync);
|
|
check(!PendingTasks.Contains(Handle));
|
|
PendingTasks.Add(Handle,AsyncTask);
|
|
AddToAsyncCompletionCounter(1);
|
|
if (!bSync)
|
|
{
|
|
AsyncTask->StartBackgroundTask(DataDeriver->GetCustomThreadPool());
|
|
}
|
|
else
|
|
{
|
|
AsyncTask->StartSynchronousTask();
|
|
}
|
|
// Must return a valid handle
|
|
check(Handle != 0);
|
|
return Handle;
|
|
}
|
|
|
|
virtual bool PollAsynchronousCompletion(uint32 Handle) override
|
|
{
|
|
DDC_SCOPE_CYCLE_COUNTER(DDC_PollAsynchronousCompletion);
|
|
FAsyncTask<FBuildAsyncWorker>* AsyncTask = NULL;
|
|
{
|
|
FScopeLock ScopeLock(&SynchronizationObject);
|
|
AsyncTask = PendingTasks.FindRef(Handle);
|
|
}
|
|
check(AsyncTask);
|
|
return AsyncTask->IsDone();
|
|
}
|
|
|
|
virtual void WaitAsynchronousCompletion(uint32 Handle) override
|
|
{
|
|
DDC_SCOPE_CYCLE_COUNTER(DDC_WaitAsynchronousCompletion);
|
|
STAT(double ThisTime = 0);
|
|
{
|
|
SCOPE_SECONDS_COUNTER(ThisTime);
|
|
FAsyncTask<FBuildAsyncWorker>* AsyncTask = NULL;
|
|
{
|
|
FScopeLock ScopeLock(&SynchronizationObject);
|
|
AsyncTask = PendingTasks.FindRef(Handle);
|
|
}
|
|
check(AsyncTask);
|
|
AsyncTask->EnsureCompletion();
|
|
UE_LOG(LogDerivedDataCache, Verbose, TEXT("WaitAsynchronousCompletion, Handle %d"), Handle);
|
|
}
|
|
INC_FLOAT_STAT_BY(STAT_DDC_ASyncWaitTime,(float)ThisTime);
|
|
}
|
|
|
|
virtual bool GetAsynchronousResults(uint32 Handle, TArray<uint8>& OutData, bool* bOutDataWasBuilt = nullptr) override
|
|
{
|
|
DDC_SCOPE_CYCLE_COUNTER(DDC_GetAsynchronousResults);
|
|
FAsyncTask<FBuildAsyncWorker>* AsyncTask = NULL;
|
|
{
|
|
FScopeLock ScopeLock(&SynchronizationObject);
|
|
PendingTasks.RemoveAndCopyValue(Handle,AsyncTask);
|
|
}
|
|
check(AsyncTask);
|
|
const bool bDataWasBuilt = AsyncTask->GetTask().bDataWasBuilt;
|
|
if (bOutDataWasBuilt)
|
|
{
|
|
*bOutDataWasBuilt = bDataWasBuilt;
|
|
}
|
|
if (!AsyncTask->GetTask().bSuccess)
|
|
{
|
|
UE_LOG(LogDerivedDataCache, Verbose, TEXT("GetAsynchronousResults, bDataWasBuilt: %d, Handle %d, FAILED"), (int32)bDataWasBuilt, Handle);
|
|
delete AsyncTask;
|
|
return false;
|
|
}
|
|
|
|
UE_LOG(LogDerivedDataCache, Verbose, TEXT("GetAsynchronousResults, bDataWasBuilt: %d, Handle %d, SUCCESS"), (int32)bDataWasBuilt, Handle);
|
|
OutData = MoveTemp(AsyncTask->GetTask().Data);
|
|
delete AsyncTask;
|
|
check(OutData.Num());
|
|
return true;
|
|
}
|
|
|
|
virtual bool GetSynchronous(const TCHAR* CacheKey, TArray<uint8>& OutData, FStringView DebugContext) override
|
|
{
|
|
DDC_SCOPE_CYCLE_COUNTER(DDC_GetSynchronous_Data);
|
|
UE_LOG(LogDerivedDataCache, VeryVerbose, TEXT("GetSynchronous %s from '%.*s'"), CacheKey, DebugContext.Len(), DebugContext.GetData());
|
|
ValidateCacheKey(CacheKey);
|
|
FAsyncTask<FBuildAsyncWorker> PendingTask((FDerivedDataPluginInterface*)NULL, CacheKey, DebugContext, true);
|
|
AddToAsyncCompletionCounter(1);
|
|
PendingTask.StartSynchronousTask();
|
|
OutData = PendingTask.GetTask().Data;
|
|
return PendingTask.GetTask().bSuccess;
|
|
}
|
|
|
|
virtual uint32 GetAsynchronous(const TCHAR* CacheKey, FStringView DebugContext) override
|
|
{
|
|
DDC_SCOPE_CYCLE_COUNTER(DDC_GetAsynchronous_Handle);
|
|
FScopeLock ScopeLock(&SynchronizationObject);
|
|
const uint32 Handle = NextHandle();
|
|
UE_LOG(LogDerivedDataCache, VeryVerbose, TEXT("GetAsynchronous %s from '%.*s', Handle %d"), CacheKey, DebugContext.Len(), DebugContext.GetData(), Handle);
|
|
ValidateCacheKey(CacheKey);
|
|
FAsyncTask<FBuildAsyncWorker>* AsyncTask = new FAsyncTask<FBuildAsyncWorker>((FDerivedDataPluginInterface*)NULL, CacheKey, DebugContext, false);
|
|
check(!PendingTasks.Contains(Handle));
|
|
PendingTasks.Add(Handle, AsyncTask);
|
|
AddToAsyncCompletionCounter(1);
|
|
// This request is I/O only, doesn't do any processing, send it to the I/O only thread-pool to avoid wasting worker threads on long I/O waits.
|
|
AsyncTask->StartBackgroundTask(GCacheThreadPool);
|
|
return Handle;
|
|
}
|
|
|
|
virtual void Put(const TCHAR* CacheKey, TArrayView<const uint8> Data, FStringView DebugContext, bool bPutEvenIfExists = false) override
|
|
{
|
|
DDC_SCOPE_CYCLE_COUNTER(DDC_Put);
|
|
UE_LOG(LogDerivedDataCache, VeryVerbose, TEXT("Put %s from '%.*s'"), CacheKey, DebugContext.Len(), DebugContext.GetData());
|
|
ValidateCacheKey(CacheKey);
|
|
STAT(double ThisTime = 0);
|
|
{
|
|
SCOPE_SECONDS_COUNTER(ThisTime);
|
|
FLegacyCachePutRequest LegacyRequest;
|
|
LegacyRequest.Name = DebugContext;
|
|
LegacyRequest.Key = FLegacyCacheKey(CacheKey, FDerivedDataBackend::Get().GetMaxKeyLength());
|
|
LegacyRequest.Value = FCompositeBuffer(FSharedBuffer::Clone(MakeMemoryView(Data)));
|
|
FRequestOwner BlockingOwner(EPriority::Blocking);
|
|
FDerivedDataBackend::Get().GetRoot().LegacyPut({LegacyRequest}, BlockingOwner, [](auto&&){});
|
|
BlockingOwner.Wait();
|
|
}
|
|
INC_FLOAT_STAT_BY(STAT_DDC_PutTime,(float)ThisTime);
|
|
INC_DWORD_STAT(STAT_DDC_NumPuts);
|
|
}
|
|
|
|
virtual void MarkTransient(const TCHAR* CacheKey) override
|
|
{
|
|
ValidateCacheKey(CacheKey);
|
|
FLegacyCacheDeleteRequest LegacyRequest;
|
|
LegacyRequest.Key = FLegacyCacheKey(CacheKey, FDerivedDataBackend::Get().GetMaxKeyLength());
|
|
LegacyRequest.Name = LegacyRequest.Key.GetFullKey();
|
|
LegacyRequest.bTransient = true;
|
|
FRequestOwner BlockingOwner(EPriority::Blocking);
|
|
FDerivedDataBackend::Get().GetRoot().LegacyDelete({LegacyRequest}, BlockingOwner, [](auto&&){});
|
|
BlockingOwner.Wait();
|
|
}
|
|
|
|
virtual bool CachedDataProbablyExists(const TCHAR* CacheKey) override
|
|
{
|
|
DDC_SCOPE_CYCLE_COUNTER(DDC_CachedDataProbablyExists);
|
|
ValidateCacheKey(CacheKey);
|
|
bool bResult;
|
|
INC_DWORD_STAT(STAT_DDC_NumExist);
|
|
STAT(double ThisTime = 0);
|
|
{
|
|
SCOPE_SECONDS_COUNTER(ThisTime);
|
|
FLegacyCacheGetRequest LegacyRequest;
|
|
LegacyRequest.Key = FLegacyCacheKey(CacheKey, FDerivedDataBackend::Get().GetMaxKeyLength());
|
|
LegacyRequest.Name = LegacyRequest.Key.GetFullKey();
|
|
LegacyRequest.Policy = ECachePolicy::Query | ECachePolicy::SkipData;
|
|
FRequestOwner BlockingOwner(EPriority::Blocking);
|
|
FDerivedDataBackend::Get().GetRoot().LegacyGet({LegacyRequest}, BlockingOwner,
|
|
[&bResult](FLegacyCacheGetResponse&& Response) { bResult = Response.Status == EStatus::Ok; });
|
|
BlockingOwner.Wait();
|
|
}
|
|
INC_FLOAT_STAT_BY(STAT_DDC_ExistTime, (float)ThisTime);
|
|
return bResult;
|
|
}
|
|
|
|
virtual TBitArray<> CachedDataProbablyExistsBatch(TConstArrayView<FString> CacheKeys) override
|
|
{
|
|
TBitArray<> Result(false, CacheKeys.Num());
|
|
if (!CacheKeys.IsEmpty())
|
|
{
|
|
DDC_SCOPE_CYCLE_COUNTER(DDC_CachedDataProbablyExistsBatch);
|
|
INC_DWORD_STAT(STAT_DDC_NumExist);
|
|
STAT(double ThisTime = 0);
|
|
{
|
|
SCOPE_SECONDS_COUNTER(ThisTime);
|
|
TArray<FLegacyCacheGetRequest, TInlineAllocator<8>> LegacyRequests;
|
|
int32 Index = 0;
|
|
for (const FString& CacheKey : CacheKeys)
|
|
{
|
|
FLegacyCacheGetRequest& LegacyRequest = LegacyRequests.AddDefaulted_GetRef();
|
|
LegacyRequest.Key = FLegacyCacheKey(CacheKey, FDerivedDataBackend::Get().GetMaxKeyLength());
|
|
LegacyRequest.Name = LegacyRequest.Key.GetFullKey();
|
|
LegacyRequest.Policy = ECachePolicy::Query | ECachePolicy::SkipData;
|
|
LegacyRequest.UserData = uint64(Index);
|
|
++Index;
|
|
}
|
|
FRequestOwner BlockingOwner(EPriority::Blocking);
|
|
FDerivedDataBackend::Get().GetRoot().LegacyGet(LegacyRequests, BlockingOwner,
|
|
[&Result](FLegacyCacheGetResponse&& Response)
|
|
{
|
|
Result[int32(Response.UserData)] = Response.Status == EStatus::Ok;
|
|
});
|
|
BlockingOwner.Wait();
|
|
}
|
|
INC_FLOAT_STAT_BY(STAT_DDC_ExistTime, (float)ThisTime);
|
|
}
|
|
return Result;
|
|
}
|
|
|
|
virtual bool AllCachedDataProbablyExists(TConstArrayView<FString> CacheKeys) override
|
|
{
|
|
return CacheKeys.Num() == 0 || CachedDataProbablyExistsBatch(CacheKeys).CountSetBits() == CacheKeys.Num();
|
|
}
|
|
|
|
virtual bool TryToPrefetch(TConstArrayView<FString> CacheKeys, FStringView DebugContext) override
|
|
{
|
|
if (!CacheKeys.IsEmpty())
|
|
{
|
|
DDC_SCOPE_CYCLE_COUNTER(DDC_TryToPrefetch);
|
|
UE_LOG(LogDerivedDataCache, VeryVerbose, TEXT("TryToPrefetch %d keys including %s from '%.*s'"),
|
|
CacheKeys.Num(), *CacheKeys[0], DebugContext.Len(), DebugContext.GetData());
|
|
TArray<FLegacyCacheGetRequest, TInlineAllocator<8>> LegacyRequests;
|
|
int32 Index = 0;
|
|
const FSharedString Name = DebugContext;
|
|
for (const FString& CacheKey : CacheKeys)
|
|
{
|
|
FLegacyCacheGetRequest& LegacyRequest = LegacyRequests.AddDefaulted_GetRef();
|
|
LegacyRequest.Name = Name;
|
|
LegacyRequest.Key = FLegacyCacheKey(CacheKey, FDerivedDataBackend::Get().GetMaxKeyLength());
|
|
LegacyRequest.Policy = ECachePolicy::Default | ECachePolicy::SkipData;
|
|
LegacyRequest.UserData = uint64(Index);
|
|
++Index;
|
|
}
|
|
bool bOk = true;
|
|
FRequestOwner BlockingOwner(EPriority::Blocking);
|
|
FDerivedDataBackend::Get().GetRoot().LegacyGet(LegacyRequests, BlockingOwner,
|
|
[&bOk](FLegacyCacheGetResponse&& Response)
|
|
{
|
|
bOk &= Response.Status == EStatus::Ok;
|
|
});
|
|
BlockingOwner.Wait();
|
|
return bOk;
|
|
}
|
|
return true;
|
|
}
|
|
|
|
void NotifyBootComplete() override
|
|
{
|
|
DDC_SCOPE_CYCLE_COUNTER(DDC_NotifyBootComplete);
|
|
FDerivedDataBackend::Get().NotifyBootComplete();
|
|
}
|
|
|
|
void AddToAsyncCompletionCounter(int32 Addend) override
|
|
{
|
|
FDerivedDataBackend::Get().AddToAsyncCompletionCounter(Addend);
|
|
}
|
|
|
|
bool AnyAsyncRequestsRemaining() const override
|
|
{
|
|
return FDerivedDataBackend::Get().AnyAsyncRequestsRemaining();
|
|
}
|
|
|
|
void WaitForQuiescence(bool bShutdown) override
|
|
{
|
|
DDC_SCOPE_CYCLE_COUNTER(DDC_WaitForQuiescence);
|
|
FDerivedDataBackend::Get().WaitForQuiescence(bShutdown);
|
|
}
|
|
|
|
/** Get whether a Shared Data Cache is in use */
|
|
virtual bool GetUsingSharedDDC() const override
|
|
{
|
|
return FDerivedDataBackend::Get().GetUsingSharedDDC();
|
|
}
|
|
|
|
virtual const TCHAR* GetGraphName() const override
|
|
{
|
|
return FDerivedDataBackend::Get().GetGraphName();
|
|
}
|
|
|
|
virtual const TCHAR* GetDefaultGraphName() const override
|
|
{
|
|
return FDerivedDataBackend::Get().GetDefaultGraphName();
|
|
}
|
|
|
|
void GetDirectories(TArray<FString>& OutResults) override
|
|
{
|
|
FDerivedDataBackend::Get().GetDirectories(OutResults);
|
|
}
|
|
|
|
PRAGMA_DISABLE_DEPRECATION_WARNINGS
|
|
virtual IDDCCleanup* GetCleanup() const override
|
|
{
|
|
return const_cast<IDDCCleanup*>(static_cast<const IDDCCleanup*>(this));
|
|
}
|
|
PRAGMA_ENABLE_DEPRECATION_WARNINGS
|
|
|
|
virtual bool IsFinished() const override
|
|
{
|
|
return IsIdle();
|
|
}
|
|
|
|
virtual void WaitBetweenDeletes(bool bWait) override
|
|
{
|
|
if (!bWait)
|
|
{
|
|
BoostPriority();
|
|
}
|
|
}
|
|
|
|
virtual void GatherUsageStats(TMap<FString, FDerivedDataCacheUsageStats>& UsageStats) override
|
|
{
|
|
GatherUsageStats()->GatherLegacyUsageStats(UsageStats, TEXT(" 0"));
|
|
}
|
|
|
|
PRAGMA_DISABLE_DEPRECATION_WARNINGS
|
|
virtual TSharedRef<FDerivedDataCacheStatsNode> GatherUsageStats() const override
|
|
{
|
|
return FDerivedDataBackend::Get().GatherUsageStats();
|
|
}
|
|
PRAGMA_ENABLE_DEPRECATION_WARNINGS
|
|
|
|
virtual void GatherResourceStats(TArray<FDerivedDataCacheResourceStat>& DDCResourceStats) const override
|
|
{
|
|
GatherDerivedDataCacheResourceStats(DDCResourceStats);
|
|
}
|
|
|
|
virtual void GatherSummaryStats(FDerivedDataCacheSummaryStats& DDCSummaryStats) const override
|
|
{
|
|
GatherDerivedDataCacheSummaryStats(DDCSummaryStats);
|
|
}
|
|
|
|
/** Get event delegate for data cache notifications */
|
|
virtual FOnDDCNotification& GetDDCNotificationEvent()
|
|
{
|
|
return DDCNotificationEvent;
|
|
}
|
|
|
|
protected:
|
|
uint32 NextHandle()
|
|
{
|
|
return (uint32)CurrentHandle.Increment();
|
|
}
|
|
|
|
|
|
private:
|
|
|
|
/**
|
|
* Internal function to build a cache key out of the plugin name, versions and plugin specific info
|
|
* @param DataDeriver plugin to produce the elements of the cache key.
|
|
* @return Assembled cache key
|
|
**/
|
|
static FString BuildCacheKey(FDerivedDataPluginInterface* DataDeriver)
|
|
{
|
|
FString Result = FDerivedDataCacheInterface::BuildCacheKey(DataDeriver->GetPluginName(), DataDeriver->GetVersionString(), *DataDeriver->GetPluginSpecificCacheKeySuffix());
|
|
return Result;
|
|
}
|
|
|
|
static void ValidateCacheKey(const TCHAR* CacheKey)
|
|
{
|
|
checkf(Algo::AllOf(FStringView(CacheKey), IsValidCacheChar),
|
|
TEXT("Invalid characters in cache key %s. Use SanitizeCacheKey or BuildCacheKey to create valid keys."), CacheKey);
|
|
}
|
|
|
|
/** Counter used to produce unique handles **/
|
|
FThreadSafeCounter CurrentHandle;
|
|
/** Object used for synchronization via a scoped lock **/
|
|
FCriticalSection SynchronizationObject;
|
|
/** Map of handle to pending task **/
|
|
TMap<uint32,FAsyncTask<FBuildAsyncWorker>*> PendingTasks;
|
|
|
|
/** Cache notification delegate */
|
|
FOnDDCNotification DDCNotificationEvent;
|
|
|
|
public:
|
|
// ICacheStore Interface
|
|
|
|
void Put(
|
|
TConstArrayView<FCachePutRequest> Requests,
|
|
IRequestOwner& Owner,
|
|
FOnCachePutComplete&& OnComplete) final
|
|
{
|
|
return FDerivedDataBackend::Get().GetRoot().Put(Requests, Owner, OnComplete ? MoveTemp(OnComplete) : [](auto&&){});
|
|
}
|
|
|
|
void Get(
|
|
TConstArrayView<FCacheGetRequest> Requests,
|
|
IRequestOwner& Owner,
|
|
FOnCacheGetComplete&& OnComplete) final
|
|
{
|
|
return FDerivedDataBackend::Get().GetRoot().Get(Requests, Owner, OnComplete ? MoveTemp(OnComplete) : [](auto&&){});
|
|
}
|
|
|
|
void PutValue(
|
|
TConstArrayView<FCachePutValueRequest> Requests,
|
|
IRequestOwner& Owner,
|
|
FOnCachePutValueComplete&& OnComplete) final
|
|
{
|
|
return FDerivedDataBackend::Get().GetRoot().PutValue(Requests, Owner, OnComplete ? MoveTemp(OnComplete) : [](auto&&){});
|
|
}
|
|
|
|
void GetValue(
|
|
TConstArrayView<FCacheGetValueRequest> Requests,
|
|
IRequestOwner& Owner,
|
|
FOnCacheGetValueComplete&& OnComplete) final
|
|
{
|
|
return FDerivedDataBackend::Get().GetRoot().GetValue(Requests, Owner, OnComplete ? MoveTemp(OnComplete) : [](auto&&){});
|
|
}
|
|
|
|
void GetChunks(
|
|
TConstArrayView<FCacheGetChunkRequest> Requests,
|
|
IRequestOwner& Owner,
|
|
FOnCacheGetChunkComplete&& OnComplete) final
|
|
{
|
|
return FDerivedDataBackend::Get().GetRoot().GetChunks(Requests, Owner, OnComplete ? MoveTemp(OnComplete) : [](auto&&){});
|
|
}
|
|
|
|
// ICache Interface
|
|
|
|
void CancelAll() final
|
|
{
|
|
}
|
|
|
|
ICacheStoreMaintainer& GetMaintainer() final
|
|
{
|
|
return *this;
|
|
}
|
|
|
|
// ICacheStoreMaintainer Interface
|
|
|
|
bool IsIdle() const final
|
|
{
|
|
return Algo::AllOf(CacheStoreMaintainers, &ICacheStoreMaintainer::IsIdle);
|
|
}
|
|
|
|
void BoostPriority() final
|
|
{
|
|
for (ICacheStoreMaintainer* Maintainer : CacheStoreMaintainers)
|
|
{
|
|
Maintainer->BoostPriority();
|
|
}
|
|
}
|
|
|
|
private:
|
|
TArray<ICacheStoreMaintainer*> CacheStoreMaintainers;
|
|
};
|
|
|
|
ICache* CreateCache(FDerivedDataCacheInterface** OutLegacyCache)
|
|
{
|
|
FDerivedDataCache* Cache = new FDerivedDataCache;
|
|
if (OutLegacyCache)
|
|
{
|
|
*OutLegacyCache = Cache;
|
|
}
|
|
return Cache;
|
|
}
|
|
|
|
} // UE::DerivedData::Private
|