You've already forked UnrealEngineUWP
mirror of
https://github.com/izzy2lost/UnrealEngineUWP.git
synced 2026-03-26 18:15:20 -07:00
2800210897
- FileSystem, Http, Pak, S3 use the ValueWithLegacyFallback mode by default, which cause them to fall back to searching for uncompressed data if compressed data is not found. - FileSystem has been fixed to store up to 1 MiB of compressed data inline with the value package rather than separately in content-addressable storage. - Pak has been optimized to have GetChunks only load the required region of the requested value, rather than the whole value. - Pak has been changed to stop storing data inline in the record package, since it will end up in the same file anyway when stored separately. - Pak will upgrade the compressor and compression level when a compressed pak file is requested. Default cache compression uses Oodle Mermaid VeryFast and will upgrade to Oodle Kraken Optimal2. - Zen does not have compression enabled by default, pending deployment of a new version that stores compressed values to Horde Storage. - Added a missing request barrier when saving uncompressed data as compressed. Example reduction in file system cache size when cooking for Windows: - CitySample dropped from 66.5 GiB to 21.8 GiB. - Lyra dropped from 2.54 GiB to 672 MiB. - ShooterGame dropped from 1.21 GiB to 380 MiB. Example reduction in compressed pak file cache size when cooking for Windows: - CitySample dropped from 22.3 GiB to 18.5 GiB. - Lyra dropped from 691 MiB to 543 MiB. - ShooterGame dropped from 387 MiB to 313 MiB. #jira UE-134381 #preflight 620a703f583261b0a658e043, 620a6fb2803d9066e6805310, 620a733117632e948459b6af #lockdown Aurel.Cordonnier #rb Zousar.Shaker #ROBOMERGE-AUTHOR: devin.doucette #ROBOMERGE-SOURCE: CL 18983671 in //UE5/Release-5.0/... via CL 18983890 via CL 18984096 #ROBOMERGE-BOT: UE5 (Release-Engine-Test -> Main) (v917-18934589) [CL 18984126 by devin doucette in ue5-main branch]
87 lines
2.6 KiB
C++
87 lines
2.6 KiB
C++
// Copyright Epic Games, Inc. All Rights Reserved.
|
|
|
|
#pragma once
|
|
|
|
#include "CoreTypes.h"
|
|
#include "Containers/Array.h"
|
|
#include "DerivedDataBackendInterface.h"
|
|
#include "HAL/UnrealMemory.h"
|
|
#include "Memory/MemoryView.h"
|
|
#include "Misc/Crc.h"
|
|
#include "Serialization/Archive.h"
|
|
|
|
namespace UE::DerivedData
|
|
{
|
|
|
|
/**
|
|
* Helper class for placing a footer at the end of of a cache file.
|
|
* No effort is made to byte-swap this as we assume local format.
|
|
*/
|
|
struct FDerivedDataTrailer
|
|
{
|
|
/** Arbitrary number used to identify corruption */
|
|
static constexpr inline uint32 MagicConstant = 0x1e873d89;
|
|
|
|
/** Arbitrary number used to identify corruption */
|
|
uint32 Magic = 0;
|
|
/** Version of the trailer */
|
|
uint32 Version = 0;
|
|
/** CRC of the payload, used to detect corruption */
|
|
uint32 CRCofPayload = 0;
|
|
/** Size of the payload, used to detect corruption */
|
|
uint32 SizeOfPayload = 0;
|
|
|
|
FDerivedDataTrailer() = default;
|
|
|
|
explicit FDerivedDataTrailer(const FMemoryView Data)
|
|
: Magic(MagicConstant)
|
|
, Version(1)
|
|
, CRCofPayload(FCrc::MemCrc_DEPRECATED(Data.GetData(), IntCastChecked<int32>(Data.GetSize())))
|
|
, SizeOfPayload(uint32(Data.GetSize()))
|
|
{
|
|
}
|
|
|
|
bool operator==(const FDerivedDataTrailer& Other) const
|
|
{
|
|
return Magic == Other.Magic
|
|
&& Version == Other.Version
|
|
&& CRCofPayload == Other.CRCofPayload
|
|
&& SizeOfPayload == Other.SizeOfPayload;
|
|
}
|
|
};
|
|
|
|
class FCorruptionWrapper
|
|
{
|
|
public:
|
|
static bool ReadTrailer(TArray<uint8>& Data, const TCHAR* CacheName, const TCHAR* CacheKey)
|
|
{
|
|
if (Data.Num() < sizeof(FDerivedDataTrailer))
|
|
{
|
|
UE_LOG(LogDerivedDataCache, Display, TEXT("%s: Corrupted file (short), ignoring and deleting %s."), CacheName, CacheKey);
|
|
return false;
|
|
}
|
|
|
|
FDerivedDataTrailer Trailer;
|
|
FMemory::Memcpy(&Trailer, &Data[Data.Num() - sizeof(FDerivedDataTrailer)], sizeof(FDerivedDataTrailer));
|
|
Data.RemoveAt(Data.Num() - sizeof(FDerivedDataTrailer), sizeof(FDerivedDataTrailer), /*bAllowShrinking*/ false);
|
|
FDerivedDataTrailer RecomputedTrailer(MakeMemoryView(Data));
|
|
if (!(Trailer == RecomputedTrailer))
|
|
{
|
|
UE_LOG(LogDerivedDataCache, Display, TEXT("%s: Corrupted file, ignoring and deleting %s."), CacheName, CacheKey);
|
|
return false;
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
static void WriteTrailer(FArchive& Ar, TConstArrayView<uint8> Data, const TCHAR* CacheName, const TCHAR* CacheKey)
|
|
{
|
|
checkf(Ar.TotalSize() + sizeof(FDerivedDataTrailer) <= MAX_int32,
|
|
TEXT("%s: Appending the trailer makes the data exceed 2 GiB for %s"), CacheName, CacheKey);
|
|
FDerivedDataTrailer Trailer(MakeMemoryView(Data));
|
|
Ar.Serialize(&Trailer, sizeof(FDerivedDataTrailer));
|
|
}
|
|
};
|
|
|
|
} // UE::DerivedData
|