// Copyright Epic Games, Inc. All Rights Reserved.

#include "IoStoreUtilities.h"

#if !UE_BUILD_SHIPPING
#include "FilePackageStoreWriter.h"
#include "HAL/FileManager.h"
#include "HAL/PlatformFileManager.h"
#include "Hash/CityHash.h"
#include "Interfaces/IPluginManager.h"
#include "Interfaces/ITargetPlatformManagerModule.h"
#include "Interfaces/ITargetPlatform.h"
#include "IO/IoDispatcher.h"
#include "Misc/App.h"
#include "Misc/CommandLine.h"
#include "Misc/ConfigCacheIni.h"
#include "Misc/PackageName.h"
#include "Misc/Paths.h"
#include "Misc/PackageName.h"
#include "Misc/Base64.h"
#include "Misc/AES.h"
#include "Misc/CoreDelegates.h"
#include "Misc/KeyChainUtilities.h"
#include "Modules/ModuleManager.h"
#include "Serialization/Archive.h"
#include "Serialization/JsonReader.h"
#include "Serialization/JsonSerializer.h"
#include "Serialization/BufferWriter.h"
#include "Serialization/LargeMemoryWriter.h"
#include "Serialization/MemoryReader.h"
#include "Serialization/AsyncLoading2.h"
#include "Serialization/ArrayReader.h"
#include "IO/PackageStore.h"
#include "Serialization/FilePackageStore.h"
#include "UObject/Class.h"
#include "UObject/NameBatchSerialization.h"
#include "UObject/PackageFileSummary.h"
#include "UObject/ObjectResource.h"
#include "UObject/Package.h"
#include "UObject/UObjectHash.h"
#include "Algo/Find.h"
#include "Misc/FileHelper.h"
#include "Misc/ScopeLock.h"
#include "Async/ParallelFor.h"
#include "Async/AsyncFileHandle.h"
#include "Async/Async.h"
#include "RSA.h"
#include "Misc/AssetRegistryInterface.h"
#include "AssetRegistryState.h"
#include "Misc/OutputDeviceFile.h"
#include "Misc/FeedbackContext.h"
#include "Serialization/LargeMemoryReader.h"
#include "Misc/StringBuilder.h"
#include "Async/Future.h"
#include "Algo/MaxElement.h"
#include "Algo/StableSort.h"
#include "PackageStoreOptimizer.h"
#include "ShaderCodeLibrary.h"
#include "ZenStoreHttpClient.h"
#include "IPlatformFilePak.h"
#include "ZenStoreWriter.h"

//PRAGMA_DISABLE_OPTIMIZATION

IMPLEMENT_MODULE(FDefaultModuleImpl, IoStoreUtilities);

DEFINE_LOG_CATEGORY_STATIC(LogIoStore, Log, All);

#define IOSTORE_CPU_SCOPE(NAME) TRACE_CPUPROFILER_EVENT_SCOPE(IoStore##NAME);
#define IOSTORE_CPU_SCOPE_DATA(NAME, DATA) TRACE_CPUPROFILER_EVENT_SCOPE(IoStore##NAME);

#define OUTPUT_CHUNKID_DIRECTORY 0

static const FName DefaultCompressionMethod = NAME_Zlib;
static const uint64 DefaultCompressionBlockSize = 64 << 10;
static const uint64 DefaultCompressionBlockAlignment = 64 << 10;
static const uint64 DefaultMemoryMappingAlignment = 16 << 10;

struct FReleasedPackages
{
	TSet<FName> PackageNames;
	TMap<FPackageId, FName> PackageIdToName;
};

static void LoadKeyChain(const TCHAR* CmdLine, FKeyChain& OutCryptoSettings)
{
	OutCryptoSettings.SigningKey = InvalidRSAKeyHandle;
	OutCryptoSettings.EncryptionKeys.Empty();

	// First, try and parse the keys from a supplied crypto key cache file
	FString CryptoKeysCacheFilename;
	if (FParse::Value(CmdLine, TEXT("cryptokeys="), CryptoKeysCacheFilename))
	{
		UE_LOG(LogIoStore, Display, TEXT("Parsing crypto keys from a crypto key cache file '%s'"), *CryptoKeysCacheFilename);
		KeyChainUtilities::LoadKeyChainFromFile(CryptoKeysCacheFilename, OutCryptoSettings);
	}
	else if (FParse::Param(CmdLine, TEXT("encryptionini")))
	{
		FString ProjectDir, EngineDir, Platform;

		if (FParse::Value(CmdLine, TEXT("projectdir="), ProjectDir, false)
			&& FParse::Value(CmdLine, TEXT("enginedir="), EngineDir, false)
			&& FParse::Value(CmdLine, TEXT("platform="), Platform, false))
		{
			UE_LOG(LogIoStore, Warning, TEXT("A legacy command line syntax is being used for crypto config. Please update to using the -cryptokey parameter as soon as possible as this mode is deprecated"));

			FConfigFile EngineConfig;

			FConfigCacheIni::LoadExternalIniFile(EngineConfig, TEXT("Engine"), *FPaths::Combine(EngineDir, TEXT("Config\\")), *FPaths::Combine(ProjectDir, TEXT("Config/")), true, *Platform);
			bool bDataCryptoRequired = false;
			EngineConfig.GetBool(TEXT("PlatformCrypto"), TEXT("PlatformRequiresDataCrypto"), bDataCryptoRequired);

			if (!bDataCryptoRequired)
			{
				return;
			}

			FConfigFile ConfigFile;
			FConfigCacheIni::LoadExternalIniFile(ConfigFile, TEXT("Crypto"), *FPaths::Combine(EngineDir, TEXT("Config\\")), *FPaths::Combine(ProjectDir, TEXT("Config/")), true, *Platform);
			bool bSignPak = false;
			bool bEncryptPakIniFiles = false;
			bool bEncryptPakIndex = false;
			bool bEncryptAssets = false;
			bool bEncryptPak = false;

			if (ConfigFile.Num())
			{
				UE_LOG(LogIoStore, Display, TEXT("Using new format crypto.ini files for crypto configuration"));

				static const TCHAR* SectionName = TEXT("/Script/CryptoKeys.CryptoKeysSettings");

				ConfigFile.GetBool(SectionName, TEXT("bEnablePakSigning"), bSignPak);
				ConfigFile.GetBool(SectionName, TEXT("bEncryptPakIniFiles"), bEncryptPakIniFiles);
				ConfigFile.GetBool(SectionName, TEXT("bEncryptPakIndex"), bEncryptPakIndex);
				ConfigFile.GetBool(SectionName, TEXT("bEncryptAssets"), bEncryptAssets);
				bEncryptPak = bEncryptPakIniFiles || bEncryptPakIndex || bEncryptAssets;

				if (bSignPak)
				{
					FString PublicExpBase64, PrivateExpBase64, ModulusBase64;
					ConfigFile.GetString(SectionName, TEXT("SigningPublicExponent"), PublicExpBase64);
					ConfigFile.GetString(SectionName, TEXT("SigningPrivateExponent"), PrivateExpBase64);
					ConfigFile.GetString(SectionName, TEXT("SigningModulus"), ModulusBase64);

					TArray<uint8> PublicExp, PrivateExp, Modulus;
					FBase64::Decode(PublicExpBase64, PublicExp);
					FBase64::Decode(PrivateExpBase64, PrivateExp);
					FBase64::Decode(ModulusBase64, Modulus);

					OutCryptoSettings.SigningKey = FRSA::CreateKey(PublicExp, PrivateExp, Modulus);

					UE_LOG(LogIoStore, Display, TEXT("Parsed signature keys from config files."));
				}

				if (bEncryptPak)
				{
					FString EncryptionKeyString;
					ConfigFile.GetString(SectionName, TEXT("EncryptionKey"), EncryptionKeyString);

					if (EncryptionKeyString.Len() > 0)
					{
						TArray<uint8> Key;
						FBase64::Decode(EncryptionKeyString, Key);
						check(Key.Num() == sizeof(FAES::FAESKey::Key));
						FNamedAESKey NewKey;
						NewKey.Name = TEXT("Default");
						NewKey.Guid = FGuid();
						FMemory::Memcpy(NewKey.Key.Key, &Key[0], sizeof(FAES::FAESKey::Key));
						OutCryptoSettings.EncryptionKeys.Add(NewKey.Guid, NewKey);
						UE_LOG(LogIoStore, Display, TEXT("Parsed AES encryption key from config files."));
					}
				}
			}
			else
			{
				static const TCHAR* SectionName = TEXT("Core.Encryption");

				UE_LOG(LogIoStore, Display, TEXT("Using old format encryption.ini files for crypto configuration"));

				FConfigCacheIni::LoadExternalIniFile(ConfigFile, TEXT("Encryption"), *FPaths::Combine(EngineDir, TEXT("Config\\")), *FPaths::Combine(ProjectDir, TEXT("Config/")), true, *Platform);
				ConfigFile.GetBool(SectionName, TEXT("SignPak"), bSignPak);
				ConfigFile.GetBool(SectionName, TEXT("EncryptPak"), bEncryptPak);

				if (bSignPak)
				{
					FString RSAPublicExp, RSAPrivateExp, RSAModulus;
					ConfigFile.GetString(SectionName, TEXT("rsa.publicexp"), RSAPublicExp);
					ConfigFile.GetString(SectionName, TEXT("rsa.privateexp"), RSAPrivateExp);
					ConfigFile.GetString(SectionName, TEXT("rsa.modulus"), RSAModulus);

					//TODO: Fix me!
					//OutSigningKey.PrivateKey.Exponent.Parse(RSAPrivateExp);
					//OutSigningKey.PrivateKey.Modulus.Parse(RSAModulus);
					//OutSigningKey.PublicKey.Exponent.Parse(RSAPublicExp);
					//OutSigningKey.PublicKey.Modulus = OutSigningKey.PrivateKey.Modulus;

					UE_LOG(LogIoStore, Display, TEXT("Parsed signature keys from config files."));
				}

				if (bEncryptPak)
				{
					FString EncryptionKeyString;
					ConfigFile.GetString(SectionName, TEXT("aes.key"), EncryptionKeyString);
					FNamedAESKey NewKey;
					NewKey.Name = TEXT("Default");
					NewKey.Guid = FGuid();
					if (EncryptionKeyString.Len() == 32 && TCString<TCHAR>::IsPureAnsi(*EncryptionKeyString))
					{
						for (int32 Index = 0; Index < 32; ++Index)
						{
							NewKey.Key.Key[Index] = (uint8)EncryptionKeyString[Index];
						}
						OutCryptoSettings.EncryptionKeys.Add(NewKey.Guid, NewKey);
						UE_LOG(LogIoStore, Display, TEXT("Parsed AES encryption key from config files."));
					}
				}
			}
		}
	}
	else
	{
		UE_LOG(LogIoStore, Display, TEXT("Using command line for crypto configuration"));

		FString EncryptionKeyString;
		FParse::Value(CmdLine, TEXT("aes="), EncryptionKeyString, false);

		if (EncryptionKeyString.Len() > 0)
		{
			UE_LOG(LogIoStore, Warning, TEXT("A legacy command line syntax is being used for crypto config. Please update to using the -cryptokey parameter as soon as possible as this mode is deprecated"));

			FNamedAESKey NewKey;
			NewKey.Name = TEXT("Default");
			NewKey.Guid = FGuid();
			const uint32 RequiredKeyLength = sizeof(NewKey.Key);

			// Error checking
			if (EncryptionKeyString.Len() < RequiredKeyLength)
			{
				UE_LOG(LogIoStore, Fatal, TEXT("AES encryption key must be %d characters long"), RequiredKeyLength);
			}

			if (EncryptionKeyString.Len() > RequiredKeyLength)
			{
				UE_LOG(LogIoStore, Warning, TEXT("AES encryption key is more than %d characters long, so will be truncated!"), RequiredKeyLength);
				EncryptionKeyString.LeftInline(RequiredKeyLength);
			}

			if (!FCString::IsPureAnsi(*EncryptionKeyString))
			{
				UE_LOG(LogIoStore, Fatal, TEXT("AES encryption key must be a pure ANSI string!"));
			}

			ANSICHAR* AsAnsi = TCHAR_TO_ANSI(*EncryptionKeyString);
			check(TCString<ANSICHAR>::Strlen(AsAnsi) == RequiredKeyLength);
			FMemory::Memcpy(NewKey.Key.Key, AsAnsi, RequiredKeyLength);
			OutCryptoSettings.EncryptionKeys.Add(NewKey.Guid, NewKey);
			UE_LOG(LogIoStore, Display, TEXT("Parsed AES encryption key from command line."));
		}
	}

	FString EncryptionKeyOverrideGuidString;
	FGuid EncryptionKeyOverrideGuid;
	if (FParse::Value(CmdLine, TEXT("EncryptionKeyOverrideGuid="), EncryptionKeyOverrideGuidString))
	{
		UE_LOG(LogIoStore, Display, TEXT("Using encryption key override '%s'"), *EncryptionKeyOverrideGuidString);
		FGuid::Parse(EncryptionKeyOverrideGuidString, EncryptionKeyOverrideGuid);
	}
	OutCryptoSettings.MasterEncryptionKey = OutCryptoSettings.EncryptionKeys.Find(EncryptionKeyOverrideGuid);
}

struct FContainerSourceFile 
{
	FString NormalizedPath;
	FString DestinationPath;
	bool bNeedsCompression = false;
	bool bNeedsEncryption = false;
};

struct FContainerSourceSpec
{
	FName Name;
	FString OutputPath;
	TArray<FContainerSourceFile> SourceFiles;
	FString PatchTargetFile;
	TArray<FString> PatchSourceContainerFiles;
	FGuid EncryptionKeyOverrideGuid;
	bool bGenerateDiffPatch = false;
};

struct FCookedFileStatData
{
	enum EFileExt { UMap, UAsset, UExp, UBulk, UPtnl, UMappedBulk, UShaderByteCode };
	enum EFileType { PackageHeader, PackageData, BulkData, ShaderLibrary };

	int64 FileSize = 0;
	EFileType FileType = PackageHeader;
	EFileExt FileExt = UMap;

	TArray<FFileRegion> FileRegions;
};

using FCookedFileStatMap = TMap<FString, FCookedFileStatData>;

struct FContainerTargetSpec;

struct FShaderInfo
{
	FIoChunkId ChunkId;
	FIoChunkId LibraryChunkId;
	FContainerTargetSpec* ContainerTarget = nullptr;
	uint64 Size = 0;
	uint32 ReferencedByPackagesCount = 0;
	uint32 LibraryOrder = 0;
};

struct FLegacyCookedPackage
{
	FPackageId GlobalPackageId;
	FName PackageName;
	FName RedirectFromPackageName;
	FString FileName;
	uint64 UAssetSize = 0;
	uint64 UExpSize = 0;
	uint64 DiskLayoutOrder = -1;
	FPackageStorePackage* OptimizedPackage = nullptr;
	TSet<FShaderInfo*> Shaders;
	const FPackageStoreEntryResource* PackageStoreEntry = nullptr;
};

enum class EContainerChunkType
{
	ShaderCodeLibrary,
	ShaderCode,
	PackageData,
	BulkData,
	OptionalBulkData,
	MemoryMappedBulkData
};

struct FContainerTargetFile
{
	FContainerTargetSpec* ContainerTarget = nullptr;
	FLegacyCookedPackage* Package = nullptr;
	FString NormalizedSourcePath;
	TOptional<FIoBuffer> SourceBuffer;
	FString TargetPath;
	FString DestinationPath;
	uint64 SourceSize = 0;
	uint64 IdealOrder = 0;
	FIoChunkId ChunkId;
	TArray<uint8> PackageHeaderData;
	EContainerChunkType ChunkType;
	bool bForceUncompressed = false;

	TArray<FFileRegion> FileRegions;
};

class FCookedPackageStore
{
public:
	FCookedPackageStore()
	{
	}

	FIoStatus Load(const FString& CookedDirectory)
	{
		FIoStatus Status = PackageStoreManifest.Load(*(CookedDirectory / FApp::GetProjectName() / TEXT("MetaData") / TEXT("packagestore.manifest")));
		if (!Status.IsOk())
		{
			return Status;
		}

		if (const FPackageStoreManifest::FZenServerInfo* ZenServerInfo = PackageStoreManifest.ReadZenServerInfo())
		{
			DataSource = MakeUnique<FZenStoreDataSource>(*ZenServerInfo);
		}
		else
		{
			TUniquePtr<FIoStoreReaderDataSource> IoStoreReaderDataSource = MakeUnique<FIoStoreReaderDataSource>();
			Status = IoStoreReaderDataSource->Initialize(*(CookedDirectory / TEXT("global")));
			if (!Status.IsOk())
			{
				return Status;
			}
			DataSource = MoveTemp(IoStoreReaderDataSource);
		}

		FIoContainerId ContainerId = FIoContainerId::FromName(TEXT("global"));

		TIoStatusOr<FIoBuffer> HeaderBuffer = DataSource->ReadChunk(CreateIoChunkId(ContainerId.Value(), 0, EIoChunkType::ContainerHeader), FIoReadOptions());
		if (!HeaderBuffer.IsOk())
		{
			return FIoStatusBuilder(EIoErrorCode::ReadError) << TEXT("Failed to read container header");
		}

		FMemoryReaderView Ar(MakeArrayView(HeaderBuffer.ValueOrDie().Data(), HeaderBuffer.ValueOrDie().DataSize()));
		FContainerHeader ContainerHeader;
		Ar << ContainerHeader;

		PackageEntries.Reserve(ContainerHeader.PackageCount);
		TArrayView<const FFilePackageStoreEntry> FilePackageEntries = MakeArrayView<const FFilePackageStoreEntry>(reinterpret_cast<const FFilePackageStoreEntry*>(ContainerHeader.StoreEntries.GetData()), ContainerHeader.PackageCount);

		int32 Idx = 0;
		for (const FFilePackageStoreEntry& FilePackageEntry : FilePackageEntries)
		{
			FPackageStoreEntryResource& Entry = PackageEntries.AddDefaulted_GetRef();
			
			Entry.ExportInfo = FPackageStoreExportInfo
			{
				FilePackageEntry.ExportBundlesSize,
				FilePackageEntry.ExportCount,
				FilePackageEntry.ExportBundleCount,
				FilePackageEntry.LoadOrder
			};
			Entry.ImportedPackageIds = MakeArrayView<const FPackageId>(FilePackageEntry.ImportedPackages.Data(), FilePackageEntry.ImportedPackages.Num());

			PackageIdToEntry.Add(ContainerHeader.PackageIds[Idx++], &Entry);
		}

		for (const FPackageStoreManifest::FFileInfo& FileInfo : PackageStoreManifest.GetFiles())
		{
			FilenameToChunkIdMap.Add(FileInfo.FileName, FileInfo.ChunkId);
		}

		for (const FPackageStoreManifest::FPackageInfo& PackageInfo : PackageStoreManifest.GetPackages())
		{
			FName PackageName = FName(PackageInfo.PackageName);
			PackageNameToPackageInfoMap.Add(PackageName, &PackageInfo);
			ChunkIdToPackageNameMap.Add(PackageInfo.PackageChunkId, PackageName);
			for (const FIoChunkId& ChunkId : PackageInfo.BulkDataChunkIds)
			{
				ChunkIdToPackageNameMap.Add(ChunkId, PackageName);
			}
		}

		return FIoStatus::Ok;
	}

	FIoChunkId GetChunkIdFromFileName(const FString& Filename) const
	{
		return FilenameToChunkIdMap.FindRef(Filename);
	}

	FName GetPackageNameFromChunkId(const FIoChunkId& ChunkId) const
	{
		return ChunkIdToPackageNameMap.FindRef(ChunkId);
	}

	const FPackageStoreManifest::FPackageInfo* GetPackageInfoFromPackageName(FName PackageName)
	{
		return PackageNameToPackageInfoMap.FindRef(PackageName);
	}

	const FPackageStoreEntryResource* GetPackageStoreEntry(FPackageId PackageId) const
	{
		return PackageIdToEntry.FindRef(PackageId);
	}

	TIoStatusOr<uint64> GetChunkSize(const FIoChunkId& ChunkId)
	{
		return DataSource->GetChunkSize(ChunkId);
	}

	void ReadChunkAsync(const FIoChunkId& ChunkId, TFunction<void(TIoStatusOr<FIoBuffer>)>&& Callback)
	{
		class FReadChunkTask
			: public FNonAbandonableTask
		{
		public:
			FReadChunkTask(IDataSource* InDataSource, const FIoChunkId& InChunkId, TFunction<void(TIoStatusOr<FIoBuffer>)>&& InCallback)
				: DataSource(InDataSource)
				, ChunkId(InChunkId)
				, Callback(MoveTemp(InCallback))
			{
			}

			void DoWork()
			{
				Callback(DataSource->ReadChunk(ChunkId, FIoReadOptions()));
			}

			TStatId GetStatId() const
			{
				return TStatId();
			}

		private:
			IDataSource* DataSource;
			FIoChunkId ChunkId;
			TFunction<void(TIoStatusOr<FIoBuffer>)> Callback;
		};

		(new FAutoDeleteAsyncTask<FReadChunkTask>(DataSource.Get(), ChunkId, MoveTemp(Callback)))->StartBackgroundTask();
	}
	
	TIoStatusOr<FIoBuffer> ReadPackageHeader(FPackageId PackageId)
	{
		if (const FPackageStoreEntryResource* Entry = PackageIdToEntry.FindRef(PackageId))
		{
			FIoReadOptions ReadOptions;
			ReadOptions.SetRange(0, 64 << 10);
			
			TIoStatusOr<FIoBuffer> Status = DataSource->ReadChunk(CreateIoChunkId(PackageId.Value(), 0, EIoChunkType::ExportBundleData), ReadOptions);
			if (!Status.IsOk())
			{
				return Status;
			}

			FIoBuffer Buffer = Status.ConsumeValueOrDie();
			uint32 HeaderSize = reinterpret_cast<const FPackageSummary*>(Buffer.Data())->HeaderSize;
			if (HeaderSize > Buffer.DataSize())
			{
				ReadOptions.SetRange(0, HeaderSize);

				Status = DataSource->ReadChunk(CreateIoChunkId(PackageId.Value(), 0, EIoChunkType::ExportBundleData), ReadOptions);
				if (!Status.IsOk())
				{
					return Status;
				}
				Buffer = Status.ConsumeValueOrDie();
			}
			
			return FIoBuffer(Buffer.Data(), HeaderSize, Buffer);
		}

		return FIoStatus(EIoErrorCode::NotFound);
	}

private:
	class IDataSource
	{
	public:
		virtual ~IDataSource() = default;
		virtual TIoStatusOr<uint64> GetChunkSize(const FIoChunkId& ChunkId) = 0;
		virtual TIoStatusOr<FIoBuffer> ReadChunk(const FIoChunkId& ChunkId, const FIoReadOptions& ReadOptions) = 0;
	};

	class FIoStoreReaderDataSource
		: public IDataSource
	{
	public:
		FIoStatus Initialize(const TCHAR* ContainerPath)
		{
			using FEncryptionKeys = TMap<FGuid, FAES::FAESKey>;
			return IoStoreReader.Initialize(ContainerPath, FEncryptionKeys());
		}

		virtual TIoStatusOr<uint64> GetChunkSize(const FIoChunkId& ChunkId) override
		{
			TIoStatusOr<FIoStoreTocChunkInfo> ChunkInfo = IoStoreReader.GetChunkInfo(ChunkId);
			if (!ChunkInfo.IsOk())
			{
				return ChunkInfo.Status();
			}
			return ChunkInfo.ValueOrDie().Size;
		}

		virtual TIoStatusOr<FIoBuffer> ReadChunk(const FIoChunkId& ChunkId, const FIoReadOptions& ReadOptions) override
		{
			return IoStoreReader.Read(ChunkId, ReadOptions);
		}

	private:
		FIoStoreReader IoStoreReader;
	};

	class FZenStoreDataSource
		: public IDataSource
	{
	public:
		FZenStoreDataSource(const FPackageStoreManifest::FZenServerInfo& ZenServerInfo)
		{
			ZenStoreClient = MakeUnique<UE::FZenStoreHttpClient>(ZenServerInfo.HostName, ZenServerInfo.Port);
			ZenStoreClient->InitializeReadOnly(ZenServerInfo.ProjectId, ZenServerInfo.OplogId);
		}

		virtual TIoStatusOr<uint64> GetChunkSize(const FIoChunkId& ChunkId) override
		{
			return ZenStoreClient->GetChunkSize(ChunkId);
		}

		virtual TIoStatusOr<FIoBuffer> ReadChunk(const FIoChunkId& ChunkId, const FIoReadOptions& ReadOptions) override
		{
			return ZenStoreClient->ReadChunk(ChunkId, ReadOptions.GetOffset(), ReadOptions.GetSize());
		}

	private:
		TUniquePtr<UE::FZenStoreHttpClient> ZenStoreClient;
	};
	
	TUniquePtr<IDataSource> DataSource;
	FPackageStoreManifest PackageStoreManifest;
	TArray<FPackageStoreEntryResource> PackageEntries;
	TMap<FPackageId, const FPackageStoreEntryResource*> PackageIdToEntry;
	TMap<FString, FIoChunkId> FilenameToChunkIdMap;
	TMap<FIoChunkId, FName> ChunkIdToPackageNameMap;
	TMap<FName, const FPackageStoreManifest::FPackageInfo*> PackageNameToPackageInfoMap;
};

struct FFileOrderMap
{
	TMap<FName, int64> PackageNameToOrder;
	FString Name;
	int32 Priority;
	int32 Index;

	FFileOrderMap(int32 InPriority, int32 InIndex)
		: Priority(InPriority)
		, Index(InIndex)
	{
	}
};

struct FIoStoreArguments
{
	FString GlobalContainerPath;
	FString CookedDir;
	ITargetPlatform* TargetPlatform = nullptr;
	FString MetaInputDir;
	FString MetaOutputDir;
	TArray<FContainerSourceSpec> Containers;
	FCookedFileStatMap CookedFileStatMap;
	TArray<FFileOrderMap> OrderMaps;
	FKeyChain KeyChain;
	FKeyChain PatchKeyChain;
	FString DLCPluginPath;
	FString DLCName;
	FString BasedOnReleaseVersionPath;
	FAssetRegistryState ReleaseAssetRegistry;
	FReleasedPackages ReleasedPackages;
	TUniquePtr<FCookedPackageStore> PackageStore;
	bool bSign = false;
	bool bRemapPluginContentToGame = false;
	bool bCreateDirectoryIndex = true;
	bool bClusterByOrderFilePriority = false;
	bool bFileRegions = false;

	bool ShouldCreateContainers() const
	{
		return GlobalContainerPath.Len() > 0 || DLCPluginPath.Len() > 0;
	}

	bool IsDLC() const
	{
		return DLCPluginPath.Len() > 0;
	}
};

struct FContainerTargetSpec
{
	FIoContainerId ContainerId;
	FContainerHeader Header;
	FName Name;
	FGuid EncryptionKeyGuid;
	FString OutputPath;
	FIoStoreWriter* IoStoreWriter;
	TArray<FContainerTargetFile> TargetFiles;
	TArray<TUniquePtr<FIoStoreReader>> PatchSourceReaders;
	EIoContainerFlags ContainerFlags = EIoContainerFlags::None;
	TArray<FLegacyCookedPackage*> Packages;
	TArray<FShaderInfo> Shaders;
	bool bGenerateDiffPatch = false;
};

using FPackageNameMap = TMap<FName, FLegacyCookedPackage*>;
using FPackageIdMap = TMap<FPackageId, FLegacyCookedPackage*>;

#if OUTPUT_CHUNKID_DIRECTORY
class FChunkIdCsv
{
public:

	~FChunkIdCsv()
	{
		if (OutputArchive)
		{
			OutputArchive->Flush();
		}
	}

	void CreateOutputFile(const FString& RootPath)
	{
		const FString OutputFilename = RootPath / TEXT("chunkid_directory.csv");
		OutputArchive.Reset(IFileManager::Get().CreateFileWriter(*OutputFilename));
		if (OutputArchive)
		{
			const ANSICHAR* Output = "NameIndex,NameNumber,ChunkIndex,ChunkType,ChunkIdHash,DebugString\n";
			OutputArchive->Serialize((void*)Output, FPlatformString::Strlen(Output));
		}
	}

	void AddChunk(uint32 NameIndex, uint32 NameNumber, uint16 ChunkIndex, uint8 ChunkType, uint32 ChunkIdHash, const TCHAR* DebugString)
	{
		ANSICHAR Buffer[MAX_SPRINTF + 1] = { 0 };
		int32 NumOfCharacters = FCStringAnsi::Sprintf(Buffer, "%u,%u,%u,%u,%u,%s\n", NameIndex, NameNumber, ChunkIndex, ChunkType, ChunkIdHash, TCHAR_TO_ANSI(DebugString));
		OutputArchive->Serialize(Buffer, NumOfCharacters);
	}	

private:
	TUniquePtr<FArchive> OutputArchive;
};
FChunkIdCsv ChunkIdCsv;

#endif

// If bClusterByOrderFilePriority is false
//		Order packages by the order of OrderMaps with their associated priority 
//		e.g. Order map 1 with priority 0 (A, B, C) 
//			 Order map 2 with priority 10 (B, D)
//			Final order: (A, C, B, D)
//		Then cluster packages in this order. 
// If bClusterByOrderFilePriority is true
//		Cluster packages first by OrderMaps in priority order, then concatenate clusters in the array order of OrderMaps.
//		e.g. Order map 1 with priority 0 (A, B, C) 
//			 Order map 2 with priority 10 (B, D)
//			Cluster packages B, D, then A, C
//			Then reassemble clusters A, C, B, D
static void AssignPackagesDiskOrder(
	const TArray<FLegacyCookedPackage*>& Packages,
	const TArray<FFileOrderMap>& OrderMaps,
	const FPackageIdMap& PackageIdMap,
	bool bClusterByOrderFilePriority
	)
{
	IOSTORE_CPU_SCOPE(AssignPackagesDiskOrder);

	struct FCluster
	{
		TArray<FLegacyCookedPackage*> Packages;
		int32 OrderFileIndex; // Index in OrderMaps of the FFileOrderMap which contained Packages.Last()

		FCluster(int32 InOrderFileIndex)
			: OrderFileIndex(InOrderFileIndex)
		{
		}
	};

	TArray<FCluster*> Clusters;
	TSet<FLegacyCookedPackage*> AssignedPackages;
	TArray<FLegacyCookedPackage*> ProcessStack;

	struct FPackageAndOrder
	{
		FLegacyCookedPackage* Package = nullptr;
		int64 LocalOrder;
		const FFileOrderMap* OrderMap;

		FPackageAndOrder(FLegacyCookedPackage* InPackage, int64 InLocalOrder, const FFileOrderMap* InOrderMap)
			: Package(InPackage)
			, LocalOrder(InLocalOrder)
			, OrderMap(InOrderMap)
		{
			check(OrderMap);
		}
	};

	// Order maps sorted by priority
	TArray<const FFileOrderMap*> PriorityOrderMaps;
	for (const FFileOrderMap& Map : OrderMaps)
	{
		PriorityOrderMaps.Add(&Map);
	}
	Algo::StableSortBy(PriorityOrderMaps, [](const FFileOrderMap* Map) { return Map->Priority; }, TGreater<int32>());

	// Create a fallback order map to avoid null checks later
	// Lowest priority, last index
	FFileOrderMap FallbackOrderMap(MIN_int32, MAX_int32);

	TArray<FPackageAndOrder> SortedPackages;
	SortedPackages.Reserve(Packages.Num());
	for (FLegacyCookedPackage* Package : Packages)
	{
		if (!Package->OptimizedPackage->GetExportBundleCount())
		{
			continue;
		}

		// Default to the fallback order map 
		// Reverse the bundle load order for the fallback map (so that packages are considered before their imports)
		const FFileOrderMap* UsedOrderMap = &FallbackOrderMap;
		int64 LocalOrder = -int64(Package->OptimizedPackage->GetLoadOrder());

		for (const FFileOrderMap* OrderMap : PriorityOrderMaps)
		{
			if (const int64* Order = OrderMap->PackageNameToOrder.Find(Package->PackageName))
			{
				LocalOrder = *Order;
				UsedOrderMap = OrderMap;
				break;
			}
		}

		SortedPackages.Emplace(Package, LocalOrder, UsedOrderMap);
	}
	const FFileOrderMap* LastBlameOrderMap = nullptr;
	int32 LastAssignedCount = 0;

	if (bClusterByOrderFilePriority)
	{
		// Sort by priority of the order map
		Algo::Sort(SortedPackages, [](const FPackageAndOrder& A, const FPackageAndOrder& B) {
			// Packages in the same order map should be sorted by their local ordering
			if (A.OrderMap == B.OrderMap)
			{
				return A.LocalOrder < B.LocalOrder;
			}

			// First priority, then index
			if (A.OrderMap->Priority != B.OrderMap->Priority)
			{
				return A.OrderMap->Priority > B.OrderMap->Priority;
			}

			check(A.OrderMap->Index != B.OrderMap->Index);
			return A.OrderMap->Index < B.OrderMap->Index;
		});
	}
	else
	{
		// Sort by the order of the order map (...)
		Algo::Sort(SortedPackages, [](const FPackageAndOrder& A, const FPackageAndOrder& B) {
			// Packages in the same order map should be sorted by their local ordering
			if (A.OrderMap == B.OrderMap)
			{
				return A.LocalOrder < B.LocalOrder;
			}

			// Blame order priority is not considered for the order in which we cluster packages, only for the order in which we assign packages to an order map
			return A.OrderMap->Index < B.OrderMap->Index;
		});
	}

	for (FPackageAndOrder& Entry : SortedPackages)
	{
		checkSlow(Entry.OrderMap); // Without this, Entry.OrderMap != LastBlameOrderMap convinces static analysis that Entry.OrderMap may be null
		if (Entry.OrderMap != LastBlameOrderMap)
		{
			if( LastBlameOrderMap != nullptr )
			{
				UE_LOG(LogIoStore, Display, TEXT("Ordered %d/%d packages using order file %s"), AssignedPackages.Num() - LastAssignedCount, Packages.Num(), *LastBlameOrderMap->Name);
			}
			LastAssignedCount = AssignedPackages.Num();
			LastBlameOrderMap = Entry.OrderMap;
		}
		if (!AssignedPackages.Contains(Entry.Package))
		{
			FCluster* Cluster = new FCluster(Entry.OrderMap->Index);
			Clusters.Add(Cluster);
			ProcessStack.Push(Entry.Package);

			while (ProcessStack.Num())
			{
				FLegacyCookedPackage* PackageToProcess = ProcessStack.Pop(false);
				if (!AssignedPackages.Contains(PackageToProcess))
				{
					AssignedPackages.Add(PackageToProcess);
					Cluster->Packages.Add(PackageToProcess);
					TArray<FPackageId> AllReferencedPackageIds;
					AllReferencedPackageIds.Append(PackageToProcess->OptimizedPackage->GetImportedPackageIds());
					AllReferencedPackageIds.Append(PackageToProcess->OptimizedPackage->GetImportedRedirectedPackageIds().Array());
					for (const FPackageId& ReferencedPackageId : AllReferencedPackageIds)
					{
						FLegacyCookedPackage* FindReferencedPackage = PackageIdMap.FindRef(ReferencedPackageId);
						if (FindReferencedPackage)
						{
							ProcessStack.Push(FindReferencedPackage);
						}
					}
				}
			}
		}
	}
	UE_LOG(LogIoStore, Display, TEXT("Ordered %d packages using fallback bundle order"), AssignedPackages.Num() - LastAssignedCount);

	check(AssignedPackages.Num() == Packages.Num());
	
	if (bClusterByOrderFilePriority)
	{
		Algo::StableSortBy(Clusters, [](FCluster* Cluster) { return Cluster->OrderFileIndex; });
	}
	
	for (FCluster* Cluster : Clusters)
	{
		Algo::Sort(Cluster->Packages, [](const FLegacyCookedPackage* A, const FLegacyCookedPackage* B)
		{
			return A->OptimizedPackage->GetLoadOrder() < B->OptimizedPackage->GetLoadOrder();
		});
	}

	uint64 LayoutIndex = 0;
	for (FCluster* Cluster : Clusters)
	{
		for (FLegacyCookedPackage* Package : Cluster->Packages)
		{
			Package->DiskLayoutOrder = LayoutIndex++;
		}
		delete Cluster;
	}
}

static void CreateDiskLayout(
	const TArray<FContainerTargetSpec*>& ContainerTargets,
	const TArray<FLegacyCookedPackage*>& Packages,
	const TArray<FFileOrderMap>& OrderMaps,
	const FPackageIdMap& PackageIdMap,
	bool bClusterByOrderFilePriority)
{
	IOSTORE_CPU_SCOPE(CreateDiskLayout);

	AssignPackagesDiskOrder(Packages, OrderMaps, PackageIdMap, bClusterByOrderFilePriority);

	for (FContainerTargetSpec* ContainerTarget : ContainerTargets)
	{
		TArray<FContainerTargetFile*> SortedTargetFiles;
		SortedTargetFiles.Reserve(ContainerTarget->TargetFiles.Num());
		TMap<FIoChunkId, FContainerTargetFile*> ShaderTargetFilesMap;
		ShaderTargetFilesMap.Reserve(ContainerTarget->Shaders.Num());
		for (FContainerTargetFile& TargetFile : ContainerTarget->TargetFiles)
		{
			if (TargetFile.ChunkType == EContainerChunkType::ShaderCode)
			{
				ShaderTargetFilesMap.Add(TargetFile.ChunkId, &TargetFile);
			}
			else
			{
				SortedTargetFiles.Add(&TargetFile);
			}
		}
		Algo::Sort(SortedTargetFiles, [](const FContainerTargetFile* A, const FContainerTargetFile* B)
		{
			if (A->ChunkType != B->ChunkType)
			{
				return A->ChunkType < B->ChunkType;
			}
			if (A->ChunkType == EContainerChunkType::ShaderCodeLibrary)
				return A->TargetPath < B->TargetPath;
			if (A->Package != B->Package)
			{
				return A->Package->DiskLayoutOrder < B->Package->DiskLayoutOrder;
			}
			check(A == B)
			return false;
		});

		TArray<const FShaderInfo*> SharedShaders;
		TSet<const FShaderInfo*> AddedSharedShaders;
		int32 Index = 0;
		int32 ShaderCodeInsertionIndex = -1;
		while (Index < SortedTargetFiles.Num())
		{
			FContainerTargetFile* TargetFile = SortedTargetFiles[Index];
			if (ShaderCodeInsertionIndex < 0 && TargetFile->ChunkType != EContainerChunkType::ShaderCodeLibrary)
			{
				ShaderCodeInsertionIndex = Index;
			}
			if (TargetFile->ChunkType == EContainerChunkType::PackageData)
			{
				TArray<FContainerTargetFile*, TInlineAllocator<1024>> PackageUniqueShaders;
				for (FShaderInfo* Shader : TargetFile->Package->Shaders)
				{
					if (Shader->ContainerTarget == ContainerTarget)
					{
						check(Shader->ReferencedByPackagesCount > 0);
						if (Shader->ReferencedByPackagesCount == 1)
						{
							FContainerTargetFile* ShaderTargetFile = ShaderTargetFilesMap.FindRef(Shader->ChunkId);
							check(ShaderTargetFile);
							PackageUniqueShaders.Add(ShaderTargetFile);
						}
						else if (!AddedSharedShaders.Contains(Shader))
						{
							SharedShaders.Add(Shader);
							AddedSharedShaders.Add(Shader);
						}
					}
				}
				if (!PackageUniqueShaders.IsEmpty())
				{
					SortedTargetFiles.Insert(PackageUniqueShaders, Index + 1);
					Index += PackageUniqueShaders.Num();
				}
			}
			++Index;
		}
		if (ShaderCodeInsertionIndex < 0)
		{
			ShaderCodeInsertionIndex = 0;
		}

		TArray<const FShaderInfo*> GlobalShaders;
		for (const FShaderInfo& Shader : ContainerTarget->Shaders)
		{
			if (Shader.ReferencedByPackagesCount == 0)
			{
				GlobalShaders.Add(&Shader);
			}
		}
		if (!GlobalShaders.IsEmpty())
		{
			TArray<FContainerTargetFile*> GlobalShaderTargetFiles;
			GlobalShaderTargetFiles.Reserve(GlobalShaders.Num());
			for (const FShaderInfo* ShaderInfo : GlobalShaders)
			{
				FContainerTargetFile* ShaderTargetFile = ShaderTargetFilesMap.FindRef(ShaderInfo->ChunkId);
				check(ShaderTargetFile);
				GlobalShaderTargetFiles.Add(ShaderTargetFile);
			}
			SortedTargetFiles.Insert(GlobalShaderTargetFiles, ShaderCodeInsertionIndex);
			ShaderCodeInsertionIndex += GlobalShaderTargetFiles.Num();
		}
		if (!SharedShaders.IsEmpty())
		{
			TArray<FContainerTargetFile*> SharedShaderTargetFiles;
			SharedShaderTargetFiles.Reserve(SharedShaders.Num());
			for (const FShaderInfo* ShaderInfo : SharedShaders)
			{
				FContainerTargetFile* ShaderTargetFile = ShaderTargetFilesMap.FindRef(ShaderInfo->ChunkId);
				check(ShaderTargetFile);
				SharedShaderTargetFiles.Add(ShaderTargetFile);
			}
			SortedTargetFiles.Insert(SharedShaderTargetFiles, ShaderCodeInsertionIndex);
		}

		check(SortedTargetFiles.Num() == ContainerTarget->TargetFiles.Num());

		uint64 IdealOrder = 0;
		for (FContainerTargetFile* TargetFile : SortedTargetFiles)
		{
			TargetFile->IdealOrder = IdealOrder++;
		}
	}
}

FContainerTargetSpec* AddContainer(
	FName Name,
	TArray<FContainerTargetSpec*>& Containers)
{
	FIoContainerId ContainerId = FIoContainerId::FromName(Name);
	for (FContainerTargetSpec* ExistingContainer : Containers)
	{
		if (ExistingContainer->Name == Name)
		{
			UE_LOG(LogIoStore, Fatal, TEXT("Duplicate container name: '%s'"), *Name.ToString());
			return nullptr;
		}
		if (ExistingContainer->ContainerId == ContainerId)
		{
			UE_LOG(LogIoStore, Fatal, TEXT("Hash collision for container names: '%s' and '%s'"), *Name.ToString(), *ExistingContainer->Name.ToString());
			return nullptr;
		}
	}
	
	FContainerTargetSpec* ContainerTargetSpec = new FContainerTargetSpec();
	ContainerTargetSpec->Name = Name;
	ContainerTargetSpec->ContainerId = ContainerId;
	Containers.Add(ContainerTargetSpec);
	return ContainerTargetSpec;
}

FLegacyCookedPackage* FindOrAddPackage(
	const FIoStoreArguments& Arguments,
	const FName& PackageName,
	TArray<FLegacyCookedPackage*>& Packages,
	FPackageNameMap& PackageNameMap,
	FPackageIdMap& PackageIdMap)
{
	FLegacyCookedPackage* Package = PackageNameMap.FindRef(PackageName);
	if (!Package)
	{
		FPackageId PackageId = FPackageId::FromName(PackageName);
		if (FLegacyCookedPackage* FindById = PackageIdMap.FindRef(PackageId))
		{
			UE_LOG(LogIoStore, Fatal, TEXT("Package name hash collision \"%s\" and \"%s"), *FindById->PackageName.ToString(), *PackageName.ToString());
		}

		if (const FName* ReleasedPackageName = Arguments.ReleasedPackages.PackageIdToName.Find(PackageId))
		{
			UE_LOG(LogIoStore, Fatal, TEXT("Package name hash collision \"%s\" and \"%s"), *ReleasedPackageName->ToString(), *PackageName.ToString());
		}

		Package = new FLegacyCookedPackage();
		Package->PackageName = PackageName;
		Package->GlobalPackageId = PackageId;
		Packages.Add(Package);
		PackageNameMap.Add(PackageName, Package);
		PackageIdMap.Add(PackageId, Package);
	}

	return Package;
}

FLegacyCookedPackage* FindOrAddPackage(
	const FIoStoreArguments& Arguments,
	const TCHAR* RelativeFileName,
	TArray<FLegacyCookedPackage*>& Packages,
	FPackageNameMap& PackageNameMap,
	FPackageIdMap& PackageIdMap)
{
	FString PackageName;
	FString ErrorMessage;
	if (!FPackageName::TryConvertFilenameToLongPackageName(RelativeFileName, PackageName, &ErrorMessage))
	{
		UE_LOG(LogIoStore, Warning, TEXT("Failed to obtain package name from file name '%s'"), *ErrorMessage);
		return nullptr;
	}

	FName PackageFName = *PackageName;
	return FindOrAddPackage(Arguments, PackageFName, Packages, PackageNameMap, PackageIdMap);
}

static void ParsePackageAssetsFromFiles(TArray<FLegacyCookedPackage*>& Packages, const FPackageStoreOptimizer& PackageStoreOptimizer)
{
	IOSTORE_CPU_SCOPE(ParsePackageAssetsFromFiles);
	UE_LOG(LogIoStore, Display, TEXT("Parsing packages..."));

	TAtomic<int32> ReadCount {0};
	TAtomic<int32> ParseCount {0};
	const int32 TotalPackageCount = Packages.Num();

	TArray<FPackageFileSummary> PackageFileSummaries;
	PackageFileSummaries.SetNum(TotalPackageCount);

	uint8* UAssetMemory = nullptr;
	TArray<uint8*> PackageAssetBuffers;
	PackageAssetBuffers.SetNum(TotalPackageCount);

	UE_LOG(LogIoStore, Display, TEXT("Reading package assets..."));
	{
		IOSTORE_CPU_SCOPE(ReadUAssetFiles);

		uint64 TotalUAssetSize = 0;
		for (const FLegacyCookedPackage* Package : Packages)
		{
			TotalUAssetSize += Package->UAssetSize;
		}
		UAssetMemory = reinterpret_cast<uint8*>(FMemory::Malloc(TotalUAssetSize));
		uint8* UAssetMemoryPtr = UAssetMemory;
		for (int32 Index = 0; Index < TotalPackageCount; ++Index)
		{
			PackageAssetBuffers[Index] = UAssetMemoryPtr;
			UAssetMemoryPtr += Packages[Index]->UAssetSize;
		}

		TAtomic<uint64> CurrentFileIndex{ 0 };
		ParallelFor(TotalPackageCount, [&ReadCount, &PackageAssetBuffers, &Packages, &CurrentFileIndex](int32 Index)
		{
			TRACE_CPUPROFILER_EVENT_SCOPE(ReadUAssetFile);
			FLegacyCookedPackage* Package = Packages[Index];
			if (!Package->UAssetSize)
			{
				return;
			}
			uint8* Buffer = PackageAssetBuffers[Index];
			IFileHandle* FileHandle = FPlatformFileManager::Get().GetPlatformFile().OpenRead(*Package->FileName);
			if (FileHandle)
			{
				bool bSuccess = FileHandle->Read(Buffer, Package->UAssetSize);
				UE_CLOG(!bSuccess, LogIoStore, Warning, TEXT("Failed reading file '%s'"), *Package->FileName);
				delete FileHandle;
			}
			else
			{
				UE_LOG(LogIoStore, Warning, TEXT("Couldn't open file '%s'"), *Package->FileName);
			}
			uint64 LocalFileIndex = CurrentFileIndex.IncrementExchange() + 1;
			UE_CLOG(LocalFileIndex % 1000 == 0, LogIoStore, Display, TEXT("Reading %d/%d: '%s'"), LocalFileIndex, Packages.Num(), *Package->FileName);
		}, EParallelForFlags::Unbalanced);
	}

	{
		IOSTORE_CPU_SCOPE(SerializeSummaries);

		ParallelFor(TotalPackageCount, [
				&ReadCount,
				&PackageAssetBuffers,
				&PackageFileSummaries,
				&Packages,
				&PackageStoreOptimizer](int32 Index)
		{
			uint8* PackageBuffer = PackageAssetBuffers[Index];
			FLegacyCookedPackage* Package = Packages[Index];

			if (Package->UAssetSize)
			{
				FIoBuffer CookedHeaderBuffer = FIoBuffer(FIoBuffer::Wrap, PackageBuffer, Package->UAssetSize);
				Package->OptimizedPackage = PackageStoreOptimizer.CreatePackageFromCookedHeader(Package->PackageName, CookedHeaderBuffer);
			}
			else
			{
				Package->OptimizedPackage = PackageStoreOptimizer.CreateMissingPackage(Package->PackageName);
			}
			check(Package->OptimizedPackage->GetId() == Package->GlobalPackageId);
		}, EParallelForFlags::Unbalanced);
	}

	FMemory::Free(UAssetMemory);
}

static void ParsePackageAssetsFromPackageStore(FCookedPackageStore& PackageStore, TArray<FLegacyCookedPackage*>& Packages, const FPackageStoreOptimizer& PackageStoreOptimizer)
{
	IOSTORE_CPU_SCOPE(ParsePackageAssetsFromPackageStore);
	UE_LOG(LogIoStore, Display, TEXT("Parsing packages..."));

	const int32 TotalPackageCount = Packages.Num();
	
	ParallelFor(TotalPackageCount, [
		&PackageStore,
		&PackageStoreOptimizer,
		&Packages](int32 Index)
	{
		FLegacyCookedPackage* Package = Packages[Index];
		TIoStatusOr<FIoBuffer> HeaderBuffer = PackageStore.ReadPackageHeader(Package->GlobalPackageId);
		Package->UExpSize = Package->UAssetSize - HeaderBuffer.ValueOrDie().DataSize();
		Package->UAssetSize = HeaderBuffer.ValueOrDie().DataSize();
		Package->OptimizedPackage = PackageStoreOptimizer.CreatePackageFromPackageStoreHeader(Package->PackageName, HeaderBuffer.ValueOrDie(), *Package->PackageStoreEntry);
		check(Package->OptimizedPackage->GetId() == Package->GlobalPackageId);
	}, EParallelForFlags::Unbalanced);
}

TUniquePtr<FIoStoreReader> CreateIoStoreReader(const TCHAR* Path, const FKeyChain& KeyChain)
{
	TUniquePtr<FIoStoreReader> IoStoreReader(new FIoStoreReader());

	TMap<FGuid, FAES::FAESKey> DecryptionKeys;
	for (const auto& KV : KeyChain.EncryptionKeys)
	{
		DecryptionKeys.Add(KV.Key, KV.Value.Key);
	}
	FIoStatus Status = IoStoreReader->Initialize(*FPaths::ChangeExtension(Path, TEXT("")), DecryptionKeys);
	if (Status.IsOk())
	{
		return IoStoreReader;
	}
	else
	{
		UE_LOG(LogIoStore, Warning, TEXT("Failed creating IoStore reader '%s' [%s]"), Path, *Status.ToString())
		return nullptr;
	}
}

TArray<TUniquePtr<FIoStoreReader>> CreatePatchSourceReaders(const TArray<FString>& Files, const FIoStoreArguments& Arguments)
{
	TArray<TUniquePtr<FIoStoreReader>> Readers;
	for (const FString& PatchSourceContainerFile : Files)
	{
		TUniquePtr<FIoStoreReader> Reader = CreateIoStoreReader(*PatchSourceContainerFile, Arguments.PatchKeyChain);
		if (Reader.IsValid())
		{
			UE_LOG(LogIoStore, Display, TEXT("Loaded patch source container '%s'"), *PatchSourceContainerFile);
			Readers.Add(MoveTemp(Reader));
		}
	}
	return Readers;
}

void ProcessShaderLibraries(const FIoStoreArguments& Arguments, TArray<FContainerTargetSpec*>& ContainerTargets)
{
	IOSTORE_CPU_SCOPE(ProcessShaderLibraries);

	TMap<FIoChunkId, FSHAHash> AllShaderCodeHashes;
	uint64 LibraryOrder = 0;
	for (FContainerTargetSpec* ContainerTarget : ContainerTargets)
	{
		TMap<FSHAHash, TArray<FIoChunkId>> ShaderChunkIdsByShaderMapHash;
		TSet<FIoChunkId> AddedShaderChunkIds;
		TArray<FContainerTargetFile> ShaderTargetFiles;
		TArray<FShaderInfo> Shaders;
		for (FContainerTargetFile& TargetFile : ContainerTarget->TargetFiles)
		{
			if (TargetFile.ChunkType == EContainerChunkType::ShaderCodeLibrary)
			{
				TArray<TTuple<FSHAHash, TArray<FIoChunkId>>> ShaderMaps;
				TMap<FIoChunkId, FIoBuffer> ShaderCodeMap;
				TTuple<FIoChunkId, FIoBuffer> LibraryChunk;
				TArray<TTuple<FIoChunkId, FIoBuffer, FSHAHash>> CodeChunks;
				TArray<TTuple<FSHAHash, TSet<FName>>> ShaderMapAssetAssociations;
				if (!FShaderLibraryCooker::ConvertToIoStoreShaderLibrary(*TargetFile.NormalizedSourcePath, LibraryChunk, CodeChunks, ShaderMaps, ShaderMapAssetAssociations))
				{
					UE_LOG(LogIoStore, Warning, TEXT("Failed converting shader library '%s'"), *TargetFile.NormalizedSourcePath);
					continue;
				}
				TargetFile.ChunkId = LibraryChunk.Key;
				TargetFile.SourceBuffer.Emplace(LibraryChunk.Value);
				for (const TTuple<FIoChunkId, FIoBuffer, FSHAHash>& CodeChunk : CodeChunks)
				{
					ShaderCodeMap.Add(CodeChunk.Get<0>(), CodeChunk.Get<1>());
					FSHAHash* FindShaderCodeHash = AllShaderCodeHashes.Find(CodeChunk.Get<0>());
					if (FindShaderCodeHash)
					{
						UE_CLOG(*FindShaderCodeHash != CodeChunk.Get<2>(), LogIoStore, Fatal, TEXT("Shader code chunk id collision"));
					}
					else
					{
						AllShaderCodeHashes.Add(CodeChunk.Get<0>(), CodeChunk.Get<2>());
					}
				}
				TMap<FName, TSet<FSHAHash>> PackageNameToShaderMaps;
				for (const TTuple<FSHAHash, TSet<FName>>& ShaderMapAssetAssociation : ShaderMapAssetAssociations)
				{
					for (const FName& PackageName : ShaderMapAssetAssociation.Value)
					{
						TSet<FSHAHash>& PackageShaderMaps = PackageNameToShaderMaps.FindOrAdd(PackageName);
						PackageShaderMaps.Add(ShaderMapAssetAssociation.Key);
					}
				}
				for (FLegacyCookedPackage* Package : ContainerTarget->Packages)
				{
					TSet<FSHAHash>* FindShaderMapHashes = PackageNameToShaderMaps.Find(Package->PackageName);
					if (FindShaderMapHashes)
					{
						for (const FSHAHash& ShaderMapHash : *FindShaderMapHashes)
						{
							Package->OptimizedPackage->AddShaderMapHash(ShaderMapHash);
						}
					}
				}

				for (TTuple<FSHAHash, TArray<FIoChunkId>>& ShaderMap : ShaderMaps)
				{
					for (const FIoChunkId& ShaderChunkId : ShaderMap.Value)
					{
						if (!AddedShaderChunkIds.Contains(ShaderChunkId))
						{
							FContainerTargetFile& ShaderTargetFile = ShaderTargetFiles.AddDefaulted_GetRef();
							ShaderTargetFile.ContainerTarget = ContainerTarget;
							ShaderTargetFile.ChunkId = ShaderChunkId;
							ShaderTargetFile.ChunkType = EContainerChunkType::ShaderCode;
							ShaderTargetFile.bForceUncompressed = true;

							FShaderInfo& ShaderInfo = Shaders.AddDefaulted_GetRef();
							ShaderInfo.ChunkId = ShaderChunkId;
							ShaderInfo.LibraryChunkId = TargetFile.ChunkId;
							ShaderInfo.ContainerTarget = ContainerTarget;
							ShaderInfo.LibraryOrder = LibraryOrder++;

							FIoBuffer* FindCodeIoBuffer = ShaderCodeMap.Find(ShaderChunkId);
							check(FindCodeIoBuffer);
							ShaderTargetFile.SourceBuffer.Emplace(*FindCodeIoBuffer);
							ShaderTargetFile.SourceSize = FindCodeIoBuffer->DataSize();
							ShaderInfo.Size = FindCodeIoBuffer->DataSize();

							AddedShaderChunkIds.Add(ShaderChunkId);
						}
					}
					ShaderChunkIdsByShaderMapHash.Add(ShaderMap.Key, MoveTemp(ShaderMap.Value));
				}
			}
		}

		check(ShaderTargetFiles.Num() == Shaders.Num());

		if (ShaderTargetFiles.Num() == 0)
		{ 
			continue;
		}

		ContainerTarget->TargetFiles.Append(ShaderTargetFiles);
		ContainerTarget->Shaders.Append(Shaders);

		TMap<FIoChunkId, FShaderInfo*> ShaderInfoByChunkIdMap;
		ShaderInfoByChunkIdMap.Reserve(ContainerTarget->Shaders.Num());
		for (FShaderInfo& ShaderInfo : ContainerTarget->Shaders)
		{
			ShaderInfoByChunkIdMap.Add(ShaderInfo.ChunkId, &ShaderInfo);
		}

		for (FLegacyCookedPackage* Package : ContainerTarget->Packages)
		{
			for (const FSHAHash& ShaderMapHash : Package->OptimizedPackage->GetShaderMapHashes())
			{
				const TArray<FIoChunkId>* FindChunkIds = ShaderChunkIdsByShaderMapHash.Find(ShaderMapHash);
				if (!FindChunkIds)
				{
					UE_LOG(LogIoStore, Warning, TEXT("Package '%s' in '%s' referencing missing shader map '%s'"), *Package->PackageName.ToString(), *ContainerTarget->Name.ToString(), *ShaderMapHash.ToString());
					continue;
				}
				for (const FIoChunkId& ShaderChunkId : *FindChunkIds)
				{
					FShaderInfo* ShaderInfo = ShaderInfoByChunkIdMap.FindRef(ShaderChunkId);
					check(ShaderInfo);
					Package->Shaders.Add(ShaderInfo);
					++ShaderInfo->ReferencedByPackagesCount;
				}
			}
		}
	}
}

void InitializeContainerTargetsAndPackages(
	const FIoStoreArguments& Arguments,
	TArray<FLegacyCookedPackage*>& Packages,
	FPackageNameMap& PackageNameMap,
	FPackageIdMap& PackageIdMap,
	TArray<FContainerTargetSpec*>& ContainerTargets,
	FPackageStoreOptimizer& PackageStoreOptimizer)
{
	FString ProjectName = FApp::GetProjectName();
	FString RelativeEnginePath = FPaths::GetRelativePathToRoot();
	FString RelativeProjectPath = FPaths::ProjectDir();
	int32 CookedEngineDirLen = Arguments.CookedDir.Len() + 1;;
	int32 CookedProjectDirLen = CookedEngineDirLen + ProjectName.Len() + 1;

	auto ConvertCookedPathToRelativePath = [
		&ProjectName,
		&CookedEngineDirLen,
		&CookedProjectDirLen,
		&RelativeEnginePath,
		&RelativeProjectPath]
		(const FString& CookedFile) -> FString
	{
		FString RelativeFileName;
		const TCHAR* FileName = *CookedFile + CookedEngineDirLen;
		if (FCString::Strncmp(FileName, *ProjectName, ProjectName.Len()))
		{
			int32 FileNameLen = CookedFile.Len() - CookedEngineDirLen;
			RelativeFileName.Reserve(RelativeEnginePath.Len() + FileNameLen);
			RelativeFileName = RelativeEnginePath;
			RelativeFileName.AppendChars(*CookedFile + CookedEngineDirLen, FileNameLen);
		}
		else
		{
			FileName = *CookedFile + CookedProjectDirLen;
			int32 FileNameLen = CookedFile.Len() - CookedProjectDirLen;
			RelativeFileName.Reserve(RelativeProjectPath.Len() + FileNameLen);
			RelativeFileName = RelativeProjectPath;
			RelativeFileName.AppendChars(*CookedFile + CookedProjectDirLen, FileNameLen);
		}
		return RelativeFileName;
	};

	auto CreateTargetFileFromCookedFile = [
		&Arguments,
		&ConvertCookedPathToRelativePath,
		&Packages,
		&PackageNameMap,
		&PackageIdMap](const FContainerSourceFile& SourceFile, const FString& RelativeFileName, FContainerTargetFile& OutTargetFile) -> bool
	{
		const FCookedFileStatData* OriginalCookedFileStatData = Arguments.CookedFileStatMap.Find(SourceFile.NormalizedPath);
		if (!OriginalCookedFileStatData)
		{
			UE_LOG(LogIoStore, Warning, TEXT("File not found: '%s'"), *SourceFile.NormalizedPath);
			return false;
		}

		const FCookedFileStatData* CookedFileStatData = OriginalCookedFileStatData;
		if (CookedFileStatData->FileType == FCookedFileStatData::PackageHeader)
		{
			OutTargetFile.NormalizedSourcePath = FPaths::ChangeExtension(SourceFile.NormalizedPath, TEXT(".uexp"));
			CookedFileStatData = Arguments.CookedFileStatMap.Find(OutTargetFile.NormalizedSourcePath);
			if (!CookedFileStatData)
			{
				UE_LOG(LogIoStore, Warning, TEXT("File not found: '%s'"), *OutTargetFile.NormalizedSourcePath);
				return false;
			}
		}
		else
		{
			OutTargetFile.NormalizedSourcePath = SourceFile.NormalizedPath;
		}
		OutTargetFile.SourceSize = uint64(CookedFileStatData->FileSize);
		
		if (CookedFileStatData->FileType == FCookedFileStatData::ShaderLibrary)
		{
			OutTargetFile.ChunkType = EContainerChunkType::ShaderCodeLibrary;
		}
		else if (CookedFileStatData->FileExt == FCookedFileStatData::EFileExt::UMappedBulk)
		{
			OutTargetFile.ChunkType = EContainerChunkType::MemoryMappedBulkData;
			FString TmpFileName = FString(RelativeFileName.Len() - 8, GetData(RelativeFileName)) + TEXT(".ubulk");
			OutTargetFile.Package = FindOrAddPackage(Arguments, *TmpFileName, Packages, PackageNameMap, PackageIdMap);
		}
		else
		{
			OutTargetFile.Package = FindOrAddPackage(Arguments, *RelativeFileName, Packages, PackageNameMap, PackageIdMap);
			if (CookedFileStatData->FileExt == FCookedFileStatData::UPtnl)
			{
				OutTargetFile.ChunkType = EContainerChunkType::OptionalBulkData;
			}
			else if (CookedFileStatData->FileType == FCookedFileStatData::BulkData)
			{
				OutTargetFile.ChunkType = EContainerChunkType::BulkData;
			}
			else
			{
				OutTargetFile.ChunkType = EContainerChunkType::PackageData;
			}
		}

		switch (OutTargetFile.ChunkType)
		{
		case EContainerChunkType::OptionalBulkData:
			OutTargetFile.ChunkId = CreateIoChunkId(OutTargetFile.Package->GlobalPackageId.Value(), 0, EIoChunkType::OptionalBulkData);
			break;
		case EContainerChunkType::MemoryMappedBulkData:
			OutTargetFile.ChunkId = CreateIoChunkId(OutTargetFile.Package->GlobalPackageId.Value(), 0, EIoChunkType::MemoryMappedBulkData);
			break;
		case EContainerChunkType::BulkData:
			OutTargetFile.ChunkId = CreateIoChunkId(OutTargetFile.Package->GlobalPackageId.Value(), 0, EIoChunkType::BulkData);
			break;
		case EContainerChunkType::PackageData:
			OutTargetFile.Package->FileName = SourceFile.NormalizedPath; // .uasset path
			OutTargetFile.Package->UAssetSize = OriginalCookedFileStatData->FileSize;
			OutTargetFile.Package->UExpSize = CookedFileStatData->FileSize;
			OutTargetFile.ChunkId = CreateIoChunkId(OutTargetFile.Package->GlobalPackageId.Value(), 0, EIoChunkType::ExportBundleData);
			break;
		}

		// Only keep the regions for the file if neither compression nor encryption are enabled, otherwise the regions will be meaningless.
		if (!SourceFile.bNeedsCompression && !SourceFile.bNeedsEncryption)
		{
			OutTargetFile.FileRegions = CookedFileStatData->FileRegions;
		}

		return true;
	};

	auto CreateTargetFileFromPackageStore = [
		&Arguments,
		&ConvertCookedPathToRelativePath,
		&Packages,
		&PackageNameMap,
		&PackageIdMap](const FContainerSourceFile& SourceFile, const FString& RelativeFileName, FContainerTargetFile& OutTargetFile) -> bool
	{
		FCookedPackageStore& PackageStore = *Arguments.PackageStore;
		
		OutTargetFile.NormalizedSourcePath = SourceFile.NormalizedPath;
		
		if (SourceFile.NormalizedPath.EndsWith(TEXT(".ushaderbytecode")))
		{
			int64 FileSize = IFileManager::Get().FileSize(*SourceFile.NormalizedPath);
			if (FileSize == INDEX_NONE)
			{
				UE_LOG(LogIoStore, Warning, TEXT("File not found: '%s'"), *SourceFile.NormalizedPath);
				return false;
			}
			OutTargetFile.ChunkType = EContainerChunkType::ShaderCodeLibrary;
			OutTargetFile.SourceSize = FileSize;
			return true;
		}

		OutTargetFile.ChunkId = PackageStore.GetChunkIdFromFileName(SourceFile.NormalizedPath);
		if (!OutTargetFile.ChunkId.IsValid())
		{
			UE_LOG(LogIoStore, Warning, TEXT("File not found in manifest: '%s'"), *SourceFile.NormalizedPath);
			return false;
		}

		TIoStatusOr<uint64> ChunkSize = PackageStore.GetChunkSize(OutTargetFile.ChunkId);
		if (!ChunkSize.IsOk())
		{
			UE_LOG(LogIoStore, Warning, TEXT("Chunk size not found for: '%s'"), *SourceFile.NormalizedPath);
			return false;
		}
		OutTargetFile.SourceSize = ChunkSize.ValueOrDie();

		FName PackageName = PackageStore.GetPackageNameFromChunkId(OutTargetFile.ChunkId);
		if (!PackageName.IsNone())
		{
			OutTargetFile.Package = FindOrAddPackage(Arguments, PackageName, Packages, PackageNameMap, PackageIdMap);
		}

		if (SourceFile.NormalizedPath.EndsWith(TEXT(".m.ubulk")))
		{
			OutTargetFile.ChunkType = EContainerChunkType::MemoryMappedBulkData;
		}
		else if (SourceFile.NormalizedPath.EndsWith(TEXT(".ubulk")))
		{
			OutTargetFile.ChunkType = EContainerChunkType::BulkData;
		}
		else if (SourceFile.NormalizedPath.EndsWith(TEXT(".uptnl")))
		{
			OutTargetFile.ChunkType = EContainerChunkType::OptionalBulkData;
		}
		else
		{
			check(OutTargetFile.Package);
			OutTargetFile.Package->PackageStoreEntry = PackageStore.GetPackageStoreEntry(OutTargetFile.Package->GlobalPackageId);
			if (!OutTargetFile.Package->PackageStoreEntry)
			{
				UE_LOG(LogIoStore, Warning, TEXT("Failed to find package store entry for package: '%s'"), *PackageName.ToString());
				return false;
			}
			OutTargetFile.ChunkType = EContainerChunkType::PackageData;
			OutTargetFile.Package->FileName = SourceFile.NormalizedPath;
			OutTargetFile.Package->UAssetSize = OutTargetFile.SourceSize;
		}
		
		// Only keep the regions for the file if neither compression nor encryption are enabled, otherwise the regions will be meaningless.
		if (Arguments.bFileRegions && !SourceFile.bNeedsCompression && !SourceFile.bNeedsEncryption)
		{
			FString RegionsFilename = OutTargetFile.ChunkType == EContainerChunkType::PackageData
				? FPaths::ChangeExtension(SourceFile.NormalizedPath, FString(TEXT(".uexp")) + FFileRegion::RegionsFileExtension)
				: SourceFile.NormalizedPath + FFileRegion::RegionsFileExtension;
			TUniquePtr<FArchive> RegionsFile(IFileManager::Get().CreateFileReader(*RegionsFilename));
			if (RegionsFile.IsValid())
			{
				FFileRegion::SerializeFileRegions(*RegionsFile.Get(), OutTargetFile.FileRegions);
			}
		}
		
		return true;
	};
	
	for (const FContainerSourceSpec& ContainerSource : Arguments.Containers)
	{
		FContainerTargetSpec* ContainerTarget = AddContainer(ContainerSource.Name, ContainerTargets);
		ContainerTarget->OutputPath = ContainerSource.OutputPath;
		ContainerTarget->bGenerateDiffPatch = ContainerSource.bGenerateDiffPatch;
		if (Arguments.bSign)
		{
			ContainerTarget->ContainerFlags |= EIoContainerFlags::Signed;
		}

		if (!ContainerTarget->EncryptionKeyGuid.IsValid())
		{
			ContainerTarget->EncryptionKeyGuid = ContainerSource.EncryptionKeyOverrideGuid;
		}

		ContainerTarget->PatchSourceReaders = CreatePatchSourceReaders(ContainerSource.PatchSourceContainerFiles, Arguments);

		{
			IOSTORE_CPU_SCOPE(ProcessSourceFiles);
			for (const FContainerSourceFile& SourceFile : ContainerSource.SourceFiles)
			{
				FString RelativeFileName = ConvertCookedPathToRelativePath(SourceFile.NormalizedPath);

				FContainerTargetFile TargetFile;
				bool bIsValidTargetFile = Arguments.PackageStore.IsValid()
					? CreateTargetFileFromPackageStore(SourceFile, RelativeFileName, TargetFile)
					: CreateTargetFileFromCookedFile(SourceFile, RelativeFileName, TargetFile);

				if (!bIsValidTargetFile)
				{
					continue;
				}

				TargetFile.ContainerTarget			= ContainerTarget;
				TargetFile.TargetPath				= MoveTemp(RelativeFileName);
				TargetFile.DestinationPath			= SourceFile.DestinationPath;
				TargetFile.bForceUncompressed		= !SourceFile.bNeedsCompression;
				
				if (SourceFile.bNeedsCompression)
				{
					ContainerTarget->ContainerFlags |= EIoContainerFlags::Compressed;
				}

				if (SourceFile.bNeedsEncryption)
				{
					ContainerTarget->ContainerFlags |= EIoContainerFlags::Encrypted;
				}

				if (TargetFile.ChunkType == EContainerChunkType::PackageData)
				{
					check(TargetFile.Package);
					ContainerTarget->Packages.Add(TargetFile.Package);
				}

				ContainerTarget->TargetFiles.Emplace(MoveTemp(TargetFile));
			}
		}
	}

	Algo::Sort(Packages, [](const FLegacyCookedPackage* A, const FLegacyCookedPackage* B)
	{
		return A->GlobalPackageId < B->GlobalPackageId;
	});
};

void LogWriterResults(const TArray<FIoStoreWriterResult>& Results)
{
	UE_LOG(LogIoStore, Display, TEXT("--------------------------------------------------- IoDispatcher --------------------------------------------------------"));
	UE_LOG(LogIoStore, Display, TEXT(""));
	UE_LOG(LogIoStore, Display, TEXT("%-30s %10s %15s %15s %15s %25s"),
		TEXT("Container"), TEXT("Flags"), TEXT("TOC Size (KB)"), TEXT("TOC Entries"), TEXT("Size (MB)"), TEXT("Compressed (MB)"));
	UE_LOG(LogIoStore, Display, TEXT("-------------------------------------------------------------------------------------------------------------------------"));
	uint64 TotalTocSize = 0;
	uint64 TotalTocEntryCount = 0;
	uint64 TotalUncompressedContainerSize = 0;
	uint64 TotalPaddingSize = 0;
	for (const FIoStoreWriterResult& Result : Results)
	{
		FString CompressionInfo = TEXT("-");

		if (Result.CompressionMethod != NAME_None)
		{
			double Procentage = (double(Result.UncompressedContainerSize - Result.CompressedContainerSize) / double(Result.UncompressedContainerSize)) * 100.0;
			CompressionInfo = FString::Printf(TEXT("%.2lf (%.2lf%% %s)"),
				(double)Result.CompressedContainerSize / 1024.0 / 1024.0,
				Procentage,
				*Result.CompressionMethod.ToString());
		}

		FString ContainerSettings = FString::Printf(TEXT("%s/%s/%s/%s"),
			EnumHasAnyFlags(Result.ContainerFlags, EIoContainerFlags::Compressed) ? TEXT("C") : TEXT("-"),
			EnumHasAnyFlags(Result.ContainerFlags, EIoContainerFlags::Encrypted) ? TEXT("E") : TEXT("-"),
			EnumHasAnyFlags(Result.ContainerFlags, EIoContainerFlags::Signed) ? TEXT("S") : TEXT("-"),
			EnumHasAnyFlags(Result.ContainerFlags, EIoContainerFlags::Indexed) ? TEXT("I") : TEXT("-"));

		UE_LOG(LogIoStore, Display, TEXT("%-30s %10s %15.2lf %15llu %15.2lf %25s"),
			*Result.ContainerName,
			*ContainerSettings,
			(double)Result.TocSize / 1024.0,
			Result.TocEntryCount,
			(double)Result.UncompressedContainerSize / 1024.0 / 1024.0,
			*CompressionInfo);


		TotalTocSize += Result.TocSize;
		TotalTocEntryCount += Result.TocEntryCount;
		TotalUncompressedContainerSize += Result.UncompressedContainerSize;
		TotalPaddingSize += Result.PaddingSize;
	}

	UE_LOG(LogIoStore, Display, TEXT("%-30s %10s %15.2lf %15llu %15.2lf %25s"),
		TEXT("TOTAL"),
		TEXT(""),
		(double)TotalTocSize / 1024.0,
		TotalTocEntryCount,
		(double)TotalUncompressedContainerSize / 1024.0 / 1024.0,
		TEXT("-"));

	UE_LOG(LogIoStore, Display, TEXT(""));
	UE_LOG(LogIoStore, Display, TEXT("** Flags: (C)ompressed / (E)ncrypted / (S)igned) / (I)ndexed) **"));
	UE_LOG(LogIoStore, Display, TEXT(""));
	UE_LOG(LogIoStore, Display, TEXT("Compression block padding: %8.2lf MB"), (double)TotalPaddingSize / 1024.0 / 1024.0);
	UE_LOG(LogIoStore, Display, TEXT(""));

	UE_LOG(LogIoStore, Display, TEXT("-------------------------------------------- Container Directory Index --------------------------------------------------"));
	UE_LOG(LogIoStore, Display, TEXT("%-30s %15s"), TEXT("Container"), TEXT("Size (KB)"));
	for (const FIoStoreWriterResult& Result : Results)
	{
		UE_LOG(LogIoStore, Display, TEXT("%-30s %15.2lf"), *Result.ContainerName, double(Result.DirectoryIndexSize) / 1024.0);
	}

	UE_LOG(LogIoStore, Display, TEXT(""));
	UE_LOG(LogIoStore, Display, TEXT("---------------------------------------------- Container Patch Report ---------------------------------------------------"));
	UE_LOG(LogIoStore, Display, TEXT("%-30s %16s %16s %16s %16s %16s"), TEXT("Container"), TEXT("Total (count)"), TEXT("Modified (count)"), TEXT("Added (count)"), TEXT("Modified (MB)"), TEXT("Added (MB)"));
	for (const FIoStoreWriterResult& Result : Results)
	{
		UE_LOG(LogIoStore, Display, TEXT("%-30s %16d %16d %16d %16.2lf %16.2lf"), *Result.ContainerName, Result.TocEntryCount, Result.ModifiedChunksCount, Result.AddedChunksCount, Result.ModifiedChunksSize / 1024.0 / 1024.0, Result.AddedChunksSize / 1024.0 / 1024.0);
	}
}

void LogContainerPackageInfo(const TArray<FContainerTargetSpec*>& ContainerTargets)
{
	uint64 TotalStoreSize = 0;
	uint64 TotalPackageCount = 0;
	uint64 TotalLocalizedPackageCount = 0;

	UE_LOG(LogIoStore, Display, TEXT(""));
	UE_LOG(LogIoStore, Display, TEXT(""));
	UE_LOG(LogIoStore, Display, TEXT("--------------------------------------------------- PackageStore (KB) ---------------------------------------------------"));
	UE_LOG(LogIoStore, Display, TEXT(""));
	UE_LOG(LogIoStore, Display, TEXT("%-30s %20s %20s %20s"),
		TEXT("Container"),
		TEXT("Store Size"),
		TEXT("Packages"),
		TEXT("Localized"));
	UE_LOG(LogIoStore, Display, TEXT("-------------------------------------------------------------------------------------------------------------------------"));

	for (const FContainerTargetSpec* ContainerTarget : ContainerTargets)
	{
		uint64 StoreSize = ContainerTarget->Header.StoreEntries.Num();
		uint64 PackageCount = ContainerTarget->Packages.Num();
		uint64 LocalizedPackageCount = 0;

		for (const auto& KV : ContainerTarget->Header.CulturePackageMap)
		{
			LocalizedPackageCount += KV.Value.Num();
		}

		UE_LOG(LogIoStore, Display, TEXT("%-30s %20.0lf %20llu %20llu"),
			*ContainerTarget->Name.ToString(),
			(double)StoreSize / 1024.0,
			PackageCount,
			LocalizedPackageCount);

		TotalStoreSize += StoreSize;
		TotalPackageCount += PackageCount;
		TotalLocalizedPackageCount += LocalizedPackageCount;
	}
	UE_LOG(LogIoStore, Display, TEXT("%-30s %20.0lf %20llu %20llu"),
		TEXT("TOTAL"),
		(double)TotalStoreSize / 1024.0,
		TotalPackageCount,
		TotalLocalizedPackageCount);


	uint64 TotalHeaderSize = 0;
	uint64 TotalGraphSize = 0;
	uint64 TotalExportBundleEntriesSize = 0;
	uint64 TotalImportMapSize = 0;
	uint64 TotalExportMapSize = 0;
	uint64 TotalNameMapSize = 0;

	UE_LOG(LogIoStore, Display, TEXT(""));
	UE_LOG(LogIoStore, Display, TEXT(""));
	UE_LOG(LogIoStore, Display, TEXT("--------------------------------------------------- PackageHeader (KB) --------------------------------------------------"));
	UE_LOG(LogIoStore, Display, TEXT(""));
	UE_LOG(LogIoStore, Display, TEXT("%-30s %13s %13s %13s %13s %13s %13s"),
		TEXT("Container"),
		TEXT("Header"),
		TEXT("Graph"),
		TEXT("ExportBundleEntries"),
		TEXT("ImportMap"),
		TEXT("ExportMap"),
		TEXT("NameMap"));
	UE_LOG(LogIoStore, Display, TEXT("-------------------------------------------------------------------------------------------------------------------------"));
	for (const FContainerTargetSpec* ContainerTarget : ContainerTargets)
	{
		uint64 HeaderSize = 0;
		uint64 GraphSize = 0;
		uint64 ExportBundleEntriesSize = 0;
		uint64 ImportMapSize = 0;
		uint64 ExportMapSize = 0;
		uint64 NameMapSize = 0;

		for (const FLegacyCookedPackage* Package : ContainerTarget->Packages)
		{
			HeaderSize += Package->OptimizedPackage->GetHeaderSize();
			GraphSize += Package->OptimizedPackage->GetGraphDataSize();
			ExportBundleEntriesSize += Package->OptimizedPackage->GetExportBundleEntriesSize();
			ImportMapSize += Package->OptimizedPackage->GetImportMapSize();
			ExportMapSize += Package->OptimizedPackage->GetExportMapSize();
			NameMapSize += Package->OptimizedPackage->GetNameMapSize();
		}

		UE_LOG(LogIoStore, Display, TEXT("%-30s %13.0lf %13.0lf %13.0lf %13.0lf %13.0lf %13.0lf"),
			*ContainerTarget->Name.ToString(),
			(double)HeaderSize / 1024.0,
			(double)GraphSize / 1024.0,
			(double)ExportBundleEntriesSize / 1024.0,
			(double)ImportMapSize / 1024.0,
			(double)ExportMapSize / 1024.0,
			(double)NameMapSize / 1024.0);

		TotalHeaderSize += HeaderSize;
		TotalGraphSize += GraphSize;
		TotalExportBundleEntriesSize += ExportBundleEntriesSize;
		TotalImportMapSize += ImportMapSize;
		TotalExportMapSize += ExportMapSize;
		TotalNameMapSize += NameMapSize;
	}

	UE_LOG(LogIoStore, Display, TEXT("%-30s %13.0lf %13.0lf %13.0lf %13.0lf %13.0lf %13.0lf"),
		TEXT("TOTAL"),
		(double)TotalHeaderSize / 1024.0,
		(double)TotalGraphSize / 1024.0,
		(double)TotalExportBundleEntriesSize / 1024.0,
		(double)TotalImportMapSize / 1024.0,
		(double)TotalExportMapSize / 1024.0,
		(double)TotalNameMapSize / 1024.0);

	UE_LOG(LogIoStore, Display, TEXT(""));
	UE_LOG(LogIoStore, Display, TEXT(""));
}

class FIoStoreWriteRequestManager
{
public:
	FIoStoreWriteRequestManager(FPackageStoreOptimizer& InPackageStoreOptimizer, FCookedPackageStore* InPackageStore)
		: PackageStoreOptimizer(InPackageStoreOptimizer)
		, PackageStore(InPackageStore)
		, MemoryAvailableEvent(FPlatformProcess::GetSynchEventFromPool(false))
	{
		InitiatorThread = Async(EAsyncExecution::Thread, [this]() { InitiatorThreadFunc(); });
		RetirerThread = Async(EAsyncExecution::Thread, [this]() { RetirerThreadFunc(); });
	}

	~FIoStoreWriteRequestManager()
	{
		InitiatorQueue.CompleteAdding();
		RetirerQueue.CompleteAdding();
		InitiatorThread.Wait();
		RetirerThread.Wait();
		FPlatformProcess::ReturnSynchEventToPool(MemoryAvailableEvent);
	}

	IIoStoreWriteRequest* Read(const FContainerTargetFile& InTargetFile)
	{
		if (PackageStore)
		{
			return new FCookedPackageStoreWriteRequest(*this, InTargetFile);
		}
		else
		{
			return new FLooseFileWriteRequest(*this, InTargetFile);
		}
	}

private:
	struct FQueueEntry;
	
	class FWriteContainerTargetFileRequest
		: public IIoStoreWriteRequest
	{
		friend class FIoStoreWriteRequestManager;

	public:
		virtual ~FWriteContainerTargetFileRequest()
		{
		}

		virtual void PrepareSourceBufferAsync(FGraphEventRef InCompletionEvent) override
		{
			CompletionEvent = InCompletionEvent;
			Manager.ScheduleLoad(this);
		}

		virtual uint64 GetOrderHint() override
		{
			return TargetFile.IdealOrder;
		}

		virtual TArrayView<const FFileRegion> GetRegions() override
		{
			return FileRegions;
		}

		virtual const FIoBuffer* GetSourceBuffer() override
		{
			return &SourceBuffer;
		}
		
		virtual void FreeSourceBuffer() override
		{
			SourceBuffer = FIoBuffer();
			Manager.OnBufferMemoryFreed(SourceBufferSize);
		}

		uint64 GetSourceBufferSize() const
		{
			return SourceBufferSize;
		}

		virtual void LoadSourceBufferAsync() = 0;

	protected:
		FWriteContainerTargetFileRequest(FIoStoreWriteRequestManager& InManager,const FContainerTargetFile& InTargetFile)
			: Manager(InManager)
			, TargetFile(InTargetFile)
			, FileRegions(TargetFile.FileRegions)
			, SourceBufferSize(TargetFile.SourceSize) { }

		void OnSourceBufferLoaded()
		{
			Manager.ScheduleRetire(this);
			CompletionEvent->DispatchSubsequents();
		}

		FIoStoreWriteRequestManager& Manager;
		const FContainerTargetFile& TargetFile;
		TArray<FFileRegion> FileRegions;
		uint64 SourceBufferSize;
		FGraphEventRef CompletionEvent;
		FIoBuffer SourceBuffer;
		bool bHasUpdatedExportBundleRegions = false;
		FQueueEntry* QueueEntry = nullptr;
	};

	// Used when staging from cooked files
	class FLooseFileWriteRequest
		: public FWriteContainerTargetFileRequest
	{
	public:
		FLooseFileWriteRequest(FIoStoreWriteRequestManager& InManager,const FContainerTargetFile& InTargetFile)
			: FWriteContainerTargetFileRequest(InManager, InTargetFile) { }

		virtual void LoadSourceBufferAsync() override
		{
			SourceBuffer = FIoBuffer(GetSourceBufferSize());

			QueueEntry->FileHandle.Reset(
				FPlatformFileManager::Get().GetPlatformFile().OpenAsyncRead(*TargetFile.NormalizedSourcePath));
			
			FAsyncFileCallBack Callback = [this](bool, IAsyncReadRequest* ReadRequest)
			{
				if (TargetFile.ChunkType == EContainerChunkType::PackageData)
				{
					SourceBuffer = Manager.PackageStoreOptimizer.CreatePackageBuffer(TargetFile.Package->OptimizedPackage, SourceBuffer, bHasUpdatedExportBundleRegions ? nullptr : &FileRegions);
					bHasUpdatedExportBundleRegions = true;
				}
				OnSourceBufferLoaded();
			};

			QueueEntry->ReadRequest.Reset(
				QueueEntry->FileHandle->ReadRequest(0, SourceBuffer.DataSize(), AIOP_Normal, &Callback, SourceBuffer.Data()));
		}
	};

	// Used when cooking directly to I/O store container file
	class FCookedPackageStoreWriteRequest
		: public FWriteContainerTargetFileRequest
	{
	public:
		FCookedPackageStoreWriteRequest(FIoStoreWriteRequestManager& InManager,const FContainerTargetFile& InTargetFile)
			: FWriteContainerTargetFileRequest(InManager, InTargetFile) {}

		virtual void LoadSourceBufferAsync() override
		{
			Manager.PackageStore->ReadChunkAsync(
				TargetFile.ChunkId,
				[this](TIoStatusOr<FIoBuffer> Status)
				{
					SourceBuffer = Status.ConsumeValueOrDie();
					if (TargetFile.ChunkType == EContainerChunkType::PackageData)
					{
						check(TargetFile.Package->UAssetSize > 0);
						const uint64 HeaderSize = TargetFile.Package->UAssetSize;
						FIoBuffer ExportsBuffer(SourceBuffer.Data() + HeaderSize, SourceBuffer.DataSize() - HeaderSize, SourceBuffer);
						SourceBuffer = Manager.PackageStoreOptimizer.CreatePackageBuffer(TargetFile.Package->OptimizedPackage, ExportsBuffer, bHasUpdatedExportBundleRegions ? nullptr : &FileRegions);
						bHasUpdatedExportBundleRegions = true;
					}
					OnSourceBufferLoaded();
				});
		}
	};

	struct FQueueEntry
	{
		FQueueEntry* Next = nullptr;
		TUniquePtr<IAsyncReadFileHandle> FileHandle;
		TUniquePtr<IAsyncReadRequest> ReadRequest;
		FWriteContainerTargetFileRequest* WriteRequest = nullptr;
	};

	class FQueue
	{
	public:
		FQueue()
			: Event(FPlatformProcess::GetSynchEventFromPool(false))
		{ }

		~FQueue()
		{
			check(Head == nullptr && Tail == nullptr);
			FPlatformProcess::ReturnSynchEventToPool(Event);
		}

		void Enqueue(FQueueEntry* Entry)
		{
			check(!bIsDoneAdding);
			{
				FScopeLock _(&CriticalSection);

				if (!Tail)
				{
					Head = Tail = Entry;
				}
				else
				{
					Tail->Next = Entry;
					Tail = Entry;
				}
				Entry->Next = nullptr;
			}

			Event->Trigger();
		}

		FQueueEntry* DequeueOrWait()
		{
			for (;;)
			{
				{
					FScopeLock _(&CriticalSection);
					if (Head)
					{
						FQueueEntry* Entry = Head;
						Head = Tail = nullptr;
						return Entry;
					}
				}

				if (bIsDoneAdding)
				{
					break;
				}

				Event->Wait();
			}

			return nullptr;
		}

		void CompleteAdding()
		{
			bIsDoneAdding = true;
			Event->Trigger();
		}

	private:
		FCriticalSection CriticalSection;
		FEvent* Event = nullptr;
		FQueueEntry* Head = nullptr;
		FQueueEntry* Tail = nullptr;
		TAtomic<bool> bIsDoneAdding{ false };
	};

	void ScheduleLoad(FWriteContainerTargetFileRequest* WriteRequest)
	{
		FQueueEntry* QueueEntry = new FQueueEntry();
		QueueEntry->WriteRequest = WriteRequest;
		WriteRequest->QueueEntry = QueueEntry;
		InitiatorQueue.Enqueue(QueueEntry);
	}

	void ScheduleRetire(FWriteContainerTargetFileRequest* WriteRequest)
	{
		RetirerQueue.Enqueue(WriteRequest->QueueEntry);
	}

	void Start(FQueueEntry* QueueEntry)
	{
		const uint64 SourceBufferSize = QueueEntry->WriteRequest->GetSourceBufferSize();

		uint64 LocalUsedBufferMemory = UsedBufferMemory.Load();
		while (LocalUsedBufferMemory > 0 && LocalUsedBufferMemory + SourceBufferSize > BufferMemoryLimit)
		{
			TRACE_CPUPROFILER_EVENT_SCOPE(WaitForBufferMemory);
			MemoryAvailableEvent->Wait();
			LocalUsedBufferMemory = UsedBufferMemory.Load();
		}

		UsedBufferMemory.AddExchange(SourceBufferSize);
		QueueEntry->WriteRequest->LoadSourceBufferAsync();
	}

	void Retire(FQueueEntry* QueueEntry)
	{
		if (QueueEntry->ReadRequest.IsValid())
		{
			QueueEntry->ReadRequest->WaitCompletion();
			QueueEntry->ReadRequest.Reset();
			QueueEntry->FileHandle.Reset();
		}
		delete QueueEntry;
	}

	void OnBufferMemoryFreed(uint64 Count)
	{
		uint64 OldValue = UsedBufferMemory.SubExchange(Count);
		check(OldValue >= Count);
		MemoryAvailableEvent->Trigger();
	}

	void InitiatorThreadFunc()
	{
		for (;;)
		{
			FQueueEntry* QueueEntry = InitiatorQueue.DequeueOrWait();
			if (!QueueEntry)
			{
				return;
			}
			while (QueueEntry)
			{
				FQueueEntry* Next = QueueEntry->Next;
				Start(QueueEntry);
				QueueEntry = Next;
			}
		}
	}

	void RetirerThreadFunc()
	{
		for (;;)
		{
			FQueueEntry* QueueEntry = RetirerQueue.DequeueOrWait();
			if (!QueueEntry)
			{
				return;
			}
			while (QueueEntry)
			{
				FQueueEntry* Next = QueueEntry->Next;
				Retire(QueueEntry);
				QueueEntry = Next;
			}
		}
	}

	FPackageStoreOptimizer& PackageStoreOptimizer;
	FCookedPackageStore* PackageStore;
	TFuture<void> InitiatorThread;
	TFuture<void> RetirerThread;
	FQueue InitiatorQueue;
	FQueue RetirerQueue;
	TAtomic<uint64> UsedBufferMemory { 0 };
	FEvent* MemoryAvailableEvent;

	static constexpr uint64 BufferMemoryLimit = 2ull << 30;
};

int32 CreateTarget(const FIoStoreArguments& Arguments, const FIoStoreWriterSettings& GeneralIoWriterSettings)
{
	TGuardValue<int32> GuardAllowUnversionedContentInEditor(GAllowUnversionedContentInEditor, 1);

#if OUTPUT_CHUNKID_DIRECTORY
	ChunkIdCsv.CreateOutputFile(CookedDir);
#endif

	TArray<FLegacyCookedPackage*> Packages;
	FPackageNameMap PackageNameMap;
	FPackageIdMap PackageIdMap;

	FPackageStoreOptimizer PackageStoreOptimizer;
	PackageStoreOptimizer.Initialize(Arguments.TargetPlatform);
	FIoStoreWriteRequestManager WriteRequestManager(PackageStoreOptimizer, Arguments.PackageStore.Get());

	TArray<FContainerTargetSpec*> ContainerTargets;
	UE_LOG(LogIoStore, Display, TEXT("Creating container targets..."));
	{
		IOSTORE_CPU_SCOPE(CreateContainerTargets);
		InitializeContainerTargetsAndPackages(Arguments, Packages, PackageNameMap, PackageIdMap, ContainerTargets, PackageStoreOptimizer);
	}

	TUniquePtr<FIoStoreWriterContext> IoStoreWriterContext(new FIoStoreWriterContext());
	TArray<FIoStoreWriter*> IoStoreWriters;
	FIoStoreWriter* GlobalIoStoreWriter = nullptr;
	{
		IOSTORE_CPU_SCOPE(InitializeIoStoreWriters);
		if (!Arguments.IsDLC())
		{
			GlobalIoStoreWriter = new FIoStoreWriter(*Arguments.GlobalContainerPath);
			IoStoreWriters.Add(GlobalIoStoreWriter);
		}
		for (FContainerTargetSpec* ContainerTarget : ContainerTargets)
		{
			check(ContainerTarget->ContainerId.IsValid());
			if (!ContainerTarget->OutputPath.IsEmpty())
			{
				ContainerTarget->IoStoreWriter = new FIoStoreWriter(*ContainerTarget->OutputPath);
				IoStoreWriters.Add(ContainerTarget->IoStoreWriter);
			}
		}
		FIoStatus IoStatus = IoStoreWriterContext->Initialize(GeneralIoWriterSettings);
		check(IoStatus.IsOk());

		FIoContainerSettings GlobalContainerSettings;
		if (Arguments.bSign)
		{
			GlobalContainerSettings.SigningKey = Arguments.KeyChain.SigningKey;
			GlobalContainerSettings.ContainerFlags |= EIoContainerFlags::Signed;
		}
		if (GlobalIoStoreWriter)
		{
			IoStatus = GlobalIoStoreWriter->Initialize(*IoStoreWriterContext, GlobalContainerSettings);
		}
		check(IoStatus.IsOk());
		for (FContainerTargetSpec* ContainerTarget : ContainerTargets)
		{
			if (ContainerTarget->IoStoreWriter)
			{
				FIoContainerSettings ContainerSettings;
				ContainerSettings.ContainerId = ContainerTarget->ContainerId;
				if (Arguments.bCreateDirectoryIndex)
				{
					ContainerSettings.ContainerFlags = ContainerTarget->ContainerFlags | EIoContainerFlags::Indexed;
				}
				if (EnumHasAnyFlags(ContainerTarget->ContainerFlags, EIoContainerFlags::Encrypted))
				{
					const FNamedAESKey* Key = Arguments.KeyChain.EncryptionKeys.Find(ContainerTarget->EncryptionKeyGuid);
					check(Key);
					ContainerSettings.EncryptionKeyGuid = ContainerTarget->EncryptionKeyGuid;
					ContainerSettings.EncryptionKey = Key->Key;
				}
				if (EnumHasAnyFlags(ContainerTarget->ContainerFlags, EIoContainerFlags::Signed))
				{
					ContainerSettings.SigningKey = Arguments.KeyChain.SigningKey;
					ContainerSettings.ContainerFlags |= EIoContainerFlags::Signed;
				}
				ContainerSettings.bGenerateDiffPatch = ContainerTarget->bGenerateDiffPatch;
				IoStatus = ContainerTarget->IoStoreWriter->Initialize(*IoStoreWriterContext, ContainerSettings);
				check(IoStatus.IsOk());
				ContainerTarget->IoStoreWriter->EnableDiskLayoutOrdering(ContainerTarget->PatchSourceReaders);
			}
		}
	}

	if (Arguments.PackageStore.IsValid())
	{
		ParsePackageAssetsFromPackageStore(*Arguments.PackageStore, Packages, PackageStoreOptimizer);
	}
	else
	{
		ParsePackageAssetsFromFiles(Packages, PackageStoreOptimizer);
	}

	UE_LOG(LogIoStore, Display, TEXT("Processing shader libraries..."));
	ProcessShaderLibraries(Arguments, ContainerTargets);

	if (Arguments.IsDLC() && Arguments.bRemapPluginContentToGame)
	{
		for (FLegacyCookedPackage* Package : Packages)
		{
			const int32 DLCNameLen = Arguments.DLCName.Len() + 1;
			FString PackageNameStr = Package->PackageName.ToString();
			FString RedirectedPackageNameStr = TEXT("/Game");
			RedirectedPackageNameStr.AppendChars(*PackageNameStr + DLCNameLen, PackageNameStr.Len() - DLCNameLen);
			FName RedirectedPackageName = FName(*RedirectedPackageNameStr);
			if (Arguments.ReleasedPackages.PackageNames.Contains(RedirectedPackageName))
			{
				Package->OptimizedPackage->RedirectFrom(RedirectedPackageName);
			}
			else
			{
				// TODO: Should NAME_None be the default source package name instead?
				// RemapLocalizationPathIfNeeded sets it to the original
				// package name when not remapping plugin content
				Package->OptimizedPackage->RedirectFrom(NAME_None);
			}
		}
	}

	for (FContainerTargetSpec* ContainerTarget : ContainerTargets)
	{
		if (ContainerTarget->IoStoreWriter)
		{
			for (FContainerTargetFile& TargetFile : ContainerTarget->TargetFiles)
			{
				if (TargetFile.ChunkType != EContainerChunkType::PackageData)
				{
					FIoWriteOptions WriteOptions;
					WriteOptions.DebugName = *TargetFile.TargetPath;
					WriteOptions.bForceUncompressed = TargetFile.bForceUncompressed;
					WriteOptions.bIsMemoryMapped = TargetFile.ChunkType == EContainerChunkType::MemoryMappedBulkData;
					WriteOptions.FileName = TargetFile.DestinationPath;
					if (TargetFile.SourceBuffer.IsSet())
					{
						ContainerTarget->IoStoreWriter->Append(TargetFile.ChunkId, *TargetFile.SourceBuffer, WriteOptions, TargetFile.IdealOrder);
					}
					else
					{
						ContainerTarget->IoStoreWriter->Append(TargetFile.ChunkId, WriteRequestManager.Read(TargetFile), WriteOptions);
					}
				}
			}
		}
	}

	TMap<FPackageId, FPackageStorePackage*> OptimizedPackagesMap;
	for (FLegacyCookedPackage* Package : Packages)
	{
		check(Package->OptimizedPackage);
		OptimizedPackagesMap.Add(Package->OptimizedPackage->GetId(), Package->OptimizedPackage);
	}

	if (Arguments.PackageStore.IsValid())
	{
		UE_LOG(LogIoStore, Display, TEXT("Processing imports.."));
		PackageStoreOptimizer.FindImports(OptimizedPackagesMap);
	}

	UE_LOG(LogIoStore, Display, TEXT("Processing redirects..."));
	PackageStoreOptimizer.ProcessRedirects(OptimizedPackagesMap);

	UE_LOG(LogIoStore, Display, TEXT("Optimizing packages..."));
	PackageStoreOptimizer.OptimizeExportBundles(OptimizedPackagesMap);

	UE_LOG(LogIoStore, Display, TEXT("Finalizing packages..."));
	for (FLegacyCookedPackage* Package : Packages)
	{
		check(Package->OptimizedPackage);
		PackageStoreOptimizer.FinalizePackage(Package->OptimizedPackage);
	}

	UE_LOG(LogIoStore, Display, TEXT("Creating disk layout..."));
	CreateDiskLayout(ContainerTargets, Packages, Arguments.OrderMaps, PackageIdMap, Arguments.bClusterByOrderFilePriority);

	for (FContainerTargetSpec* ContainerTarget : ContainerTargets)
	{
		if (ContainerTarget->IoStoreWriter)
		{
			TArray<FPackageStoreEntryResource> PackageStoreEntries;
			for (FContainerTargetFile& TargetFile : ContainerTarget->TargetFiles)
			{
				if (TargetFile.ChunkType == EContainerChunkType::PackageData)
				{
					check(TargetFile.Package);
					FIoWriteOptions WriteOptions;
					WriteOptions.DebugName = *TargetFile.TargetPath;
					WriteOptions.bForceUncompressed = TargetFile.bForceUncompressed;
					WriteOptions.FileName = TargetFile.DestinationPath;
					if (TargetFile.SourceBuffer.IsSet())
					{
						ContainerTarget->IoStoreWriter->Append(TargetFile.ChunkId, *TargetFile.SourceBuffer, WriteOptions, TargetFile.IdealOrder);
					}
					else
					{
						ContainerTarget->IoStoreWriter->Append(TargetFile.ChunkId, WriteRequestManager.Read(TargetFile), WriteOptions);
					}
					PackageStoreEntries.Add(PackageStoreOptimizer.CreatePackageStoreEntry(TargetFile.Package->OptimizedPackage));
				}
			}

			ContainerTarget->Header = PackageStoreOptimizer.CreateContainerHeader(ContainerTarget->ContainerId, PackageStoreEntries);
			FLargeMemoryWriter HeaderAr(0, true);
			HeaderAr << ContainerTarget->Header;
			int64 DataSize = HeaderAr.TotalSize();
			FIoBuffer ContainerHeaderBuffer(FIoBuffer::AssumeOwnership, HeaderAr.ReleaseOwnership(), DataSize);

			FIoWriteOptions WriteOptions;
			WriteOptions.DebugName = TEXT("ContainerHeader");
			ContainerTarget->IoStoreWriter->Append(
				CreateIoChunkId(ContainerTarget->ContainerId.Value(), 0, EIoChunkType::ContainerHeader),
				ContainerHeaderBuffer,
				WriteOptions);
		}

		// Check if we need to dump the final order of the packages. Useful, to debug packing.
		if (FParse::Param(FCommandLine::Get(), TEXT("writefinalorder")))
		{
			FString FinalContainerOrderFile = FPaths::GetPath(ContainerTarget->OutputPath) + FPaths::GetBaseFilename(ContainerTarget->OutputPath) + TEXT("-order.txt");
			TUniquePtr<FArchive> IoOrderListArchive(IFileManager::Get().CreateFileWriter(*FinalContainerOrderFile));
			if (IoOrderListArchive)
			{
				IoOrderListArchive->SetIsTextFormat(true);

				for (const FContainerTargetFile& TargetFile : ContainerTarget->TargetFiles)
				{
					if (TargetFile.Package)
					{
						FString Line = FString::Printf(TEXT("%s"), *TargetFile.Package->FileName);
						IoOrderListArchive->Logf(TEXT("%s"), *Line);
					}
				}

				IoOrderListArchive->Close();
			}
		}

	}

	uint64 InitialLoadSize = 0;
	if (GlobalIoStoreWriter)
	{
		FIoBuffer ScriptObjectsBuffer = PackageStoreOptimizer.CreateScriptObjectsBuffer();
		InitialLoadSize = ScriptObjectsBuffer.DataSize();
		FIoWriteOptions WriteOptions;
		WriteOptions.DebugName = TEXT("ScriptObjects");
		GlobalIoStoreWriter->Append(CreateIoChunkId(0, 0, EIoChunkType::ScriptObjects), ScriptObjectsBuffer, WriteOptions);
	}

	UE_LOG(LogIoStore, Display, TEXT("Serializing container(s)..."));

	TArray<FIoStoreWriterResult> IoStoreWriterResults;
	IoStoreWriterResults.Reserve(IoStoreWriters.Num());
	for (FIoStoreWriter* IoStoreWriter : IoStoreWriters)
	{
		TFuture<FIoStoreWriterResult> FlushTask = Async(EAsyncExecution::Thread, [IoStoreWriter]()
		{
			return IoStoreWriter->Flush().ConsumeValueOrDie();
		});

		while (!FlushTask.IsReady())
		{
			FlushTask.WaitFor(FTimespan::FromSeconds(2.0));
			FIoStoreWriterContext::FProgress Progress = IoStoreWriterContext->GetProgress();
			UE_LOG(LogIoStore, Display, TEXT("Hashed, Compressed, Serialized: %lld, %lld, %lld / %lld"), Progress.HashedChunksCount, Progress.CompressedChunksCount, Progress.SerializedChunksCount, Progress.TotalChunksCount);
		}
		IoStoreWriterResults.Emplace(FlushTask.Get());
		delete IoStoreWriter;
	}
	IoStoreWriters.Empty();

	IOSTORE_CPU_SCOPE(CalculateStats);

	UE_LOG(LogIoStore, Display, TEXT("Calculating stats..."));
	uint64 UExpSize = 0;
	uint64 UAssetSize = 0;
	uint64 ImportedPackagesCount = 0;
	uint64 NoImportedPackagesCount = 0;
	uint64 NameMapCount = 0;

	for (const FLegacyCookedPackage* Package : Packages)
	{
		UExpSize += Package->UExpSize;
		UAssetSize += Package->UAssetSize;
		NameMapCount += Package->OptimizedPackage->GetNameCount();
		int32 PackageImportedPackagesCount = Package->OptimizedPackage->GetImportedPackageIds().Num();
		ImportedPackagesCount += PackageImportedPackagesCount;
		NoImportedPackagesCount += PackageImportedPackagesCount == 0;
	}

	uint64 GlobalShaderSize = 0;
	uint64 SharedShaderSize = 0;
	uint64 UniqueShaderSize = 0;
	for (const FContainerTargetSpec* ContainerTarget : ContainerTargets)
	{
		for (const FShaderInfo& ShaderInfo : ContainerTarget->Shaders)
		{
			if (ShaderInfo.ReferencedByPackagesCount == 0)
			{
				GlobalShaderSize += ShaderInfo.Size;
			}
			else if (ShaderInfo.ReferencedByPackagesCount == 1)
			{
				UniqueShaderSize += ShaderInfo.Size;
			}
			else
			{
				SharedShaderSize += ShaderInfo.Size;
			}
		}
	}

	LogWriterResults(IoStoreWriterResults);
	LogContainerPackageInfo(ContainerTargets);
	
	UE_LOG(LogIoStore, Display, TEXT("Input:  %8.2lf MB UExp"), (double)UExpSize / 1024.0 / 1024.0);
	UE_LOG(LogIoStore, Display, TEXT("Input:  %8.2lf MB UAsset"), (double)UAssetSize / 1024.0 / 1024.0);
	UE_LOG(LogIoStore, Display, TEXT("Input:  %8d Packages"), Packages.Num());
	UE_LOG(LogIoStore, Display, TEXT("Input:  %8.2f MB Global shaders"), (double)GlobalShaderSize / 1024.0 / 1024.0);
	UE_LOG(LogIoStore, Display, TEXT("Input:  %8.2f MB Shared shaders"), (double)SharedShaderSize / 1024.0 / 1024.0);
	UE_LOG(LogIoStore, Display, TEXT("Input:  %8.2f MB Unique shaders"), (double)UniqueShaderSize / 1024.0 / 1024.0);
	UE_LOG(LogIoStore, Display, TEXT(""));
	UE_LOG(LogIoStore, Display, TEXT("Output: %8llu Export bundle entries"), PackageStoreOptimizer.GetTotalExportBundleEntryCount());
	UE_LOG(LogIoStore, Display, TEXT("Output: %8llu Export bundles"), PackageStoreOptimizer.GetTotalExportBundleCount());
	UE_LOG(LogIoStore, Display, TEXT("Output: %8llu Internal export bundle arcs"), PackageStoreOptimizer.GetTotalInternalBundleArcsCount());
	UE_LOG(LogIoStore, Display, TEXT("Output: %8llu External export bundle arcs"), PackageStoreOptimizer.GetTotalExternalBundleArcsCount());
	UE_LOG(LogIoStore, Display, TEXT("Output: %8llu Name map entries"), NameMapCount);
	UE_LOG(LogIoStore, Display, TEXT("Output: %8llu Imported package entries"), ImportedPackagesCount);
	UE_LOG(LogIoStore, Display, TEXT("Output: %8llu Packages without imports"), NoImportedPackagesCount);
	UE_LOG(LogIoStore, Display, TEXT("Output: %8d Public runtime script objects"), PackageStoreOptimizer.GetTotalScriptObjectCount());
	UE_LOG(LogIoStore, Display, TEXT("Output: %8.2lf MB InitialLoadData"), (double)InitialLoadSize / 1024.0 / 1024.0);

	return 0;
}

int32 CreateContentPatch(const FIoStoreArguments& Arguments, const FIoStoreWriterSettings& GeneralIoWriterSettings)
{
	UE_LOG(LogIoStore, Display, TEXT("Building patch..."));
	TUniquePtr<FIoStoreWriterContext> IoStoreWriterContext(new FIoStoreWriterContext());
	FIoStatus IoStatus = IoStoreWriterContext->Initialize(GeneralIoWriterSettings);
	check(IoStatus.IsOk());
	TArray<FIoStoreWriterResult> Results;
	for (const FContainerSourceSpec& Container : Arguments.Containers)
	{
		TArray<TUniquePtr<FIoStoreReader>> SourceReaders = CreatePatchSourceReaders(Container.PatchSourceContainerFiles, Arguments);
		TUniquePtr<FIoStoreReader> TargetReader = CreateIoStoreReader(*Container.PatchTargetFile, Arguments.KeyChain);
		if (!TargetReader.IsValid())
		{
			UE_LOG(LogIoStore, Error, TEXT("Failed loading target container"));
			return -1;
		}

		FIoStoreWriter IoStoreWriter(*Container.OutputPath);
		
		EIoContainerFlags TargetContainerFlags = TargetReader->GetContainerFlags();

		FIoContainerSettings ContainerSettings;
		if (Arguments.bCreateDirectoryIndex)
		{
			ContainerSettings.ContainerFlags |= EIoContainerFlags::Indexed;
		}

		ContainerSettings.ContainerId = TargetReader->GetContainerId();
		if (Arguments.bSign || EnumHasAnyFlags(TargetContainerFlags, EIoContainerFlags::Signed))
		{
			ContainerSettings.SigningKey = Arguments.KeyChain.SigningKey;
			ContainerSettings.ContainerFlags |= EIoContainerFlags::Signed;
		}

		if (EnumHasAnyFlags(TargetContainerFlags, EIoContainerFlags::Encrypted))
		{
			ContainerSettings.ContainerFlags |= EIoContainerFlags::Encrypted;
			const FNamedAESKey* Key = Arguments.KeyChain.EncryptionKeys.Find(TargetReader->GetEncryptionKeyGuid());
			if (!Key)
			{
				UE_LOG(LogIoStore, Error, TEXT("Missing encryption key for target container"));
				return -1;
			}
			ContainerSettings.EncryptionKeyGuid = Key->Guid;
			ContainerSettings.EncryptionKey = Key->Key;
		}

		IoStatus = IoStoreWriter.Initialize(*IoStoreWriterContext, ContainerSettings);
		check(IoStatus.IsOk());

		TMap<FIoChunkId, FIoChunkHash> SourceHashByChunkId;
		for (const TUniquePtr<FIoStoreReader>& SourceReader : SourceReaders)
		{
			SourceReader->EnumerateChunks([&SourceHashByChunkId](const FIoStoreTocChunkInfo& ChunkInfo)
			{
				SourceHashByChunkId.Add(ChunkInfo.Id, ChunkInfo.Hash);
				return true;
			});
		}

		TMap<FIoChunkId, FString> ChunkFileNamesMap;
		TargetReader->GetDirectoryIndexReader().IterateDirectoryIndex(FIoDirectoryIndexHandle::RootDirectory(), TEXT(""),
		[&ChunkFileNamesMap, &TargetReader](FString Filename, uint32 TocEntryIndex) -> bool
		{
			TIoStatusOr<FIoStoreTocChunkInfo> ChunkInfo = TargetReader->GetChunkInfo(TocEntryIndex);
			if (ChunkInfo.IsOk())
			{
				ChunkFileNamesMap.Add(ChunkInfo.ValueOrDie().Id, Filename);
			}
			return true;
		});

		TargetReader->EnumerateChunks([&TargetReader, &SourceHashByChunkId, &IoStoreWriter, &ChunkFileNamesMap](const FIoStoreTocChunkInfo& ChunkInfo)
		{
			FIoChunkHash* FindSourceHash = SourceHashByChunkId.Find(ChunkInfo.Id);
			if (!FindSourceHash || *FindSourceHash != ChunkInfo.Hash)
			{
				FIoReadOptions ReadOptions;
				TIoStatusOr<FIoBuffer> ChunkBuffer = TargetReader->Read(ChunkInfo.Id, ReadOptions);
				FIoWriteOptions WriteOptions;
				FString* FindFileName = ChunkFileNamesMap.Find(ChunkInfo.Id);
				if (FindFileName)
				{
					WriteOptions.FileName = *FindFileName;
					if (FindSourceHash)
					{
						UE_LOG(LogIoStore, Display, TEXT("Modified: %s"), **FindFileName);
					}
					else
					{
						UE_LOG(LogIoStore, Display, TEXT("Added: %s"), **FindFileName);
					}
				}
				WriteOptions.bIsMemoryMapped = ChunkInfo.bIsMemoryMapped;
				WriteOptions.bForceUncompressed = ChunkInfo.bForceUncompressed; 
				IoStoreWriter.Append(ChunkInfo.Id, ChunkBuffer.ConsumeValueOrDie(), WriteOptions);
			}
			return true;
		});


		Results.Emplace(IoStoreWriter.Flush().ConsumeValueOrDie());
	}

	LogWriterResults(Results);

	return 0;
}

using DirectoryIndexVisitorFunction = TFunctionRef<bool(FString, const uint32)>;

bool IterateDirectoryIndex(FIoDirectoryIndexHandle Directory, const FString& Path, const FIoDirectoryIndexReader& Reader, DirectoryIndexVisitorFunction Visit)
{
	FIoDirectoryIndexHandle File = Reader.GetFile(Directory);
	while (File.IsValid())
	{
		const uint32 TocEntryIndex = Reader.GetFileData(File);
		FStringView FileName = Reader.GetFileName(File);
		FString FilePath = Reader.GetMountPoint() / Path / FString(FileName);

		if (!Visit(MoveTemp(FilePath), TocEntryIndex))
		{
			return false;
		}

		File = Reader.GetNextFile(File);
	}

	FIoDirectoryIndexHandle ChildDirectory = Reader.GetChildDirectory(Directory);
	while (ChildDirectory.IsValid())
	{
		FStringView DirectoryName = Reader.GetDirectoryName(ChildDirectory);
		FString ChildDirectoryPath = Path / FString(DirectoryName);

		if (!IterateDirectoryIndex(ChildDirectory, ChildDirectoryPath, Reader, Visit))
		{
			return false;
		}

		ChildDirectory = Reader.GetNextDirectory(ChildDirectory);
	}

	return true;
}

int32 ListContainer(
	const FIoStoreArguments& Arguments,
	const FString& ContainerPathOrWildcard,
	const FString& CsvPath)
{
	TArray<FString> ContainerFilePaths;

	if (IFileManager::Get().FileExists(*ContainerPathOrWildcard))
	{
		ContainerFilePaths.Add(ContainerPathOrWildcard);
	}
	else
	{
		FString Directory = FPaths::GetPath(ContainerPathOrWildcard);
		FPaths::NormalizeDirectoryName(Directory);

		TArray<FString> FoundContainerFiles;
		IFileManager::Get().FindFiles(FoundContainerFiles, *ContainerPathOrWildcard, true, false);

		for (const FString& Filename : FoundContainerFiles)
		{
			ContainerFilePaths.Emplace(Directory / Filename);
		}
	}

	if (ContainerFilePaths.Num() == 0)
	{
		UE_LOG(LogIoStore, Error, TEXT("Container '%s' doesn't exist and no container matches wildcard."), *ContainerPathOrWildcard);
		return -1;
	}

	TArray<FString> CsvLines;
	
	CsvLines.Add(TEXT("PackageId, PackageName, Filename, ContainerName, Offset, Size, CompressedSize, Hash"));

	for (const FString& ContainerFilePath : ContainerFilePaths)
	{
		TUniquePtr<FIoStoreReader> Reader = CreateIoStoreReader(*ContainerFilePath, Arguments.KeyChain);
		if (!Reader.IsValid())
		{
			UE_LOG(LogIoStore, Warning, TEXT("Failed to read container '%s'"), *ContainerFilePath);
			continue;
		}

		if (!EnumHasAnyFlags(Reader->GetContainerFlags(), EIoContainerFlags::Indexed))
		{
			UE_LOG(LogIoStore, Warning, TEXT("Missing directory index for container '%s'"), *ContainerFilePath);
		}

		UE_LOG(LogIoStore, Display, TEXT("Listing container '%s'"), *ContainerFilePath);

		FString ContainerName = FPaths::GetBaseFilename(ContainerFilePath);
		const FIoDirectoryIndexReader& IndexReader = Reader->GetDirectoryIndexReader();
		TMap<FIoChunkId, FString> ChunkFileNamesMap;
		IterateDirectoryIndex(
			FIoDirectoryIndexHandle::RootDirectory(),
			TEXT(""),
			IndexReader,
			[&ChunkFileNamesMap, &Reader](FString Filename, uint32 TocEntryIndex) -> bool
		{
			TIoStatusOr<FIoStoreTocChunkInfo> ChunkInfo = Reader->GetChunkInfo(TocEntryIndex);
			if (ChunkInfo.IsOk())
			{
				ChunkFileNamesMap.Add(ChunkInfo.ValueOrDie().Id, Filename);
			}
			return true;
		});

		Reader->EnumerateChunks([&CsvLines, &ChunkFileNamesMap, &ContainerName](const FIoStoreTocChunkInfo& ChunkInfo)
		{
			FString PackageName;
			FString* FindFileName = ChunkFileNamesMap.Find(ChunkInfo.Id);
			if (FindFileName)
			{
				FPackageName::TryConvertFilenameToLongPackageName(*FindFileName, PackageName, nullptr);
			}
			FPackageId PackageId = PackageName.Len() > 0
				? FPackageId::FromName(FName(*PackageName))
				: FPackageId();

			CsvLines.Emplace(
				FString::Printf(TEXT("0x%llX, %s, %s, %s, %lld, %lld, %lld, 0x%s"),
					PackageId.ValueForDebugging(),
					*PackageName,
					FindFileName ? **FindFileName : TEXT(""),
					*ContainerName,
					ChunkInfo.Offset,
					ChunkInfo.Size,
					ChunkInfo.CompressedSize,
					*ChunkInfo.Hash.ToString()));

			return true;
		});
	}

	if (CsvLines.Num())
	{
		UE_LOG(LogIoStore, Display, TEXT("Saving '%d' file entries to '%s'"), CsvLines.Num(), *CsvPath);
		FFileHelper::SaveStringArrayToFile(CsvLines, *CsvPath);
	}
	else
	{
		UE_LOG(LogIoStore, Warning, TEXT("No file entries to save from '%s'"), *ContainerPathOrWildcard);
	}

	return 0;
}

int32 Describe(
	const FString& GlobalContainerPath,
	const FKeyChain& KeyChain,
	const FString& PackageFilter,
	const FString& OutPath,
	bool bIncludeExportHashes)
{
	struct FPackageDesc;

	struct FPackageRedirect
	{
		FName Culture;
		FPackageDesc* Source = nullptr;
		FPackageDesc* Target = nullptr;
	};

	struct FContainerDesc
	{
		FName Name;
		FIoContainerId Id;
		FGuid EncryptionKeyGuid;
		TArray<FPackageRedirect> PackageRedirects;
		bool bCompressed;
		bool bSigned;
		bool bEncrypted;
		bool bIndexed;
	};

	struct FPackageLocation
	{
		FContainerDesc* Container = nullptr;
		uint64 Offset = -1;
	};

	struct FExportDesc
	{
		FPackageDesc* Package = nullptr;
		FName Name;
		FName FullName;
		FPackageObjectIndex OuterIndex;
		FPackageObjectIndex ClassIndex;
		FPackageObjectIndex SuperIndex;
		FPackageObjectIndex TemplateIndex;
		FPackageObjectIndex GlobalImportIndex;
		uint64 SerialOffset = 0;
		uint64 SerialSize = 0;
		FSHAHash Hash;
	};

	struct FExportBundleEntryDesc
	{
		FExportBundleEntry::EExportCommandType CommandType = FExportBundleEntry::ExportCommandType_Count;
		int32 LocalExportIndex = -1;
		FExportDesc* Export = nullptr;
	};

	struct FImportDesc
	{
		FName Name;
		FPackageObjectIndex GlobalImportIndex;
		FExportDesc* Export = nullptr;
	};

	struct FScriptObjectDesc
	{
		FName Name;
		FName FullName;
		FPackageObjectIndex GlobalImportIndex;
		FPackageObjectIndex OuterIndex;
	};

	struct FPackageDesc
	{
		FPackageId PackageId;
		FName PackageName;
		uint64 Size = 0;
		uint32 LoadOrder = uint32(-1);
		uint32 PackageFlags = 0;
		int32 NameCount = -1;
		int32 ExportBundleCount = -1;
		TArray<FPackageLocation, TInlineAllocator<1>> Locations;
		TArray<FImportDesc> Imports;
		TArray<FExportDesc> Exports;
		TArray<TArray<FExportBundleEntryDesc>, TInlineAllocator<1>> ExportBundles;
	};

	if (!IFileManager::Get().FileExists(*GlobalContainerPath))
	{
		UE_LOG(LogIoStore, Error, TEXT("Global container '%s' doesn't exist."), *GlobalContainerPath);
		return -1;
	}

	TUniquePtr<FIoStoreReader> GlobalReader = CreateIoStoreReader(*GlobalContainerPath, KeyChain);
	if (!GlobalReader.IsValid())
	{
		UE_LOG(LogIoStore, Warning, TEXT("Failed reading global container '%s'"), *GlobalContainerPath);
		return -1;
	}

	UE_LOG(LogIoStore, Display, TEXT("Loading script imports..."));

	TIoStatusOr<FIoBuffer> ScriptObjectsBuffer = GlobalReader->Read(CreateIoChunkId(0, 0, EIoChunkType::ScriptObjects), FIoReadOptions());
	if (!ScriptObjectsBuffer.IsOk())
	{
		UE_LOG(LogIoStore, Warning, TEXT("Failed reading initial load meta chunk from global container '%s'"), *GlobalContainerPath);
		return -1;
	}

	TMap<FPackageObjectIndex, FScriptObjectDesc> ScriptObjectByGlobalIdMap;
	FLargeMemoryReader ScriptObjectsArchive(ScriptObjectsBuffer.ValueOrDie().Data(), ScriptObjectsBuffer.ValueOrDie().DataSize());
	TArray<FNameEntryId> GlobalNameMap = LoadNameBatch(ScriptObjectsArchive);
	int32 NumScriptObjects = 0;
	ScriptObjectsArchive << NumScriptObjects;
	const FScriptObjectEntry* ScriptObjectEntries = reinterpret_cast<const FScriptObjectEntry*>(ScriptObjectsBuffer.ValueOrDie().Data() + ScriptObjectsArchive.Tell());
	for (int32 ScriptObjectIndex = 0; ScriptObjectIndex < NumScriptObjects; ++ScriptObjectIndex)
	{
		const FScriptObjectEntry& ScriptObjectEntry = ScriptObjectEntries[ScriptObjectIndex];
		const FMappedName& MappedName = FMappedName::FromMinimalName(ScriptObjectEntry.ObjectName);
		check(MappedName.IsGlobal());
		FScriptObjectDesc& ScriptObjectDesc = ScriptObjectByGlobalIdMap.Add(ScriptObjectEntry.GlobalIndex);
		ScriptObjectDesc.Name = FName::CreateFromDisplayId(GlobalNameMap[MappedName.GetIndex()], MappedName.GetNumber());
		ScriptObjectDesc.GlobalImportIndex = ScriptObjectEntry.GlobalIndex;
		ScriptObjectDesc.OuterIndex = ScriptObjectEntry.OuterIndex;
	}
	for (auto& KV : ScriptObjectByGlobalIdMap)
	{
		FScriptObjectDesc& ScriptObjectDesc = KV.Get<1>();
		if (ScriptObjectDesc.FullName.IsNone())
		{
			TArray<FScriptObjectDesc*> ScriptObjectStack;
			FScriptObjectDesc* Current = &ScriptObjectDesc;
			FString FullName;
			while (Current)
			{
				if (!Current->FullName.IsNone())
				{
					FullName = Current->FullName.ToString();
					break;
				}
				ScriptObjectStack.Push(Current);
				Current = ScriptObjectByGlobalIdMap.Find(Current->OuterIndex);
			}
			while (ScriptObjectStack.Num() > 0)
			{
				Current = ScriptObjectStack.Pop();
				FullName /= Current->Name.ToString();
				Current->FullName = FName(FullName);
			}
		}
	}

	FString Directory = FPaths::GetPath(GlobalContainerPath);
	FPaths::NormalizeDirectoryName(Directory);

	TArray<FString> FoundContainerFiles;
	IFileManager::Get().FindFiles(FoundContainerFiles, *(Directory / TEXT("*.utoc")), true, false);
	TArray<FString> ContainerFilePaths;
	for (const FString& Filename : FoundContainerFiles)
	{
		ContainerFilePaths.Emplace(Directory / Filename);
	}

	UE_LOG(LogIoStore, Display, TEXT("Loading containers..."));

	TArray<TUniquePtr<FIoStoreReader>> Readers;

	struct FLoadContainerHeaderJob
	{
		FName ContainerName;
		FContainerDesc* ContainerDesc = nullptr;
		TArray<FPackageDesc*> Packages;
		FIoStoreReader* Reader = nullptr;
		FCulturePackageMap RawCulturePackageMap;
		TArray<TPair<FPackageId, FPackageId>> RawPackageRedirects;
	};

	TArray<FLoadContainerHeaderJob> LoadContainerHeaderJobs;

	for (const FString& ContainerFilePath : ContainerFilePaths)
	{
		TUniquePtr<FIoStoreReader> Reader = CreateIoStoreReader(*ContainerFilePath, KeyChain);
		if (!Reader.IsValid())
		{
			UE_LOG(LogIoStore, Warning, TEXT("Failed to read container '%s'"), *ContainerFilePath);
			continue;
		}

		FLoadContainerHeaderJob& LoadContainerHeaderJob = LoadContainerHeaderJobs.AddDefaulted_GetRef();
		LoadContainerHeaderJob.Reader = Reader.Get();
		LoadContainerHeaderJob.ContainerName = FName(FPaths::GetBaseFilename(ContainerFilePath));
		
		Readers.Emplace(MoveTemp(Reader));
	}
	
	TAtomic<int32> TotalPackageCount{ 0 };
	ParallelFor(LoadContainerHeaderJobs.Num(), [&LoadContainerHeaderJobs, &TotalPackageCount](int32 Index)
	{
		TRACE_CPUPROFILER_EVENT_SCOPE(LoadContainerHeader);

		FLoadContainerHeaderJob& Job = LoadContainerHeaderJobs[Index];

		FContainerDesc* ContainerDesc = new FContainerDesc();
		ContainerDesc->Name = Job.ContainerName;
		ContainerDesc->Id = Job.Reader->GetContainerId();
		ContainerDesc->EncryptionKeyGuid = Job.Reader->GetEncryptionKeyGuid();
		EIoContainerFlags Flags = Job.Reader->GetContainerFlags();
		ContainerDesc->bCompressed = bool(Flags & EIoContainerFlags::Compressed);
		ContainerDesc->bEncrypted = bool(Flags & EIoContainerFlags::Encrypted);
		ContainerDesc->bSigned = bool(Flags & EIoContainerFlags::Signed);
		ContainerDesc->bIndexed = bool(Flags & EIoContainerFlags::Indexed);
		Job.ContainerDesc = ContainerDesc;

		TIoStatusOr<FIoBuffer> IoBuffer = Job.Reader->Read(CreateIoChunkId(Job.Reader->GetContainerId().Value(), 0, EIoChunkType::ContainerHeader), FIoReadOptions());
		if (IoBuffer.IsOk())
		{
			FMemoryReaderView Ar(MakeArrayView(IoBuffer.ValueOrDie().Data(), IoBuffer.ValueOrDie().DataSize()));
			FContainerHeader ContainerHeader;
			Ar << ContainerHeader;

			Job.RawCulturePackageMap = ContainerHeader.CulturePackageMap;
			Job.RawPackageRedirects = ContainerHeader.PackageRedirects;

			TArrayView<FFilePackageStoreEntry> StoreEntries(reinterpret_cast<FFilePackageStoreEntry*>(ContainerHeader.StoreEntries.GetData()), ContainerHeader.PackageCount);

			int32 PackageIndex = 0;
			Job.Packages.Reserve(StoreEntries.Num());
			for (FFilePackageStoreEntry& ContainerEntry : StoreEntries)
			{
				const FPackageId& PackageId = ContainerHeader.PackageIds[PackageIndex++];
				FPackageDesc* PackageDesc = new FPackageDesc();
				PackageDesc->PackageId = PackageId;
				PackageDesc->Size = ContainerEntry.ExportBundlesSize;
				PackageDesc->Exports.SetNum(ContainerEntry.ExportCount);
				PackageDesc->ExportBundleCount = ContainerEntry.ExportBundleCount;
				PackageDesc->LoadOrder = ContainerEntry.LoadOrder;
				Job.Packages.Add(PackageDesc);
				++TotalPackageCount;
			}
		}
	}, EParallelForFlags::Unbalanced);

	struct FLoadPackageSummaryJob
	{
		FPackageDesc* PackageDesc = nullptr;
		FIoChunkId ChunkId;
		TArray<FLoadContainerHeaderJob*, TInlineAllocator<1>> Containers;
	};

	TArray<FLoadPackageSummaryJob> LoadPackageSummaryJobs;

	TArray<FContainerDesc*> Containers;
	TArray<FPackageDesc*> Packages;
	TMap<FPackageId, FPackageDesc*> PackageByIdMap;
	TMap<FPackageId, FLoadPackageSummaryJob*> PackageJobByIdMap;
	Containers.Reserve(LoadContainerHeaderJobs.Num());
	Packages.Reserve(TotalPackageCount);
	PackageByIdMap.Reserve(TotalPackageCount);
	PackageJobByIdMap.Reserve(TotalPackageCount);
	LoadPackageSummaryJobs.Reserve(TotalPackageCount);
	for (FLoadContainerHeaderJob& LoadContainerHeaderJob : LoadContainerHeaderJobs)
	{
		Containers.Add(LoadContainerHeaderJob.ContainerDesc);
		for (FPackageDesc* PackageDesc : LoadContainerHeaderJob.Packages)
		{
			FLoadPackageSummaryJob*& UniquePackageJob = PackageJobByIdMap.FindOrAdd(PackageDesc->PackageId);
			if (!UniquePackageJob)
			{
				Packages.Add(PackageDesc);
				PackageByIdMap.Add(PackageDesc->PackageId, PackageDesc);
				FLoadPackageSummaryJob& LoadPackageSummaryJob = LoadPackageSummaryJobs.AddDefaulted_GetRef();
				LoadPackageSummaryJob.PackageDesc = PackageDesc;
				LoadPackageSummaryJob.ChunkId = CreateIoChunkId(PackageDesc->PackageId.Value(), 0, EIoChunkType::ExportBundleData);
				UniquePackageJob = &LoadPackageSummaryJob;
			}
			UniquePackageJob->Containers.Add(&LoadContainerHeaderJob);
		}
	}
	for (FLoadContainerHeaderJob& LoadContainerHeaderJob : LoadContainerHeaderJobs)
	{
		for (const auto& RedirectPair : LoadContainerHeaderJob.RawPackageRedirects)
		{
			FPackageRedirect& PackageRedirect = LoadContainerHeaderJob.ContainerDesc->PackageRedirects.AddDefaulted_GetRef();
			PackageRedirect.Source = PackageByIdMap.FindRef(RedirectPair.Get<0>());
			PackageRedirect.Target = PackageByIdMap.FindRef(RedirectPair.Get<1>());
		}
		for (const auto& CultureRedirectsPair : LoadContainerHeaderJob.RawCulturePackageMap)
		{
			FName Culture(CultureRedirectsPair.Get<0>());
			for (const auto& RedirectPair : CultureRedirectsPair.Get<1>())
			{
				FPackageRedirect& PackageRedirect = LoadContainerHeaderJob.ContainerDesc->PackageRedirects.AddDefaulted_GetRef();
				PackageRedirect.Source = PackageByIdMap.FindRef(RedirectPair.Get<0>());
				PackageRedirect.Target = PackageByIdMap.FindRef(RedirectPair.Get<1>());
				PackageRedirect.Culture = Culture;
			}
		}
	}
	
	ParallelFor(LoadPackageSummaryJobs.Num(), [&LoadPackageSummaryJobs, bIncludeExportHashes](int32 Index)
	{
		TRACE_CPUPROFILER_EVENT_SCOPE(LoadPackageSummary);

		FLoadPackageSummaryJob& Job = LoadPackageSummaryJobs[Index];
		for (FLoadContainerHeaderJob* LoadContainerHeaderJob : Job.Containers)
		{
			TIoStatusOr<FIoStoreTocChunkInfo> ChunkInfo = LoadContainerHeaderJob->Reader->GetChunkInfo(Job.ChunkId);
			check(ChunkInfo.IsOk());
			FPackageLocation& Location = Job.PackageDesc->Locations.AddDefaulted_GetRef();
			Location.Container = LoadContainerHeaderJob->ContainerDesc;
			Location.Offset = ChunkInfo.ValueOrDie().Offset;
		}

		FIoStoreReader* Reader = Job.Containers[0]->Reader;
		FIoReadOptions ReadOptions;
		if (!bIncludeExportHashes)
		{
			ReadOptions.SetRange(0, 16 << 10);
		}
		TIoStatusOr<FIoBuffer> IoBuffer = Reader->Read(Job.ChunkId, ReadOptions);
		check(IoBuffer.IsOk());
		const uint8* PackageSummaryData = IoBuffer.ValueOrDie().Data();
		const FPackageSummary* PackageSummary = reinterpret_cast<const FPackageSummary*>(PackageSummaryData);
		if (PackageSummary->HeaderSize > IoBuffer.ValueOrDie().DataSize())
		{
			ReadOptions.SetRange(0, PackageSummary->HeaderSize);
			IoBuffer = Reader->Read(Job.ChunkId, ReadOptions);
			PackageSummaryData = IoBuffer.ValueOrDie().Data();
			PackageSummary = reinterpret_cast<const FPackageSummary*>(PackageSummaryData);
		}

		TArray<FNameEntryId> PackageNameMap;
		{
			TRACE_CPUPROFILER_EVENT_SCOPE(LoadNameBatch);
			TArrayView<const uint8> NameMapView(PackageSummaryData + sizeof(FPackageSummary), PackageSummary->HeaderSize - sizeof(FPackageSummary));
			FMemoryReaderView NameMapReader(NameMapView);
			PackageNameMap = LoadNameBatch(NameMapReader);
		}

		Job.PackageDesc->PackageName = FName::CreateFromDisplayId(PackageNameMap[PackageSummary->Name.GetIndex()], PackageSummary->Name.GetNumber());
		Job.PackageDesc->PackageFlags = PackageSummary->PackageFlags;
		Job.PackageDesc->NameCount = PackageNameMap.Num();

		const FPackageObjectIndex* ImportMap = reinterpret_cast<const FPackageObjectIndex*>(PackageSummaryData + PackageSummary->ImportMapOffset);
		Job.PackageDesc->Imports.SetNum((PackageSummary->ExportMapOffset - PackageSummary->ImportMapOffset) / sizeof(FPackageObjectIndex));
		for (int32 ImportIndex = 0; ImportIndex < Job.PackageDesc->Imports.Num(); ++ImportIndex)
		{
			FImportDesc& ImportDesc = Job.PackageDesc->Imports[ImportIndex];
			ImportDesc.GlobalImportIndex = ImportMap[ImportIndex];
		}

		const FExportMapEntry* ExportMap = reinterpret_cast<const FExportMapEntry*>(PackageSummaryData + PackageSummary->ExportMapOffset);
		for (int32 ExportIndex = 0; ExportIndex < Job.PackageDesc->Exports.Num(); ++ExportIndex)
		{
			const FExportMapEntry& ExportMapEntry = ExportMap[ExportIndex];
			FExportDesc& ExportDesc = Job.PackageDesc->Exports[ExportIndex];
			ExportDesc.Package = Job.PackageDesc;
			ExportDesc.Name = FName::CreateFromDisplayId(PackageNameMap[ExportMapEntry.ObjectName.GetIndex()], ExportMapEntry.ObjectName.GetNumber());
			ExportDesc.OuterIndex = ExportMapEntry.OuterIndex;
			ExportDesc.ClassIndex = ExportMapEntry.ClassIndex;
			ExportDesc.SuperIndex = ExportMapEntry.SuperIndex;
			ExportDesc.TemplateIndex = ExportMapEntry.TemplateIndex;
			ExportDesc.GlobalImportIndex = ExportMapEntry.GlobalImportIndex;
			ExportDesc.SerialSize = ExportMapEntry.CookedSerialSize;
		}

		const FExportBundleHeader* ExportBundleHeaders = reinterpret_cast<const FExportBundleHeader*>(PackageSummaryData + PackageSummary->GraphDataOffset);
		const FExportBundleEntry* ExportBundleEntries = reinterpret_cast<const FExportBundleEntry*>(PackageSummaryData + PackageSummary->ExportBundleEntriesOffset);
		uint64 CurrentExportOffset = PackageSummary->HeaderSize;
		for (int32 ExportBundleIndex = 0; ExportBundleIndex < Job.PackageDesc->ExportBundleCount; ++ExportBundleIndex)
		{
			TArray<FExportBundleEntryDesc>& ExportBundleDesc = Job.PackageDesc->ExportBundles.AddDefaulted_GetRef();
			const FExportBundleHeader* ExportBundle = ExportBundleHeaders + ExportBundleIndex;
			const FExportBundleEntry* BundleEntry = ExportBundleEntries + ExportBundle->FirstEntryIndex;
			const FExportBundleEntry* BundleEntryEnd = BundleEntry + ExportBundle->EntryCount;
			check(BundleEntry <= BundleEntryEnd);
			while (BundleEntry < BundleEntryEnd)
			{
				FExportBundleEntryDesc& EntryDesc = ExportBundleDesc.AddDefaulted_GetRef();
				EntryDesc.CommandType = FExportBundleEntry::EExportCommandType(BundleEntry->CommandType);
				EntryDesc.LocalExportIndex = BundleEntry->LocalExportIndex;
				EntryDesc.Export = &Job.PackageDesc->Exports[BundleEntry->LocalExportIndex];
				if (BundleEntry->CommandType == FExportBundleEntry::ExportCommandType_Serialize)
				{
					EntryDesc.Export->SerialOffset = CurrentExportOffset;
					CurrentExportOffset += EntryDesc.Export->SerialSize;

					if (bIncludeExportHashes)
					{
						check(EntryDesc.Export->SerialOffset + EntryDesc.Export->SerialSize <= IoBuffer.ValueOrDie().DataSize());
						FSHA1::HashBuffer(IoBuffer.ValueOrDie().Data() + EntryDesc.Export->SerialOffset, EntryDesc.Export->SerialSize, EntryDesc.Export->Hash.Hash);
					}
				}
				++BundleEntry;
			}
		}
	}, EParallelForFlags::Unbalanced);

	UE_LOG(LogIoStore, Display, TEXT("Connecting imports and exports..."));
	TMap<FPackageObjectIndex, FExportDesc*> ExportByGlobalIdMap;
	{
		TRACE_CPUPROFILER_EVENT_SCOPE(ConnectImportsAndExports);

		for (FPackageDesc* PackageDesc : Packages)
		{
			for (FExportDesc& ExportDesc : PackageDesc->Exports)
			{
				if (!ExportDesc.GlobalImportIndex.IsNull())
				{
					ExportByGlobalIdMap.Add(ExportDesc.GlobalImportIndex, &ExportDesc);
				}
			}
		}

		ParallelFor(Packages.Num(), [&Packages](int32 Index)
		{
			FPackageDesc* PackageDesc = Packages[Index];
			for (FExportDesc& ExportDesc : PackageDesc->Exports)
			{
				if (ExportDesc.FullName.IsNone())
				{
					TRACE_CPUPROFILER_EVENT_SCOPE(GenerateExportFullName);

					TArray<FExportDesc*> ExportStack;
					
					FExportDesc* Current = &ExportDesc;
					TStringBuilder<2048> FullNameBuilder;
					TCHAR NameBuffer[FName::StringBufferSize];
					for (;;)
					{
						if (!Current->FullName.IsNone())
						{
							Current->FullName.ToString(NameBuffer);
							FullNameBuilder.Append(NameBuffer);
							break;
						}
						ExportStack.Push(Current);
						if (Current->OuterIndex.IsNull())
						{
							PackageDesc->PackageName.ToString(NameBuffer);
							FullNameBuilder.Append(NameBuffer);
							break;
						}
						Current = &PackageDesc->Exports[Current->OuterIndex.Value()];
					}
					while (ExportStack.Num() > 0)
					{
						Current = ExportStack.Pop(false);
						FullNameBuilder.Append(TEXT("/"));
						Current->Name.ToString(NameBuffer);
						FullNameBuilder.Append(NameBuffer);
						Current->FullName = FName(FullNameBuilder);
					}
				}
			}
		}, EParallelForFlags::Unbalanced);

		for (FPackageDesc* PackageDesc : Packages)
		{
			for (FImportDesc& Import : PackageDesc->Imports)
			{
				if (!Import.GlobalImportIndex.IsNull())
				{
					if (Import.GlobalImportIndex.IsPackageImport())
					{
						Import.Export = ExportByGlobalIdMap.FindRef(Import.GlobalImportIndex);
						if (!Import.Export)
						{
							UE_LOG(LogIoStore, Warning, TEXT("Missing import: 0x%llX in package 0x%llX '%s'"), Import.GlobalImportIndex.Value(), PackageDesc->PackageId.ValueForDebugging(), *PackageDesc->PackageName.ToString());
						}
						else
						{
							Import.Name = Import.Export->FullName;
						}
					}
					else
					{
						FScriptObjectDesc* ScriptObjectDesc = ScriptObjectByGlobalIdMap.Find(Import.GlobalImportIndex);
						check(ScriptObjectDesc);
						Import.Name = ScriptObjectDesc->FullName;
					}
				}
			}
		}
	}

	UE_LOG(LogIoStore, Display, TEXT("Collecting output packages..."));
	TArray<const FPackageDesc*> OutputPackages;
	{
		TRACE_CPUPROFILER_EVENT_SCOPE(CollectOutputPackages);

		if (PackageFilter.IsEmpty())
		{
			OutputPackages.Append(Packages);
		}
		else
		{
			TArray<FString> SplitPackageFilters;
			const TCHAR* Delimiters[] = { TEXT(","), TEXT(" ") };
			PackageFilter.ParseIntoArray(SplitPackageFilters, Delimiters, UE_ARRAY_COUNT(Delimiters), true);

			TArray<FString> PackageNameFilter;
			TSet<FPackageId> PackageIdFilter;
			for (const FString& PackageNameOrId : SplitPackageFilters)
			{
				if (PackageNameOrId.Len() > 0 && FChar::IsDigit(PackageNameOrId[0]))
				{
					uint64 Value;
					LexFromString(Value, *PackageNameOrId);
					PackageIdFilter.Add(*(FPackageId*)(&Value));
				}
				else
				{
					PackageNameFilter.Add(PackageNameOrId);
				}
			}

			TArray<const FPackageDesc*> PackageStack;
			for (const FPackageDesc* PackageDesc : Packages)
			{
				bool bInclude = false;
				if (PackageIdFilter.Contains(PackageDesc->PackageId))
				{
					bInclude = true;
				}
				else
				{
					FString PackageName = PackageDesc->PackageName.ToString();
					for (const FString& Wildcard : PackageNameFilter)
					{
						if (PackageName.MatchesWildcard(Wildcard))
						{
							bInclude = true;
							break;
						}
					}
				}
				if (bInclude)
				{
					PackageStack.Push(PackageDesc);
				}
			}
			TSet<const FPackageDesc*> Visited;
			while (PackageStack.Num() > 0)
			{
				const FPackageDesc* PackageDesc = PackageStack.Pop();
				if (!Visited.Contains(PackageDesc))
				{
					Visited.Add(PackageDesc);
					OutputPackages.Add(PackageDesc);
					for (const FImportDesc& Import : PackageDesc->Imports)
					{
						if (Import.Export && Import.Export->Package)
						{
							PackageStack.Push(Import.Export->Package);
						}
					}
				}
			}
		}
		Algo::Sort(OutputPackages, [](const FPackageDesc* A, const FPackageDesc* B)
			{
				return A->LoadOrder < B->LoadOrder;
			});
	}

	UE_LOG(LogIoStore, Display, TEXT("Generating report..."));

	FOutputDevice* OutputOverride = GWarn;
	FString OutputFilename;
	TUniquePtr<FOutputDeviceFile> OutputBuffer;
	if (!OutPath.IsEmpty())
	{
		OutputBuffer = MakeUnique<FOutputDeviceFile>(*OutPath, true);
		OutputBuffer->SetSuppressEventTag(true);
		OutputOverride = OutputBuffer.Get();
	}
	{
		TRACE_CPUPROFILER_EVENT_SCOPE(GenerateReport);
		TGuardValue<ELogTimes::Type> GuardPrintLogTimes(GPrintLogTimes, ELogTimes::None);
		TGuardValue<bool> GuardPrintLogCategory(GPrintLogCategory, false);
		TGuardValue<bool> GuardPrintLogVerbosity(GPrintLogVerbosity, false);

		auto PackageObjectIndexToString = [&ScriptObjectByGlobalIdMap, &ExportByGlobalIdMap](const FPackageObjectIndex& PackageObjectIndex, bool bIncludeName) -> FString
		{
			if (PackageObjectIndex.IsNull())
			{
				return TEXT("<null>");
			}
			else if (PackageObjectIndex.IsPackageImport())
			{
				FExportDesc* ExportDesc = ExportByGlobalIdMap.FindRef(PackageObjectIndex);
				if (ExportDesc && bIncludeName)
				{
					return FString::Printf(TEXT("0x%llX '%s'"), PackageObjectIndex.Value(), *ExportDesc->FullName.ToString());
				}
				else
				{
					return FString::Printf(TEXT("0x%llX"), PackageObjectIndex.Value());
				}
			}
			else if (PackageObjectIndex.IsScriptImport())
			{
				FScriptObjectDesc* ScriptObjectDesc = ScriptObjectByGlobalIdMap.Find(PackageObjectIndex);
				if (ScriptObjectDesc && bIncludeName)
				{
					return FString::Printf(TEXT("0x%llX '%s'"), PackageObjectIndex.Value(), *ScriptObjectDesc->FullName.ToString());
				}
				else
				{
					return FString::Printf(TEXT("0x%llX"), PackageObjectIndex.Value());
				}
			}
			else if (PackageObjectIndex.IsExport())
			{
				return FString::Printf(TEXT("%d"), PackageObjectIndex.Value());
			}
			else
			{
				return FString::Printf(TEXT("0x%llX"), PackageObjectIndex.Value());
			}
		};

		for (const FContainerDesc* ContainerDesc : Containers)
		{
			OutputOverride->Logf(ELogVerbosity::Display, TEXT("********************************************"));
			OutputOverride->Logf(ELogVerbosity::Display, TEXT("Container '%s' Summary"), *ContainerDesc->Name.ToString());
			OutputOverride->Logf(ELogVerbosity::Display, TEXT("--------------------------------------------"));

			OutputOverride->Logf(ELogVerbosity::Display, TEXT("\t\t      ContainerId: 0x%llX"), ContainerDesc->Id.Value());
			OutputOverride->Logf(ELogVerbosity::Display, TEXT("\t\t       Compressed: %s"), ContainerDesc->bCompressed ? TEXT("Yes") : TEXT("No"));
			OutputOverride->Logf(ELogVerbosity::Display, TEXT("\t\t           Signed: %s"), ContainerDesc->bSigned ? TEXT("Yes") : TEXT("No"));
			OutputOverride->Logf(ELogVerbosity::Display, TEXT("\t\t          Indexed: %s"), ContainerDesc->bIndexed ? TEXT("Yes") : TEXT("No"));
			if (ContainerDesc->bEncrypted)
			{
				OutputOverride->Logf(ELogVerbosity::Display, TEXT("\t\tEncryptionKeyGuid: %s"), *ContainerDesc->EncryptionKeyGuid.ToString());
			}

			if (ContainerDesc->PackageRedirects.Num())
			{
				OutputOverride->Logf(ELogVerbosity::Display, TEXT("--------------------------------------------"));
				OutputOverride->Logf(ELogVerbosity::Display, TEXT("Package Redirects"));
				OutputOverride->Logf(ELogVerbosity::Display, TEXT("=========="));
				for (const FPackageRedirect& Redirect : ContainerDesc->PackageRedirects)
				{
					OutputOverride->Logf(ELogVerbosity::Display, TEXT("\t*************************"));
					if (!Redirect.Culture.IsNone())
					{
						OutputOverride->Logf(ELogVerbosity::Display, TEXT("\t\t          Culture: %s"), *Redirect.Culture.ToString());
					}
					OutputOverride->Logf(ELogVerbosity::Display, TEXT("\t\t           Source: 0x%llX '%s'"), Redirect.Source->PackageId.ValueForDebugging(), *Redirect.Source->PackageName.ToString());
					OutputOverride->Logf(ELogVerbosity::Display, TEXT("\t\t           Target: 0x%llX '%s'"), Redirect.Target->PackageId.ValueForDebugging(), *Redirect.Target->PackageName.ToString());
				}
			}
		}

		for (const FPackageDesc* PackageDesc : OutputPackages)
		{
			OutputOverride->Logf(ELogVerbosity::Display, TEXT("********************************************"));
			OutputOverride->Logf(ELogVerbosity::Display, TEXT("Package '%s' Summary"), *PackageDesc->PackageName.ToString());
			OutputOverride->Logf(ELogVerbosity::Display, TEXT("--------------------------------------------"));

			OutputOverride->Logf(ELogVerbosity::Display, TEXT("\t\t        PackageId: 0x%llX"), PackageDesc->PackageId.ValueForDebugging());
			OutputOverride->Logf(ELogVerbosity::Display, TEXT("\t\t             Size: %lld"), PackageDesc->Size);
			OutputOverride->Logf(ELogVerbosity::Display, TEXT("\t\t        LoadOrder: %d"), PackageDesc->LoadOrder);
			OutputOverride->Logf(ELogVerbosity::Display, TEXT("\t\t     PackageFlags: %X"), PackageDesc->PackageFlags);
			OutputOverride->Logf(ELogVerbosity::Display, TEXT("\t\t        NameCount: %d"), PackageDesc->NameCount);
			OutputOverride->Logf(ELogVerbosity::Display, TEXT("\t\t      ImportCount: %d"), PackageDesc->Imports.Num());
			OutputOverride->Logf(ELogVerbosity::Display, TEXT("\t\t      ExportCount: %d"), PackageDesc->Exports.Num());
			OutputOverride->Logf(ELogVerbosity::Display, TEXT("\t\tExportBundleCount: %d"), PackageDesc->ExportBundleCount);

			OutputOverride->Logf(ELogVerbosity::Display, TEXT("--------------------------------------------"));
			OutputOverride->Logf(ELogVerbosity::Display, TEXT("Locations"));
			OutputOverride->Logf(ELogVerbosity::Display, TEXT("=========="));
			int32 Index = 0;
			for (const FPackageLocation& Location : PackageDesc->Locations)
			{
				OutputOverride->Logf(ELogVerbosity::Display, TEXT("\t*************************"));
				OutputOverride->Logf(ELogVerbosity::Display, TEXT("\tLocation %d: '%s'"), Index++, *Location.Container->Name.ToString());
				OutputOverride->Logf(ELogVerbosity::Display, TEXT("\t\t           Offset: %lld"), Location.Offset);
			}

			OutputOverride->Logf(ELogVerbosity::Display, TEXT("--------------------------------------------"));
			OutputOverride->Logf(ELogVerbosity::Display, TEXT("Imports"));
			OutputOverride->Logf(ELogVerbosity::Display, TEXT("=========="));
			Index = 0;
			for (const FImportDesc& Import : PackageDesc->Imports)
			{
				OutputOverride->Logf(ELogVerbosity::Display, TEXT("\t*************************"));
				OutputOverride->Logf(ELogVerbosity::Display, TEXT("\tImport %d: '%s'"), Index++, *Import.Name.ToString());
				OutputOverride->Logf(ELogVerbosity::Display, TEXT("\t\tGlobalImportIndex: %s"), *PackageObjectIndexToString(Import.GlobalImportIndex, false));
			}

			OutputOverride->Logf(ELogVerbosity::Display, TEXT("--------------------------------------------"));
			OutputOverride->Logf(ELogVerbosity::Display, TEXT("Exports"));
			OutputOverride->Logf(ELogVerbosity::Display, TEXT("=========="));
			Index = 0;
			for (const FExportDesc& Export : PackageDesc->Exports)
			{
				OutputOverride->Logf(ELogVerbosity::Display, TEXT("\t*************************"));
				OutputOverride->Logf(ELogVerbosity::Display, TEXT("\tExport %d: '%s'"), Index++, *Export.Name.ToString());
				OutputOverride->Logf(ELogVerbosity::Display, TEXT("\t\t       OuterIndex: %s"), *PackageObjectIndexToString(Export.OuterIndex, true));
				OutputOverride->Logf(ELogVerbosity::Display, TEXT("\t\t       ClassIndex: %s"), *PackageObjectIndexToString(Export.ClassIndex, true));
				OutputOverride->Logf(ELogVerbosity::Display, TEXT("\t\t       SuperIndex: %s"), *PackageObjectIndexToString(Export.SuperIndex, true));
				OutputOverride->Logf(ELogVerbosity::Display, TEXT("\t\t    TemplateIndex: %s"), *PackageObjectIndexToString(Export.TemplateIndex, true));
				OutputOverride->Logf(ELogVerbosity::Display, TEXT("\t\tGlobalImportIndex: %s"), *PackageObjectIndexToString(Export.GlobalImportIndex, false));
				OutputOverride->Logf(ELogVerbosity::Display, TEXT("\t\t           Offset: %lld"), Export.SerialOffset);
				OutputOverride->Logf(ELogVerbosity::Display, TEXT("\t\t             Size: %lld"), Export.SerialSize);
				if (bIncludeExportHashes)
				{
					OutputOverride->Logf(ELogVerbosity::Display, TEXT("\t\t             Hash: %s"), *Export.Hash.ToString());
				}
			}

			OutputOverride->Logf(ELogVerbosity::Display, TEXT("--------------------------------------------"));
			OutputOverride->Logf(ELogVerbosity::Display, TEXT("Export Bundles"));
			OutputOverride->Logf(ELogVerbosity::Display, TEXT("=========="));
			Index = 0;
			for (const TArray<FExportBundleEntryDesc>& ExportBundle : PackageDesc->ExportBundles)
			{
				OutputOverride->Logf(ELogVerbosity::Display, TEXT("\t*************************"));
				OutputOverride->Logf(ELogVerbosity::Display, TEXT("\tExport Bundle %d"), Index++);
				for (const FExportBundleEntryDesc& ExportBundleEntry : ExportBundle)
				{
					if (ExportBundleEntry.CommandType == FExportBundleEntry::ExportCommandType_Create)
					{
						OutputOverride->Logf(ELogVerbosity::Display, TEXT("\t\t           Create: %d '%s'"), ExportBundleEntry.LocalExportIndex, *ExportBundleEntry.Export->Name.ToString());
					}
					else
					{
						OutputOverride->Logf(ELogVerbosity::Display, TEXT("\t\t        Serialize: %d '%s'"), ExportBundleEntry.LocalExportIndex, *ExportBundleEntry.Export->Name.ToString());
					}
				}
			}
		}
	}

	for (FPackageDesc* PackageDesc : Packages)
	{
		delete PackageDesc;
	}
	for (FContainerDesc* ContainerDesc : Containers)
	{
		delete ContainerDesc;
	}

	return 0;
}

static int32 Diff(
	const FString& SourcePath,
	const FKeyChain& SourceKeyChain,
	const FString& TargetPath,
	const FKeyChain& TargetKeyChain,
	const FString& OutPath)
{
	struct FContainerChunkInfo
	{
		FString ContainerName;
		TMap<FIoChunkId, FIoStoreTocChunkInfo> ChunkInfoById;
		int64 UncompressedContainerSize = 0;
		int64 CompressedContainerSize = 0;
	};

	struct FContainerDiff
	{
		TSet<FIoChunkId> Unmodified;
		TSet<FIoChunkId> Modified;
		TSet<FIoChunkId> Added;
		TSet<FIoChunkId> Removed;
		int64 UnmodifiedCompressedSize = 0;
		int64 ModifiedCompressedSize = 0;
		int64 AddedCompressedSize = 0;
		int64 RemovedCompressedSize = 0;
	};

	using FContainers = TMap<FString, FContainerChunkInfo>;

	auto ReadContainers = [](const FString& Directory, const FKeyChain& KeyChain, FContainers& OutContainers)
	{
		TArray<FString> ContainerFileNames;
		IFileManager::Get().FindFiles(ContainerFileNames, *(Directory / TEXT("*.utoc")), true, false);

		for (const FString& ContainerFileName : ContainerFileNames)
		{
			FString ContainerFilePath = Directory / ContainerFileName;
			UE_LOG(LogIoStore, Display, TEXT("Reading container '%s'"), *ContainerFilePath);

			TUniquePtr<FIoStoreReader> Reader = CreateIoStoreReader(*ContainerFilePath, KeyChain);
			if (!Reader.IsValid())
			{
				UE_LOG(LogIoStore, Warning, TEXT("Failed to read container '%s'"), *ContainerFilePath);
				continue;
			}

			FString ContainerName = FPaths::GetBaseFilename(ContainerFileName);
			FContainerChunkInfo& ContainerChunkInfo = OutContainers.FindOrAdd(ContainerName);
			ContainerChunkInfo.ContainerName = MoveTemp(ContainerName);

			Reader->EnumerateChunks([&ContainerChunkInfo](const FIoStoreTocChunkInfo& ChunkInfo)
			{
				ContainerChunkInfo.ChunkInfoById.Add(ChunkInfo.Id, ChunkInfo);
				ContainerChunkInfo.UncompressedContainerSize += ChunkInfo.Size;
				ContainerChunkInfo.CompressedContainerSize += ChunkInfo.CompressedSize;
				return true;
			});
		}
	};

	auto ComputeDiff = [](const FContainerChunkInfo& SourceContainer, const FContainerChunkInfo& TargetContainer) -> FContainerDiff 
	{
		check(SourceContainer.ContainerName == TargetContainer.ContainerName);

		FContainerDiff ContainerDiff;

		for (const auto& TargetChunkInfo : TargetContainer.ChunkInfoById)
		{
			if (const FIoStoreTocChunkInfo* SourceChunkInfo = SourceContainer.ChunkInfoById.Find(TargetChunkInfo.Key))
			{
				if (SourceChunkInfo->Hash != TargetChunkInfo.Value.Hash)
				{
					ContainerDiff.Modified.Add(TargetChunkInfo.Key);
					ContainerDiff.ModifiedCompressedSize += TargetChunkInfo.Value.CompressedSize;
				}
				else
				{
					ContainerDiff.Unmodified.Add(TargetChunkInfo.Key);
					ContainerDiff.UnmodifiedCompressedSize += TargetChunkInfo.Value.CompressedSize;
				}
			}
			else
			{
				ContainerDiff.Added.Add(TargetChunkInfo.Key);
				ContainerDiff.AddedCompressedSize += TargetChunkInfo.Value.CompressedSize;
			}
		}

		for (const auto& SourceChunkInfo : SourceContainer.ChunkInfoById)
		{
			if (!TargetContainer.ChunkInfoById.Contains(SourceChunkInfo.Key))
			{
				ContainerDiff.Removed.Add(SourceChunkInfo.Key);
				ContainerDiff.RemovedCompressedSize += SourceChunkInfo.Value.CompressedSize;
			}
		}

		return MoveTemp(ContainerDiff);
	};

	FOutputDevice* OutputDevice = GWarn;
	TUniquePtr<FOutputDeviceFile> FileOutputDevice;

	if (!OutPath.IsEmpty())
	{
		UE_LOG(LogIoStore, Error, TEXT("Redirecting output to: '%s'"), *OutPath);

		FileOutputDevice = MakeUnique<FOutputDeviceFile>(*OutPath, true);
		FileOutputDevice->SetSuppressEventTag(true);
		OutputDevice = FileOutputDevice.Get();
	}

	FContainers SourceContainers, TargetContainers;
	TArray<FString> AddedContainers, ModifiedContainers, RemovedContainers;
	TArray<FContainerDiff> ContainerDiffs;

	UE_LOG(LogIoStore, Display, TEXT("Reading source container(s) from '%s':"), *SourcePath);
	ReadContainers(SourcePath, SourceKeyChain, SourceContainers);

	if (!SourceContainers.Num())
	{
		UE_LOG(LogIoStore, Error, TEXT("Failed to read source container(s) from '%s':"), *SourcePath);
		return -1;
	}

	UE_LOG(LogIoStore, Display, TEXT("Reading target container(s) from '%s':"), *TargetPath);
	ReadContainers(TargetPath, TargetKeyChain, TargetContainers);

	if (!TargetContainers.Num())
	{
		UE_LOG(LogIoStore, Error, TEXT("Failed to read target container(s) from '%s':"), *SourcePath);
		return -1;
	}

	for (const auto& TargetContainer : TargetContainers)
	{
		if (SourceContainers.Contains(TargetContainer.Key))
		{
			ModifiedContainers.Add(TargetContainer.Key);
		}
		else
		{
			AddedContainers.Add(TargetContainer.Key);
		}
	}

	for (const auto& SourceContainer : SourceContainers)
	{
		if (!TargetContainers.Contains(SourceContainer.Key))
		{
			RemovedContainers.Add(SourceContainer.Key);
		}
	}

	for (const FString& ModifiedContainer : ModifiedContainers)
	{
		ContainerDiffs.Emplace(ComputeDiff(*SourceContainers.Find(ModifiedContainer), *TargetContainers.Find(ModifiedContainer)));
	}

	OutputDevice->Logf(ELogVerbosity::Display, TEXT(""));
	OutputDevice->Logf(ELogVerbosity::Display, TEXT("------------------------------ Container Diff Summary ------------------------------"));
	OutputDevice->Logf(ELogVerbosity::Display, TEXT("Source path '%s'"), *SourcePath);
	OutputDevice->Logf(ELogVerbosity::Display, TEXT("Target path '%s'"), *TargetPath);

	OutputDevice->Logf(ELogVerbosity::Display, TEXT(""));
	OutputDevice->Logf(ELogVerbosity::Display, TEXT("Source container file(s):"));
	OutputDevice->Logf(ELogVerbosity::Display, TEXT(""));
	OutputDevice->Logf(ELogVerbosity::Display, TEXT("%-40s %15s %15s"), TEXT("Container"), TEXT("Size (MB)"), TEXT("Chunks"));
	OutputDevice->Logf(ELogVerbosity::Display, TEXT("-------------------------------------------------------------------------"));

	{
		uint64 TotalSourceBytes = 0;
		uint64 TotalSourceChunks = 0;

		for (const auto& NameContainerPair : SourceContainers)
		{
			const FContainerChunkInfo& SourceContainer = NameContainerPair.Value;
			OutputDevice->Logf(ELogVerbosity::Display, TEXT("%-40s %15.2lf %15d"), *SourceContainer.ContainerName, double(SourceContainer.CompressedContainerSize) / 1024.0 / 1024.0, SourceContainer.ChunkInfoById.Num());

			TotalSourceBytes += SourceContainer.CompressedContainerSize;
			TotalSourceChunks += SourceContainer.ChunkInfoById.Num();
		}

		OutputDevice->Logf(ELogVerbosity::Display, TEXT("-------------------------------------------------------------------------"));
		OutputDevice->Logf(ELogVerbosity::Display, TEXT("%-40s %15.2lf %15d"), *FString::Printf(TEXT("Total of %d container file(s)"), SourceContainers.Num()), double(TotalSourceBytes) / 1024.0 / 1024.0, TotalSourceChunks);
	}

	{
		uint64 TotalTargetBytes = 0;
		uint64 TotalTargetChunks = 0;
		uint64 TotalUnmodifiedChunks = 0;
		uint64 TotalUnmodifiedCompressedBytes = 0;
		uint64 TotalModifiedChunks = 0;
		uint64 TotalModifiedCompressedBytes = 0;
		uint64 TotalAddedChunks = 0;
		uint64 TotalAddedCompressedBytes = 0;
		uint64 TotalRemovedChunks = 0;
		uint64 TotalRemovedCompressedBytes = 0;

		if (ModifiedContainers.Num())
		{
			OutputDevice->Logf(ELogVerbosity::Display, TEXT(""));
			OutputDevice->Logf(ELogVerbosity::Display, TEXT("Target container file(s):"));
			OutputDevice->Logf(ELogVerbosity::Display, TEXT(""));
			OutputDevice->Logf(ELogVerbosity::Display, TEXT("%-40s %15s %15s %25s %25s %25s %25s %25s %25s %25s %25s"), TEXT("Container"), TEXT("Size (MB)"), TEXT("Chunks"), TEXT("Unmodified"), TEXT("Unmodified (MB)"), TEXT("Modified"), TEXT("Modified (MB)"), TEXT("Added"), TEXT("Added (MB)"), TEXT("Removed"), TEXT("Removed (MB)"));
			OutputDevice->Logf(ELogVerbosity::Display, TEXT("----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------"));

			for (int32 Idx = 0; Idx < ModifiedContainers.Num(); Idx++)
			{
				const FContainerChunkInfo& SourceContainer = *SourceContainers.Find(ModifiedContainers[Idx]);
				const FContainerChunkInfo& TargetContainer = *TargetContainers.Find(ModifiedContainers[Idx]);
				const FContainerDiff& Diff = ContainerDiffs[Idx];

				const int32 NumChunks = TargetContainer.ChunkInfoById.Num();
				const int32 NumSourceChunks = SourceContainer.ChunkInfoById.Num();

				OutputDevice->Logf(ELogVerbosity::Display, TEXT("%-40s %15s %15d %25s %25s %25s %25s %25s %25s %25s %25s"),
					*TargetContainer.ContainerName,
					*FString::Printf(TEXT("%.2lf"),
						double(TargetContainer.CompressedContainerSize) / 1024.0 / 1024.0),
					NumChunks,
					*FString::Printf(TEXT("%d (%.2lf%%)"),
						Diff.Unmodified.Num(),
						100.0 * (double(Diff.Unmodified.Num()) / double(NumChunks))),
					*FString::Printf(TEXT("%.2lf (%.2lf%%)"),
						double(Diff.UnmodifiedCompressedSize) / 1024.0 / 1024.0,
						100.0 * (Diff.UnmodifiedCompressedSize) / double(TargetContainer.CompressedContainerSize)),
					*FString::Printf(TEXT("%d (%.2lf%%)"),
						Diff.Modified.Num(),
						100.0 * (double(Diff.Modified.Num()) / double(NumChunks))),
					*FString::Printf(TEXT("%.2lf (%.2lf%%)"),
						double(Diff.ModifiedCompressedSize) / 1024.0 / 1024.0,
						100.0 * (Diff.ModifiedCompressedSize) / double(TargetContainer.CompressedContainerSize)),
					*FString::Printf(TEXT("%d (%.2lf%%)"),
						Diff.Added.Num(),
						100.0 * (double(Diff.Added.Num()) / double(NumChunks))),
					*FString::Printf(TEXT("%.2lf (%.2lf%%)"),
						double(Diff.AddedCompressedSize) / 1024.0 / 1024.0,
						100.0 * (Diff.AddedCompressedSize) / double(TargetContainer.CompressedContainerSize)),
					*FString::Printf(TEXT("%d/%d (%.2lf%%)"),
						Diff.Removed.Num(),
						NumSourceChunks,
						100.0 * (double(Diff.Removed.Num()) / double(NumSourceChunks))),
					*FString::Printf(TEXT("%.2lf (%.2lf%%)"),
						double(Diff.RemovedCompressedSize) / 1024.0 / 1024.0,
						100.0 * (Diff.RemovedCompressedSize) / double(SourceContainer.CompressedContainerSize)));

				TotalTargetBytes += TargetContainer.CompressedContainerSize;
				TotalTargetChunks += NumChunks;
				TotalUnmodifiedChunks += Diff.Unmodified.Num();
				TotalUnmodifiedCompressedBytes += Diff.UnmodifiedCompressedSize;
				TotalModifiedChunks += Diff.Modified.Num();
				TotalModifiedCompressedBytes += Diff.ModifiedCompressedSize;
				TotalAddedChunks += Diff.Added.Num();
				TotalAddedCompressedBytes += Diff.AddedCompressedSize;
				TotalRemovedChunks += Diff.Removed.Num();
				TotalRemovedCompressedBytes += Diff.RemovedCompressedSize;
			}
		}

		if (AddedContainers.Num())
		{
			for (const FString& AddedContainer : AddedContainers)
			{
				const FContainerChunkInfo& TargetContainer = *TargetContainers.Find(AddedContainer);
				OutputDevice->Logf(ELogVerbosity::Display, TEXT("+%-39s %15.2lf %15d %25s %25s %25s %25s %25s %25s %25s %25s"),
					*TargetContainer.ContainerName,
					double(TargetContainer.CompressedContainerSize) / 1024.0 / 1024.0,
					TargetContainer.ChunkInfoById.Num(),
					TEXT("-"), TEXT("-"), TEXT("-"), TEXT("-"), TEXT("-"), TEXT("-"), TEXT("-"), TEXT("-"));

				TotalTargetBytes += TargetContainer.CompressedContainerSize;
				TotalTargetChunks += TargetContainer.ChunkInfoById.Num();
			}
		}

		OutputDevice->Logf(ELogVerbosity::Display, TEXT("----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------"));
		OutputDevice->Logf(ELogVerbosity::Display, TEXT("%-40s %15.2lf %15d %25d %25.2f %25d %25.2f %25d %25.2f %25d %25.2f"),
			*FString::Printf(TEXT("Total of %d container file(s)"), TargetContainers.Num()),
			double(TotalTargetBytes) / 1024.0 / 1024.0,
			TotalTargetChunks,
			TotalUnmodifiedChunks,
			double(TotalUnmodifiedCompressedBytes) / 1024.0 / 1024.0,
			TotalModifiedChunks,
			double(TotalModifiedCompressedBytes) / 1024.0 / 1024.0,
			TotalAddedChunks,
			double(TotalAddedCompressedBytes) / 1024.0 / 1024.0,
			TotalRemovedChunks,
			double(TotalRemovedCompressedBytes) / 1024.0 / 1024.0);
	}

	return 0;
}

int32 Staged2Zen(const FString& BuildPath, const FKeyChain& KeyChain, const ITargetPlatform* TargetPlatform)
{
	FString PlatformName = TargetPlatform->PlatformName();
	FString CookedOutputPath = FPaths::Combine(FPaths::ProjectDir(), TEXT("Saved"), TEXT("Cooked"), PlatformName);
	if (IFileManager::Get().DirectoryExists(*CookedOutputPath))
	{
		UE_LOG(LogIoStore, Error, TEXT("'%s' already exists"), *CookedOutputPath);
		return -1;
	}

	TArray<FString> ContainerFiles;
	IFileManager::Get().FindFilesRecursive(ContainerFiles, *BuildPath, TEXT("*.utoc"), true, false);
	if (ContainerFiles.IsEmpty())
	{
		UE_LOG(LogIoStore, Error, TEXT("No container files found"));
		return -1;
	}

	TArray<FString> PakFiles;
	IFileManager::Get().FindFilesRecursive(PakFiles, *BuildPath, TEXT("*.pak"), true, false);
	if (PakFiles.IsEmpty())
	{
		UE_LOG(LogIoStore, Error, TEXT("No pak files found"));
		return -1;
	}

	UE_LOG(LogIoStore, Display, TEXT("Extracting files from paks..."));
	FPakPlatformFile PakPlatformFile;
	for (const auto& KV : KeyChain.EncryptionKeys)
	{
		FCoreDelegates::GetRegisterEncryptionKeyMulticastDelegate().Broadcast(KV.Key, KV.Value.Key);
	}
	PakPlatformFile.Initialize(&FPlatformFileManager::Get().GetPlatformFile(), TEXT(""));
	FString CookedEngineContentPath = FPaths::Combine(CookedOutputPath, TEXT("Engine"), TEXT("Content"));
	IFileManager::Get().MakeDirectory(*CookedEngineContentPath, true);
	FString CookedProjectContentPath = FPaths::Combine(CookedOutputPath, FApp::GetProjectName(), TEXT("Content"));
	IFileManager::Get().MakeDirectory(*CookedProjectContentPath, true);
	FString EngineContentPakPath = TEXT("../../../Engine/Content/");
	FString ProjectContentPakPath = FPaths::Combine(TEXT("../../.."), FApp::GetProjectName(), TEXT("Content"));
	for (const FString& PakFilePath : PakFiles)
	{
		PakPlatformFile.Mount(*PakFilePath, 0);
		TArray<FString> FilesInPak;
		PakPlatformFile.GetPrunedFilenamesInPakFile(PakFilePath, FilesInPak);
		for (const FString& FileInPak : FilesInPak)
		{
			FString FileName = FPaths::GetCleanFilename(FileInPak);
			if (FileName == TEXT("AssetRegistry.bin"))
			{
				FString TargetPath = FPaths::Combine(CookedOutputPath, FApp::GetProjectName(), FileName);
				PakPlatformFile.CopyFile(*TargetPath, *FileInPak);
			}
			else if (FileName.EndsWith(TEXT(".ushaderbytecode")))
			{
				FString TargetPath = FPaths::Combine(CookedProjectContentPath, FileName);
				PakPlatformFile.CopyFile(*TargetPath, *FileInPak);
			}
			else if (FileName.StartsWith("GlobalShaderCache"))
			{
				FString TargetPath = FPaths::Combine(CookedOutputPath, TEXT("Engine"), FileName);
				PakPlatformFile.CopyFile(*TargetPath, *FileInPak);
			}
			else if (FileName.EndsWith(TEXT(".ufont")))
			{
				FString TargetPath;
				if (FileInPak.StartsWith(EngineContentPakPath))
				{
					TargetPath = FPaths::Combine(CookedEngineContentPath, *FileInPak + EngineContentPakPath.Len());
				}
				else if (FileInPak.StartsWith(ProjectContentPakPath))
				{
					TargetPath = FPaths::Combine(CookedProjectContentPath, *FileInPak + ProjectContentPakPath.Len());
				}
				else
				{
					UE_DEBUG_BREAK();
					continue;
				}
				IFileManager::Get().MakeDirectory(*FPaths::GetPath(TargetPath), true);
				PakPlatformFile.CopyFile(*TargetPath, *FileInPak);
			}
		}
	}

	struct FBulkDataInfo
	{
		FString FileName;
		IPackageStoreWriter::FBulkDataInfo::EType BulkDataType;
		TTuple<FIoStoreReader*, FIoChunkId> Chunk;
	};

	struct FPackageInfo
	{
		FName PackageName;
		FString FileName;
		TTuple<FIoStoreReader*, FIoChunkId> Chunk;
		TArray<FBulkDataInfo> BulkData;
		FPackageStoreEntryResource PackageStoreEntry;
	};

	struct FCollectedData
	{
		TSet<FIoChunkId> SeenChunks;
		TMap<FName, FPackageInfo> Packages;
		TMap<FPackageId, FName> PackageIdToName;
		TArray<TTuple<FIoStoreReader*, FIoChunkId>> ContainerHeaderChunks;
	} CollectedData;

	UE_LOG(LogIoStore, Display, TEXT("Collecting chunks..."));
	TArray<TUniquePtr<FIoStoreReader>> IoStoreReaders;
	IoStoreReaders.Reserve(ContainerFiles.Num());
	for (const FString& ContainerFilePath : ContainerFiles)
	{
		TUniquePtr<FIoStoreReader> Reader = CreateIoStoreReader(*ContainerFilePath, KeyChain);
		if (!Reader.IsValid())
		{
			UE_LOG(LogIoStore, Warning, TEXT("Failed to read container '%s'"), *ContainerFilePath);
			continue;
		}

		TMap<FIoChunkId, FString> ChunkFileNamesMap;
		if (EnumHasAnyFlags(Reader->GetContainerFlags(), EIoContainerFlags::Indexed))
		{
			const FIoDirectoryIndexReader& IndexReader = Reader->GetDirectoryIndexReader();
			IterateDirectoryIndex(
				FIoDirectoryIndexHandle::RootDirectory(),
				TEXT(""),
				IndexReader,
				[&ChunkFileNamesMap, &Reader](FString Filename, uint32 TocEntryIndex) -> bool
				{
					TIoStatusOr<FIoStoreTocChunkInfo> ChunkInfo = Reader->GetChunkInfo(TocEntryIndex);
					if (ChunkInfo.IsOk())
					{
						ChunkFileNamesMap.Add(ChunkInfo.ValueOrDie().Id, Filename);
					}
					return true;
				});
		}
		Reader->EnumerateChunks([&ChunkFileNamesMap, &Reader, &CollectedData](const FIoStoreTocChunkInfo& ChunkInfo)
			{
				if (CollectedData.SeenChunks.Contains(ChunkInfo.Id))
				{
					return true;
				}
				CollectedData.SeenChunks.Add(ChunkInfo.Id);
				EIoChunkType ChunkType = static_cast<EIoChunkType>(ChunkInfo.Id.GetData()[11]);
				if (ChunkType == EIoChunkType::ExportBundleData ||
					ChunkType == EIoChunkType::BulkData ||
					ChunkType == EIoChunkType::OptionalBulkData ||
					ChunkType == EIoChunkType::MemoryMappedBulkData)
				{
					FString PackageNameStr;
					FString* FindFileName = ChunkFileNamesMap.Find(ChunkInfo.Id);
					UE_CLOG(!FindFileName, LogIoStore, Fatal, TEXT("Missing file name for package chunk"));
					if (FPackageName::TryConvertFilenameToLongPackageName(*FindFileName, PackageNameStr, nullptr))
					{
						FName PackageName(PackageNameStr);
						CollectedData.PackageIdToName.Add(FPackageId::FromName(PackageName), PackageName);
						FPackageInfo& PackageInfo = CollectedData.Packages.FindOrAdd(PackageName);
						if (ChunkType == EIoChunkType::ExportBundleData)
						{
							PackageInfo.FileName = *FindFileName;
							PackageInfo.PackageName = PackageName;
							PackageInfo.Chunk = MakeTuple(Reader.Get(), ChunkInfo.Id);
						}
						else
						{
							FBulkDataInfo& BulkDataInfo = PackageInfo.BulkData.AddDefaulted_GetRef();
							BulkDataInfo.FileName = *FindFileName;
							BulkDataInfo.Chunk = MakeTuple(Reader.Get(), ChunkInfo.Id);
							if (ChunkType == EIoChunkType::OptionalBulkData)
							{
								BulkDataInfo.BulkDataType = IPackageStoreWriter::FBulkDataInfo::Optional;
							}
							else if (ChunkType == EIoChunkType::MemoryMappedBulkData)
							{
								BulkDataInfo.BulkDataType = IPackageStoreWriter::FBulkDataInfo::Mmap;
							}
							else
							{
								BulkDataInfo.BulkDataType = IPackageStoreWriter::FBulkDataInfo::Standard;
							}
						}
					}
					else
					{
						UE_LOG(LogIoStore, Warning, TEXT("Failed to convert file name '%s' to package name"), **FindFileName);
					}
				}
				else if (ChunkType == EIoChunkType::ContainerHeader)
				{
					CollectedData.ContainerHeaderChunks.Emplace(Reader.Get(), ChunkInfo.Id);
				}
				return true;
			});

		IoStoreReaders.Emplace(MoveTemp(Reader));
	}

	UE_LOG(LogIoStore, Display, TEXT("Reading container headers..."));
	for (const auto& ContainerHeaderChunk : CollectedData.ContainerHeaderChunks)
	{
		FIoBuffer ContainerHeaderBuffer = ContainerHeaderChunk.Key->Read(ContainerHeaderChunk.Value, FIoReadOptions()).ValueOrDie();
		FMemoryReaderView Ar(MakeArrayView(ContainerHeaderBuffer.Data(), ContainerHeaderBuffer.DataSize()));
		FContainerHeader ContainerHeader;
		Ar << ContainerHeader;
		
		const FFilePackageStoreEntry* StoreEntry = reinterpret_cast<const FFilePackageStoreEntry*>(ContainerHeader.StoreEntries.GetData());
		for (const FPackageId& PackageId : ContainerHeader.PackageIds)
		{
			const FName* FindPackageName = CollectedData.PackageIdToName.Find(PackageId);
			if (FindPackageName)
			{
				FPackageInfo* FindPackageInfo = CollectedData.Packages.Find(*FindPackageName);
				check(FindPackageInfo);
				
				FPackageStoreEntryResource& PackageStoreEntryResource = FindPackageInfo->PackageStoreEntry;
				PackageStoreEntryResource.PackageName = *FindPackageName;
				PackageStoreEntryResource.ExportInfo.ExportBundleCount = StoreEntry->ExportBundleCount;
				PackageStoreEntryResource.ExportInfo.ExportBundlesSize = StoreEntry->ExportBundlesSize;
				PackageStoreEntryResource.ExportInfo.ExportCount = StoreEntry->ExportCount;
				PackageStoreEntryResource.ExportInfo.LoadOrder = StoreEntry->LoadOrder;
				PackageStoreEntryResource.ImportedPackageIds.SetNum(StoreEntry->ImportedPackages.Num());
				FMemory::Memcpy(PackageStoreEntryResource.ImportedPackageIds.GetData(), StoreEntry->ImportedPackages.Data(), sizeof(FPackageId) * StoreEntry->ImportedPackages.Num());
			}
			++StoreEntry;
		}
	}

	FString MetaDataOutputPath = FPaths::Combine(CookedOutputPath, FApp::GetProjectName(), TEXT("Metadata"));
	TUniquePtr<FZenStoreWriter> ZenStoreWriter = MakeUnique<FZenStoreWriter>(CookedOutputPath, MetaDataOutputPath, TargetPlatform, true);
	
	IPackageStoreWriter::FCookInfo CookInfo;
	ZenStoreWriter->BeginCook(CookInfo);
	int32 LocalPackageIndex = 0;
	TArray<FPackageInfo> PackagesArray;
	CollectedData.Packages.GenerateValueArray(PackagesArray);
	TAtomic<int32> UploadCount { 0 };
	ParallelFor(PackagesArray.Num(), [&UploadCount, &PackagesArray, &ZenStoreWriter](int32 Index)
	{
		const FPackageInfo& PackageInfo = PackagesArray[Index];

		IPackageStoreWriter::FBeginPackageInfo BeginPackageInfo;
		BeginPackageInfo.PackageName = PackageInfo.PackageName;

		ZenStoreWriter->BeginPackage(BeginPackageInfo);

		IPackageStoreWriter::FPackageInfo PackageStorePackageInfo;
		PackageStorePackageInfo.PackageName = PackageInfo.PackageName;
		PackageStorePackageInfo.LooseFilePath = PackageInfo.FileName;
		PackageStorePackageInfo.ChunkId = PackageInfo.Chunk.Value;

		FIoBuffer PackageDataBuffer = PackageInfo.Chunk.Key->Read(PackageInfo.Chunk.Value, FIoReadOptions()).ValueOrDie();
		ZenStoreWriter->WriteIoStorePackageData(PackageStorePackageInfo, PackageDataBuffer, PackageInfo.PackageStoreEntry, TArray<FFileRegion>());

		for (const FBulkDataInfo& BulkDataInfo : PackageInfo.BulkData)
		{
			IPackageStoreWriter::FBulkDataInfo PackageStoreBulkDataInfo;
			PackageStoreBulkDataInfo.PackageName = PackageInfo.PackageName;
			PackageStoreBulkDataInfo.LooseFilePath = BulkDataInfo.FileName;
			PackageStoreBulkDataInfo.ChunkId = BulkDataInfo.Chunk.Value;
			PackageStoreBulkDataInfo.BulkdataType = BulkDataInfo.BulkDataType;
			FIoBuffer BulkDataBuffer = BulkDataInfo.Chunk.Key->Read(BulkDataInfo.Chunk.Value, FIoReadOptions()).ValueOrDie();
			ZenStoreWriter->WriteBulkdata(PackageStoreBulkDataInfo, BulkDataBuffer, TArray<FFileRegion>());
		}
		
		IPackageStoreWriter::FCommitPackageInfo CommitInfo;
		CommitInfo.PackageName = PackageInfo.PackageName;
		ZenStoreWriter->CommitPackage(CommitInfo);

		int32 LocalUploadCount = UploadCount.IncrementExchange() + 1;
		UE_CLOG(LocalUploadCount % 1000 == 0, LogIoStore, Display, TEXT("Uploading package %d/%d"), LocalUploadCount, PackagesArray.Num());
	}, EParallelForFlags::ForceSingleThread); // Single threaded for now to limit memory usage

	UE_LOG(LogIoStore, Display, TEXT("Waiting for uploads to finish..."));
	ZenStoreWriter->EndCook();
	return 0;
}

static bool ParsePakResponseFile(const TCHAR* FilePath, TArray<FContainerSourceFile>& OutFiles)
{
	TArray<FString> ResponseFileContents;
	if (!FFileHelper::LoadFileToStringArray(ResponseFileContents, FilePath))
	{
		UE_LOG(LogIoStore, Error, TEXT("Failed to read response file '%s'."), *FilePath);
		return false;
	}

	for (const FString& ResponseLine : ResponseFileContents)
	{
		TArray<FString> SourceAndDest;
		TArray<FString> Switches;

		FString NextToken;
		const TCHAR* ResponseLinePtr = *ResponseLine;
		while (FParse::Token(ResponseLinePtr, NextToken, false))
		{
			if ((**NextToken == TCHAR('-')))
			{
				new(Switches) FString(NextToken.Mid(1));
			}
			else
			{
				new(SourceAndDest) FString(NextToken);
			}
		}

		if (SourceAndDest.Num() == 0)
		{
			continue;
		}

		if (SourceAndDest.Num() != 2)
		{
			UE_LOG(LogIoStore, Error, TEXT("Invalid line in response file '%s'."), *ResponseLine);
			return false;
		}

		FPaths::NormalizeFilename(SourceAndDest[0]);

		FContainerSourceFile& FileEntry = OutFiles.AddDefaulted_GetRef();
		FileEntry.NormalizedPath = MoveTemp(SourceAndDest[0]);
		FileEntry.DestinationPath = MoveTemp(SourceAndDest[1]);

		for (int32 Index = 0; Index < Switches.Num(); ++Index)
		{
			if (Switches[Index] == TEXT("compress"))
			{
				FileEntry.bNeedsCompression = true;
			}
			if (Switches[Index] == TEXT("encrypt"))
			{
				FileEntry.bNeedsEncryption = true;
			}
		}
	}
	return true;
}

static bool ParsePakOrderFile(const TCHAR* FilePath, FFileOrderMap& Map)
{
	IOSTORE_CPU_SCOPE(ParsePakOrderFile);

	TArray<FString> OrderFileContents;
	if (!FFileHelper::LoadFileToStringArray(OrderFileContents, FilePath))
	{
		UE_LOG(LogIoStore, Error, TEXT("Failed to read order file '%s'."), *FilePath);
		return false;
	}

	Map.Name = FPaths::GetCleanFilename(FilePath);
	UE_LOG(LogIoStore, Display, TEXT("Order file %s (short name %s) priority %d"), FilePath, *Map.Name, Map.Priority);
	int64 NextOrder = 0;
	for (const FString& OrderLine : OrderFileContents)
	{
		const TCHAR* OrderLinePtr = *OrderLine;
		FString Path;

		// Skip comments
		if (FCString::Strncmp(OrderLinePtr, TEXT("#"), 1) == 0 || FCString::Strncmp(OrderLinePtr, TEXT("//"), 2) == 0)
		{
			continue;
		}

		if (!FParse::Token(OrderLinePtr, Path, false))
		{
			UE_LOG(LogIoStore, Error, TEXT("Invalid line in order file '%s'."), *OrderLine);
			return false;
		}
		FString PackageName;
		if (!FPackageName::TryConvertFilenameToLongPackageName(Path, PackageName, nullptr))
		{
			continue;;
		}

		FName PackageFName(MoveTemp(PackageName));
		if (!Map.PackageNameToOrder.Contains(PackageFName))
		{
			Map.PackageNameToOrder.Emplace(PackageFName, NextOrder++);
		}
	}

	UE_LOG(LogIoStore, Display, TEXT("Order file %s (short name %s) contained %d valid entries"), FilePath, *Map.Name, Map.PackageNameToOrder.Num());
	return true;
}

class FCookedFileVisitor : public IPlatformFile::FDirectoryStatVisitor
{
	FCookedFileStatMap& CookedFileStatMap;
	FContainerSourceSpec* ContainerSpec = nullptr;
	bool bFileRegions;

public:
	FCookedFileVisitor(FCookedFileStatMap& InCookedFileSizes, FContainerSourceSpec* InContainerSpec, bool bInFileRegions)
		: CookedFileStatMap(InCookedFileSizes)
		, ContainerSpec(InContainerSpec)
		, bFileRegions(bInFileRegions)
	{}

	FCookedFileVisitor(FCookedFileStatMap& InFileSizes, bool bInFileRegions)
		: CookedFileStatMap(InFileSizes)
		, bFileRegions(bInFileRegions)
	{}

	virtual bool Visit(const TCHAR* FilenameOrDirectory, const FFileStatData& StatData)
	{
		// Should match FCookedFileStatData::EFileExt
		static const TCHAR* Extensions[] = { TEXT("umap"), TEXT("uasset"), TEXT("uexp"), TEXT("ubulk"), TEXT("uptnl"), TEXT("m.ubulk"), TEXT("ushaderbytecode") };
		static const int32 NumPackageExtensions = 2;
		static const int32 UExpExtensionIndex = 2;
		static const int32 UShaderByteCodeExtensionIndex = 6;

		if (StatData.bIsDirectory)
		{
			return true;
		}

		const TCHAR* Extension = FCString::Strrchr(FilenameOrDirectory, '.');
		if (!Extension || *(++Extension) == TEXT('\0'))
		{
			return true;
		}

		int32 ExtIndex = 0;
		if (0 == FCString::Stricmp(Extension, Extensions[3]))
		{
			ExtIndex = 3;
			if (0 == FCString::Stricmp(Extension - 3, TEXT(".m.ubulk")))
			{
				ExtIndex = 5;
			}
		}
		else
		{
			for (ExtIndex = 0; ExtIndex < UE_ARRAY_COUNT(Extensions); ++ExtIndex)
			{
				if (0 == FCString::Stricmp(Extension, Extensions[ExtIndex]))
					break;
			}
		}

		if (ExtIndex >= UE_ARRAY_COUNT(Extensions))
		{
			return true;
		}

		FString Path = FilenameOrDirectory;
		FPaths::NormalizeFilename(Path);

		if (ContainerSpec && ExtIndex != UExpExtensionIndex)
		{
			FContainerSourceFile& FileEntry = ContainerSpec->SourceFiles.AddDefaulted_GetRef();
			FileEntry.NormalizedPath = Path;
		}

		// Read the matching regions file, if it exists.
		TUniquePtr<FArchive> RegionsFile;
		if (bFileRegions)
		{
			RegionsFile.Reset(IFileManager::Get().CreateFileReader(*(Path + FFileRegion::RegionsFileExtension)));
		}

		FCookedFileStatData& CookedFileStatData = CookedFileStatMap.Add(MoveTemp(Path));
		CookedFileStatData.FileSize = StatData.FileSize;
		CookedFileStatData.FileExt = FCookedFileStatData::EFileExt(ExtIndex);
		if (ExtIndex < NumPackageExtensions)
		{
			CookedFileStatData.FileType = FCookedFileStatData::PackageHeader;
		}
		else if (ExtIndex == UExpExtensionIndex)
		{
			CookedFileStatData.FileType = FCookedFileStatData::PackageData;
		}
		else if (ExtIndex == UShaderByteCodeExtensionIndex)
		{
			CookedFileStatData.FileType = FCookedFileStatData::ShaderLibrary;
		}
		else
		{
			CookedFileStatData.FileType = FCookedFileStatData::BulkData;
		}

		if (RegionsFile.IsValid())
		{
			FFileRegion::SerializeFileRegions(*RegionsFile.Get(), CookedFileStatData.FileRegions);
		}

		return true;
	}
};

static bool ParseSizeArgument(const TCHAR* CmdLine, const TCHAR* Argument, uint64& OutSize, uint64 DefaultSize = 0)
{
	FString SizeString;
	if (FParse::Value(CmdLine, Argument, SizeString) && FParse::Value(CmdLine, Argument, OutSize))
	{
		if (SizeString.EndsWith(TEXT("MB")))
		{
			OutSize *= 1024*1024;
		}
		else if (SizeString.EndsWith(TEXT("KB")))
		{
			OutSize *= 1024;
		}
		return true;
	}
	else
	{
		OutSize = DefaultSize;
		return false;
	}
}

int32 CreateIoStoreContainerFiles(const TCHAR* CmdLine)
{
	IOSTORE_CPU_SCOPE(CreateIoStoreContainerFiles);

	UE_LOG(LogIoStore, Display, TEXT("==================== IoStore Utils ===================="));

	FIoStoreArguments Arguments;

	LoadKeyChain(FCommandLine::Get(), Arguments.KeyChain);

	if (FParse::Param(FCommandLine::Get(), TEXT("sign")))
	{
		Arguments.bSign = true;
	}

	UE_LOG(LogIoStore, Display, TEXT("Container signing - %s"), Arguments.bSign ? TEXT("ENABLED") : TEXT("DISABLED"));

	Arguments.bCreateDirectoryIndex = !FParse::Param(FCommandLine::Get(), TEXT("NoDirectoryIndex"));
	UE_LOG(LogIoStore, Display, TEXT("Directory index - %s"), Arguments.bCreateDirectoryIndex  ? TEXT("ENABLED") : TEXT("DISABLED"));

	FString PatchReferenceCryptoKeysFilename;
	FKeyChain PatchKeyChain;
	if (FParse::Value(FCommandLine::Get(), TEXT("PatchCryptoKeys="), PatchReferenceCryptoKeysFilename))
	{
		KeyChainUtilities::LoadKeyChainFromFile(PatchReferenceCryptoKeysFilename, Arguments.PatchKeyChain);
	}

	uint64 OrderMapStartIndex = 0;
	FString OrderFileStr;
	if (FParse::Value(FCommandLine::Get(), TEXT("Order="), OrderFileStr, false))
	{
		TArray<int32> OrderFilePriorities;
		TArray<FString> OrderFilePaths;
		OrderFileStr.ParseIntoArray(OrderFilePaths, TEXT(","), true);

		FString LegacyParam;
		if (FParse::Value(FCommandLine::Get(), TEXT("GameOrder="), LegacyParam, false))
		{
			UE_LOG(LogIoStore, Warning, TEXT("-GameOrder= and -CookerOrder= are deprecated in favor of -Order"));
			TArray<FString> LegacyPaths;
			LegacyParam.ParseIntoArray(LegacyPaths, TEXT(","), true);
			OrderFilePaths.Append(LegacyPaths);
		}
		if (FParse::Value(FCommandLine::Get(), TEXT("CookerOrder="), LegacyParam, false))
		{
			UE_LOG(LogIoStore, Warning, TEXT("-GameOrder= and -CookerOrder= are deprecated in favor of -Order"));
			TArray<FString> LegacyPaths;
			LegacyParam.ParseIntoArray(LegacyPaths, TEXT(","), true);
			OrderFilePaths.Append(LegacyPaths);
		}

		FString OrderPriorityString;
		if (FParse::Value(FCommandLine::Get(), TEXT("OrderPriority="), OrderPriorityString, false))
		{
			TArray<FString> PriorityStrings;
			OrderPriorityString.ParseIntoArray(PriorityStrings, TEXT(","), true);
			if (PriorityStrings.Num() != OrderFilePaths.Num())
			{
				UE_LOG(LogIoStore, Error, TEXT("Number of parameters to -Order= and -OrderPriority= do not match"));
				return -1;
			}

			for (const FString& PriorityString : PriorityStrings)
			{
				int32 Priority = FCString::Atoi(*PriorityString);
				OrderFilePriorities.Add(Priority);
			}
		}
		else
		{
			OrderFilePriorities.AddZeroed(OrderFilePaths.Num());
		}

		check(OrderFilePaths.Num() == OrderFilePriorities.Num());

		bool bMerge = false;
		for (int32 i=0; i < OrderFilePaths.Num(); ++i)
		{
			FString& OrderFile = OrderFilePaths[i];
			int32 Priority = OrderFilePriorities[i];

			FFileOrderMap OrderMap(Priority, i);
			if (!ParsePakOrderFile(*OrderFile, OrderMap))
			{
				return -1;
			}
			Arguments.OrderMaps.Add(OrderMap);
		}
	}

	FString TargetPlatformName;
	if (FParse::Value(FCommandLine::Get(), TEXT("TargetPlatform="), TargetPlatformName))
	{
		UE_LOG(LogIoStore, Display, TEXT("Using target platform '%s'"), *TargetPlatformName);
		ITargetPlatformManagerModule& TPM = GetTargetPlatformManagerRef();
		Arguments.TargetPlatform = TPM.FindTargetPlatform(TargetPlatformName);
		if (!Arguments.TargetPlatform)
		{
			UE_LOG(LogIoStore, Error, TEXT("Invalid TargetPlatform: '%s'"), *TargetPlatformName);
			return 1;
		}
	}

	Arguments.bClusterByOrderFilePriority = !FParse::Param(CmdLine, TEXT("DoNotClusterByOrderPriority"));

	FIoStoreWriterSettings GeneralIoWriterSettings { DefaultCompressionMethod, DefaultCompressionBlockSize, false };
	GeneralIoWriterSettings.bEnableCsvOutput = FParse::Param(CmdLine, TEXT("-csvoutput"));

	TArray<FName> CompressionFormats;
	FString DesiredCompressionFormats;
	if (FParse::Value(CmdLine, TEXT("-compressionformats="), DesiredCompressionFormats) ||
		FParse::Value(CmdLine, TEXT("-compressionformat="), DesiredCompressionFormats))
	{
		TArray<FString> Formats;
		DesiredCompressionFormats.ParseIntoArray(Formats, TEXT(","));
		for (FString& Format : Formats)
		{
			// look until we have a valid format
			FName FormatName = *Format;

			if (FCompression::IsFormatValid(FormatName))
			{
				GeneralIoWriterSettings.CompressionMethod = FormatName;
				break;
			}
		}

		if (GeneralIoWriterSettings.CompressionMethod == NAME_None)
		{
			UE_LOG(LogIoStore, Warning, TEXT("Failed to find desired compression format(s) '%s'. Using falling back to '%s'"),
				*DesiredCompressionFormats, *DefaultCompressionMethod.ToString());
		}
		else
		{
			UE_LOG(LogIoStore, Display, TEXT("Using compression format '%s'"), *GeneralIoWriterSettings.CompressionMethod.ToString());
		}
	}

	ParseSizeArgument(CmdLine, TEXT("-alignformemorymapping="), GeneralIoWriterSettings.MemoryMappingAlignment, DefaultMemoryMappingAlignment);
	ParseSizeArgument(CmdLine, TEXT("-compressionblocksize="), GeneralIoWriterSettings.CompressionBlockSize, DefaultCompressionBlockSize);
		
	GeneralIoWriterSettings.CompressionBlockAlignment = DefaultCompressionBlockAlignment;
	
	uint64 BlockAlignment = 0;
	if (ParseSizeArgument(CmdLine, TEXT("-blocksize="), BlockAlignment))
	{
		GeneralIoWriterSettings.CompressionBlockAlignment = BlockAlignment;
	}
	
	uint64 PatchPaddingAlignment = 0;
	if (ParseSizeArgument(CmdLine, TEXT("-patchpaddingalign="), PatchPaddingAlignment))
	{
		if (PatchPaddingAlignment < GeneralIoWriterSettings.CompressionBlockAlignment)
		{
			GeneralIoWriterSettings.CompressionBlockAlignment = PatchPaddingAlignment;
		}
	}
	
	// Temporary, this command-line allows us to explicitly override the value otherwise shared between pak building and iostore
	uint64 IOStorePatchPaddingAlignment = 0;
	if (ParseSizeArgument(CmdLine, TEXT("-iostorepatchpaddingalign="), IOStorePatchPaddingAlignment))
	{
		GeneralIoWriterSettings.CompressionBlockAlignment = IOStorePatchPaddingAlignment;
	}

	uint64 MaxPartitionSize = 0;
	if (ParseSizeArgument(CmdLine, TEXT("-maxPartitionSize="), MaxPartitionSize))
	{
		GeneralIoWriterSettings.MaxPartitionSize = MaxPartitionSize;
	}

	if (Arguments.TargetPlatform)
	{
		GeneralIoWriterSettings.InitializePlatformSpecificSettings(Arguments.TargetPlatform);
	}

	UE_LOG(LogIoStore, Display, TEXT("Using memory mapping alignment '%ld'"), GeneralIoWriterSettings.MemoryMappingAlignment);
	UE_LOG(LogIoStore, Display, TEXT("Using compression block size '%ld'"), GeneralIoWriterSettings.CompressionBlockSize);
	UE_LOG(LogIoStore, Display, TEXT("Using compression block alignment '%ld'"), GeneralIoWriterSettings.CompressionBlockAlignment);
	UE_LOG(LogIoStore, Display, TEXT("Using compression min bytes saved '%d'"), GeneralIoWriterSettings.CompressionMinBytesSaved);
	UE_LOG(LogIoStore, Display, TEXT("Using compression min percent saved '%d'"), GeneralIoWriterSettings.CompressionMinPercentSaved);
	UE_LOG(LogIoStore, Display, TEXT("Using max partition size '%lld'"), GeneralIoWriterSettings.MaxPartitionSize);

	FParse::Value(CmdLine, TEXT("-MetaOutputDirectory="), Arguments.MetaOutputDir);
	FParse::Value(CmdLine, TEXT("-MetaInputDirectory="), Arguments.MetaInputDir);

	FString CommandListFile;
	if (FParse::Value(FCommandLine::Get(), TEXT("Commands="), CommandListFile))
	{
		UE_LOG(LogIoStore, Display, TEXT("Using command list file: '%s'"), *CommandListFile);
		TArray<FString> Commands;
		if (!FFileHelper::LoadFileToStringArray(Commands, *CommandListFile))
		{
			UE_LOG(LogIoStore, Error, TEXT("Failed to read command list file '%s'."), *CommandListFile);
			return -1;
		}

		Arguments.Containers.Reserve(Commands.Num());
		for (const FString& Command : Commands)
		{
			FContainerSourceSpec& ContainerSpec = Arguments.Containers.AddDefaulted_GetRef();

			if (!FParse::Value(*Command, TEXT("Output="), ContainerSpec.OutputPath))
			{
				UE_LOG(LogIoStore, Error, TEXT("Output argument missing from command '%s'"), *Command);
				return -1;
			}
			ContainerSpec.OutputPath = FPaths::ChangeExtension(ContainerSpec.OutputPath, TEXT(""));

			FString ContainerName;
			if (FParse::Value(*Command, TEXT("ContainerName="), ContainerName))
			{
				ContainerSpec.Name = FName(ContainerName);
			}

			FString PatchSourceWildcard;
			if (FParse::Value(*Command, TEXT("PatchSource="), PatchSourceWildcard))
			{
				IFileManager::Get().FindFiles(ContainerSpec.PatchSourceContainerFiles, *PatchSourceWildcard, true, false);
				FString PatchSourceContainersDirectory = FPaths::GetPath(*PatchSourceWildcard);
				for (FString& PatchSourceContainerFile : ContainerSpec.PatchSourceContainerFiles)
				{
					PatchSourceContainerFile = PatchSourceContainersDirectory / PatchSourceContainerFile;
					FPaths::NormalizeFilename(PatchSourceContainerFile);
				}
			}

			ContainerSpec.bGenerateDiffPatch = FParse::Param(*Command, TEXT("GenerateDiffPatch"));

			FParse::Value(*Command, TEXT("PatchTarget="), ContainerSpec.PatchTargetFile);

			FString ResponseFilePath;
			if (FParse::Value(*Command, TEXT("ResponseFile="), ResponseFilePath))
			{
				if (!ParsePakResponseFile(*ResponseFilePath, ContainerSpec.SourceFiles))
				{
					UE_LOG(LogIoStore, Error, TEXT("Failed to parse Pak response file '%s'"), *ResponseFilePath);
					return -1;
				}

				FString EncryptionKeyOverrideGuidString;
				if (FParse::Value(*Command, TEXT("EncryptionKeyOverrideGuid="), EncryptionKeyOverrideGuidString))
				{
					FGuid::Parse(EncryptionKeyOverrideGuidString, ContainerSpec.EncryptionKeyOverrideGuid);
				}
			}
		}
	}

	if (FParse::Value(FCommandLine::Get(), TEXT("BasedOnReleaseVersionPath="), Arguments.BasedOnReleaseVersionPath))
	{
		UE_LOG(LogIoStore, Display, TEXT("Based on release version path: '%s'"), *Arguments.BasedOnReleaseVersionPath);
	}

	if (FParse::Value(FCommandLine::Get(), TEXT("DLCFile="), Arguments.DLCPluginPath))
	{
		Arguments.DLCName = FPaths::GetBaseFilename(*Arguments.DLCPluginPath);
		Arguments.bRemapPluginContentToGame = FParse::Param(FCommandLine::Get(), TEXT("RemapPluginContentToGame"));

		UE_LOG(LogIoStore, Display, TEXT("DLC: '%s'"), *Arguments.DLCPluginPath);
		UE_LOG(LogIoStore, Display, TEXT("Remapping plugin content to game: '%s'"), Arguments.bRemapPluginContentToGame ? TEXT("True") : TEXT("False"));

		if (Arguments.BasedOnReleaseVersionPath.IsEmpty())
		{
			UE_LOG(LogIoStore, Error, TEXT("Based on release version path is needed for DLC"));
			return -1;
		}

		FString DevelopmentAssetRegistryPath = FPaths::Combine(Arguments.BasedOnReleaseVersionPath, TEXT("Metadata/DevelopmentAssetRegistry.bin"));

		bool bAssetRegistryLoaded = false;
		FArrayReader SerializedAssetData;
		if (FFileHelper::LoadFileToArray(SerializedAssetData, *DevelopmentAssetRegistryPath))
		{
			FAssetRegistrySerializationOptions Options;
			if (Arguments.ReleaseAssetRegistry.Serialize(SerializedAssetData, Options))
			{
				UE_LOG(LogIoStore, Display, TEXT("Loaded asset registry '%s'"), *DevelopmentAssetRegistryPath);
				bAssetRegistryLoaded = true;

				TArray<FName> PackageNames;
				Arguments.ReleaseAssetRegistry.GetPackageNames(PackageNames);
				Arguments.ReleasedPackages.PackageNames.Reserve(PackageNames.Num());
				Arguments.ReleasedPackages.PackageIdToName.Reserve(PackageNames.Num());

				for (FName PackageName : PackageNames)
				{
					Arguments.ReleasedPackages.PackageNames.Add(PackageName);
					Arguments.ReleasedPackages.PackageIdToName.Add(FPackageId::FromName(PackageName), PackageName);
				}
			}
		}

		if (!bAssetRegistryLoaded)
		{
			UE_LOG(LogIoStore, Warning, TEXT("Failed to load Asset registry '%s'. Needed to verify DLC package names"), *DevelopmentAssetRegistryPath);
		}
	}

	if (FParse::Value(FCommandLine::Get(), TEXT("CreateGlobalContainer="), Arguments.GlobalContainerPath))
	{
		Arguments.GlobalContainerPath = FPaths::ChangeExtension(Arguments.GlobalContainerPath, TEXT(""));
	}

	if (Arguments.ShouldCreateContainers())
	{
		if (!Arguments.TargetPlatform)
		{
			UE_LOG(LogIoStore, Error, TEXT("TargetPlatform must be specified"));
			return 1;
		}

		// Now that we have the target platform we need to error out if LegacyBulkDataOffsets is enabled for it
		{
			FConfigFile PlatformEngineIni;
			FConfigCacheIni::LoadLocalIniFile(PlatformEngineIni, TEXT("Engine"), true, *Arguments.TargetPlatform->IniPlatformName());

			bool bLegacyBulkDataOffsets = false;
			PlatformEngineIni.GetBool(TEXT("Core.System"), TEXT("LegacyBulkDataOffsets"), bLegacyBulkDataOffsets);

			if (bLegacyBulkDataOffsets)
			{
				UE_LOG(LogIoStore, Error, TEXT("'LegacyBulkDataOffsets' is enabled for the target platform '%s', this needs to be disabled and the data recooked in order for the IoStore to work"), *Arguments.TargetPlatform->IniPlatformName());
				return 1;
			}
		}

		if (!FParse::Value(FCommandLine::Get(), TEXT("CookedDirectory="), Arguments.CookedDir))
		{
			UE_LOG(LogIoStore, Error, TEXT("CookedDirectory must be specified"));
			return 1;
		}

		for (const FContainerSourceSpec& Container : Arguments.Containers)
		{
			if (Container.Name.IsNone())
			{
				UE_LOG(LogIoStore, Error, TEXT("ContainerName argument missing for container '%s'"), *Container.OutputPath);
				return -1;
			}
		}

		// Enable file region metadata if required by the target platform.
		Arguments.bFileRegions = Arguments.TargetPlatform->SupportsFeature(ETargetPlatformFeatures::CookFileRegionMetadata);
		GeneralIoWriterSettings.bEnableFileRegions = Arguments.bFileRegions;

		TUniquePtr<FCookedPackageStore> PackageStore = MakeUnique<FCookedPackageStore>();
		FIoStatus Status = PackageStore->Load(Arguments.CookedDir);
		if (Status.IsOk())
		{
			UE_LOG(LogIoStore, Display, TEXT("Using package store"));
			Arguments.PackageStore = MoveTemp(PackageStore);
		}
		else
		{
			IOSTORE_CPU_SCOPE(FindCookedAssets);
			UE_LOG(LogIoStore, Display, TEXT("Searching for cooked assets in folder '%s'"), *Arguments.CookedDir);
			FCookedFileVisitor CookedFileVistor(Arguments.CookedFileStatMap, nullptr, Arguments.bFileRegions);
			IFileManager::Get().IterateDirectoryStatRecursively(*Arguments.CookedDir, CookedFileVistor);
			UE_LOG(LogIoStore, Display, TEXT("Found '%d' files"), Arguments.CookedFileStatMap.Num());
		}

		const int32 ReturnValue = CreateTarget(Arguments, GeneralIoWriterSettings);
		return ReturnValue;
	}
	else if (FParse::Param(FCommandLine::Get(), TEXT("CreateContentPatch")))
	{
		for (const FContainerSourceSpec& Container : Arguments.Containers)
		{
			if (Container.PatchTargetFile.IsEmpty())
			{
				UE_LOG(LogIoStore, Error, TEXT("PatchTarget argument missing for container '%s'"), *Container.OutputPath);
				return -1;
			}
		}

		int32 ReturnValue = CreateContentPatch(Arguments, GeneralIoWriterSettings);
		if (ReturnValue != 0)
		{
			return ReturnValue;
		}
	}
	else
	{
		FString ContainerPathOrWildcard;
		if (FParse::Value(FCommandLine::Get(), TEXT("List="), ContainerPathOrWildcard))
		{
			FString CsvPath;
			if (!FParse::Value(FCommandLine::Get(), TEXT("csv="), CsvPath))
			{
				UE_LOG(LogIoStore, Error, TEXT("Incorrect arguments. Expected: -list=<ContainerFile> -csv=<path>"));
			}

			return ListContainer(Arguments, ContainerPathOrWildcard, CsvPath);
		}
		else if (FParse::Value(FCommandLine::Get(), TEXT("Describe="), ContainerPathOrWildcard))
		{
			FString PackageFilter;
			FParse::Value(FCommandLine::Get(), TEXT("PackageFilter="), PackageFilter);
			FString OutPath;
			FParse::Value(FCommandLine::Get(), TEXT("DumpToFile="), OutPath);
			bool bIncludeExportHashes = FParse::Param(FCommandLine::Get(), TEXT("IncludeExportHashes"));
			return Describe(ContainerPathOrWildcard, Arguments.KeyChain, PackageFilter, OutPath, bIncludeExportHashes);
		}
		else if (FParse::Param(FCommandLine::Get(), TEXT("Diff")))
		{
			FString SourcePath, TargetPath, OutPath;
			FKeyChain SourceKeyChain, TargetKeyChain;

			if (!FParse::Value(FCommandLine::Get(), TEXT("Source="), SourcePath))
			{
				UE_LOG(LogIoStore, Error, TEXT("Incorrect arguments. Expected: -Diff -Source=<Path> -Target=<path>"));
				return -1;
			}

			if (!IFileManager::Get().DirectoryExists(*SourcePath))
			{
				UE_LOG(LogIoStore, Error, TEXT("Source directory '%s' doesn't exist"), *SourcePath);
				return -1;
			}

			if (!FParse::Value(FCommandLine::Get(), TEXT("Target="), TargetPath))
			{
				UE_LOG(LogIoStore, Error, TEXT("Incorrect arguments. Expected: -Diff -Source=<Path> -Target=<path>"));
			}

			if (!IFileManager::Get().DirectoryExists(*TargetPath))
			{
				UE_LOG(LogIoStore, Error, TEXT("Target directory '%s' doesn't exist"), *TargetPath);
				return -1;
			}

			FParse::Value(FCommandLine::Get(), TEXT("DumpToFile="), OutPath);

			FString CryptoKeysCacheFilename;
			if (FParse::Value(CmdLine, TEXT("SourceCryptoKeys="), CryptoKeysCacheFilename))
			{
				UE_LOG(LogIoStore, Display, TEXT("Parsing source crypto keys from '%s'"), *CryptoKeysCacheFilename);
				KeyChainUtilities::LoadKeyChainFromFile(CryptoKeysCacheFilename, SourceKeyChain);
			}

			if (FParse::Value(CmdLine, TEXT("TargetCryptoKeys="), CryptoKeysCacheFilename))
			{
				UE_LOG(LogIoStore, Display, TEXT("Parsing target crypto keys from '%s'"), *CryptoKeysCacheFilename);
				KeyChainUtilities::LoadKeyChainFromFile(CryptoKeysCacheFilename, TargetKeyChain);
			}
			
			return Diff(SourcePath, SourceKeyChain, TargetPath, TargetKeyChain, OutPath);
		}
		else if (FParse::Param(FCommandLine::Get(), TEXT("Staged2Zen")))
		{
			FString BuildPath;
			if (!FParse::Value(FCommandLine::Get(), TEXT("BuildPath="), BuildPath) || !Arguments.TargetPlatform)
			{
				UE_LOG(LogIoStore, Error, TEXT("Incorrect arguments. Expected: -Staged2Zen -BuildPath=<Path> -TargetPlatform=<Platform>"));
			}
			return Staged2Zen(BuildPath, Arguments.KeyChain, Arguments.TargetPlatform);
		}
		else
		{
			UE_LOG(LogIoStore, Display, TEXT("Usage:"));
			UE_LOG(LogIoStore, Display, TEXT(" -List=</path/to/[container.utoc|*.utoc]> -CSV=<list.csv> [-CryptoKeys=</path/to/crypto.json>]"));
			UE_LOG(LogIoStore, Display, TEXT(" -Describe=</path/to/global.utoc> [-PackageFilter=<PackageName>] [-DumpToFile=<describe.txt>] [-CryptoKeys=</path/to/crypto.json>]"));
			return -1;
		}
	}

	return 0;
}

#endif