Added support for per-platform mapping from chunkid to pakchunk index. Part 1

- This feature allows different chunk layout on different platforms.
- Most mapping work is done in AssetRegistryGenerator.
- Terminology of ChunkID and Pakchunk Index are now separated and they are no longer the same if chunk id to pakchunk index mapping is defined.  The ChunkIDs in FAssetData is not renamed to keep consistency, but it actually is pakchunk index.  FAssetData.ChunkIDs is used for ChunkInstall.
- Chunks with encryption key guid or unique assetregistry name are not allowed to be mapped.  There are code to filter out those chunks and log errors while initializing mapping
- CheckChunkAssetsAreNotInChild() and ResolveChunkDependencyGraph() is skipped when making build for XboxOne and PS4, since they are not needed.  If for some reason, we need chunk dependency on console builds, we need to implement per-platform dependency graph, since now we support different chunk layout on different platforms.
- Disabled a few pak file overide rules for console builds, since they should only be applied to builds with Kairos chunk layout.

#test A preflight was kicked and tested on PS4 and XboxOne.  No new issue was found.
#rb Daniel.Lamb


#ROBOMERGE-SOURCE: CL 5470414 via CL 5474050

[CL 5474308 by hongyi yu in Main branch]

Engine/Source/Developer/AllDesktopTargetPlatform/Private/AllDesktopTargetPlatform.h

@@ -82,7 +82,7 @@ public:
 	{
 	}
 
-	virtual bool GenerateStreamingInstallManifest(const TMultiMap<FString, int32>& ChunkMap, const TSet<int32>& ChunkIDsInUse) const override
+	virtual bool GenerateStreamingInstallManifest(const TMultiMap<FString, int32>& PakchunkMap, const TSet<int32>& PakchunkIndicesInUse) const override
 	{
 		return true;
 	}

Engine/Source/Developer/Android/AndroidTargetPlatform/Private/AndroidTargetPlatform.h

@@ -144,7 +144,7 @@ public:
 
 	virtual void GetAllDevices( TArray<ITargetDevicePtr>& OutDevices ) const override;
 
-	virtual bool GenerateStreamingInstallManifest(const TMultiMap<FString, int32>& ChunkMap, const TSet<int32>& ChunkIDsInUse) const override
+	virtual bool GenerateStreamingInstallManifest(const TMultiMap<FString, int32>& PakchunkMap, const TSet<int32>& PakchunkIndicesInUse) const override
 	{
 		return true;
 	}

Engine/Source/Developer/HTML5/HTML5TargetPlatform/Private/HTML5TargetPlatform.h

@@ -40,7 +40,7 @@ public:
 
 	virtual void GetAllDevices( TArray<ITargetDevicePtr>& OutDevices ) const override;
 
-	virtual bool GenerateStreamingInstallManifest(const TMultiMap<FString, int32>& ChunkMap, const TSet<int32>& ChunkIDsInUse) const override
+	virtual bool GenerateStreamingInstallManifest(const TMultiMap<FString, int32>& PakchunkMap, const TSet<int32>& PakchunkIndicesInUse) const override
 	{
 		return true;
 	}

Engine/Source/Developer/IOS/IOSTargetPlatform/Private/IOSTargetPlatform.h

@@ -78,7 +78,7 @@ public:
 
 	virtual void GetAllDevices( TArray<ITargetDevicePtr>& OutDevices ) const override;
 
-	virtual bool GenerateStreamingInstallManifest(const TMultiMap<FString, int32>& ChunkMap, const TSet<int32>& ChunkIDsInUse) const override
+	virtual bool GenerateStreamingInstallManifest(const TMultiMap<FString, int32>& PakchunkMap, const TSet<int32>& PakchunkIndicesInUse) const override
 	{
 		return true;
 	}

Engine/Source/Developer/Linux/LinuxTargetPlatform/Private/LinuxTargetPlatform.h

@@ -118,7 +118,7 @@ public:
 		}
 	}
 
-	virtual bool GenerateStreamingInstallManifest(const TMultiMap<FString, int32>& ChunkMap, const TSet<int32>& ChunkIDsInUse) const override
+	virtual bool GenerateStreamingInstallManifest(const TMultiMap<FString, int32>& PakchunkMap, const TSet<int32>& PakchunkIndicesInUse) const override
 	{
 		return true;
 	}

Engine/Source/Developer/Mac/MacTargetPlatform/Private/GenericMacTargetPlatform.h

@@ -66,7 +66,7 @@ public:
 		}
 	}
 
-	virtual bool GenerateStreamingInstallManifest(const TMultiMap<FString, int32>& ChunkMap, const TSet<int32>& ChunkIDsInUse) const override
+	virtual bool GenerateStreamingInstallManifest(const TMultiMap<FString, int32>& PakchunkMap, const TSet<int32>& PakchunkIndicesInUse) const override
 	{
 		return true;
 	}

Engine/Source/Developer/TargetPlatform/Public/Interfaces/ITargetPlatform.h

@@ -213,11 +213,11 @@ public:
 	/**
 	 * Generates a platform specific asset manifest given an array of FAssetData.
 	 *
-	 * @param ChunkMap A map of asset path to ChunkIDs for all of the assets.
-	 * @param ChunkIDsInUse A set of all ChunkIDs used by this set of assets.
+	 * @param PakchunkMap A map of asset path to Pakchunk file indices for all of the assets.
+	 * @param PakchunkIndicesInUse A set of all Pakchunk file indices used by this set of assets.
 	 * @return true if the manifest was successfully generated, or if the platform doesn't need a manifest .
 	 */
-	virtual bool GenerateStreamingInstallManifest( const TMultiMap<FString, int32>& ChunkMap, const TSet<int32>& ChunkIDsInUse ) const = 0;
+	virtual bool GenerateStreamingInstallManifest( const TMultiMap<FString, int32>& PakchunkMap, const TSet<int32>& PakchunkIndicesInUse) const = 0;
 
 	/**
 	 * Gets the default device.

Engine/Source/Developer/Windows/WindowsTargetPlatform/Private/GenericWindowsTargetPlatform.h

@@ -110,7 +110,7 @@ public:
 		}
 	}
 
-	virtual bool GenerateStreamingInstallManifest(const TMultiMap<FString, int32>& ChunkMap, const TSet<int32>& ChunkIDsInUse) const override
+	virtual bool GenerateStreamingInstallManifest(const TMultiMap<FString, int32>& PakchunkMap, const TSet<int32>& PakchunkIndicesInUse) const override
 	{
 		return true;
 	}

Engine/Source/Editor/UnrealEd/Private/Commandlets/AssetRegistryGenerator.cpp

@@ -83,6 +83,7 @@ FAssetRegistryGenerator::FAssetRegistryGenerator(const ITargetPlatform* InPlatfo
 	, TargetPlatform(InPlatform)
 	, bGenerateChunks(false)
 	, bUseAssetManager(false)
+	, HighestChunkId(0)
 {
 	DependencyInfo = GetMutableDefault<UChunkDependencyInfo>();
 
@@ -101,6 +102,8 @@ FAssetRegistryGenerator::FAssetRegistryGenerator(const ITargetPlatform* InPlatfo
 
 		UAssetManager::Get().UpdateManagementDatabase();
 	}
+
+	InitializeChunkIdPakchunkIndexMapping();
 }
 
 FAssetRegistryGenerator::~FAssetRegistryGenerator()
@@ -244,36 +247,37 @@ bool FAssetRegistryGenerator::GenerateStreamingInstallManifest(int64 InExtraFlav
 				}
 
 				int32 ChunkID = UAssetManager::Get().GetContentEncryptionGroupChunkID(GroupName);
-				if (ChunkID >= FinalChunkManifests.Num())
+				int32 PakchunkIndex = GetPakchunkIndex(ChunkID);
+				if (PakchunkIndex >= FinalChunkManifests.Num())
 				{
-					FinalChunkManifests.AddZeroed(ChunkID - FinalChunkManifests.Num() + 1);
+					FinalChunkManifests.AddZeroed(PakchunkIndex - FinalChunkManifests.Num() + 1);
 				}
-				checkf(ChunkID < FinalChunkManifests.Num(), TEXT("Chunk %i out of range. %i manifests available"), ChunkID, FinalChunkManifests.Num() - 1);
-				checkf(FinalChunkManifests[ChunkID] == nullptr, TEXT("Manifest already exists for chunk %i"), ChunkID);
-				FinalChunkManifests[ChunkID] = NewManifest;
+				checkf(PakchunkIndex < FinalChunkManifests.Num(), TEXT("Chunk %i out of range. %i manifests available"), PakchunkIndex, FinalChunkManifests.Num() - 1);
+				checkf(FinalChunkManifests[PakchunkIndex] == nullptr, TEXT("Manifest already exists for chunk %i"), PakchunkIndex);
+				FinalChunkManifests[PakchunkIndex] = NewManifest;
 			}
 		}
 	}
 
 	// generate per-chunk pak list files
-	for (int32 Index = 0; Index < FinalChunkManifests.Num(); ++Index)
+	for (int32 PakchunkIndex = 0; PakchunkIndex < FinalChunkManifests.Num(); ++PakchunkIndex)
 	{
 		// Is this chunk empty?
-		if (!FinalChunkManifests[Index] || FinalChunkManifests[Index]->Num() == 0)
+		if (!FinalChunkManifests[PakchunkIndex] || FinalChunkManifests[PakchunkIndex]->Num() == 0)
 		{
 			continue;
 		}
 
 		int32 FilenameIndex = 0;
 		TArray<FString> ChunkFilenames;
-		FinalChunkManifests[Index]->GenerateValueArray(ChunkFilenames);
+		FinalChunkManifests[PakchunkIndex]->GenerateValueArray(ChunkFilenames);
 		int32 SubChunkIndex = 0;
 		while ( true )
 		{
-			FString PakChunkFilename = FString::Printf(TEXT("pakchunk%d.txt"), Index);
+			FString PakChunkFilename = FString::Printf(TEXT("pakchunk%d.txt"), PakchunkIndex);
 			if ( SubChunkIndex > 0 )
 			{
-				PakChunkFilename = FString::Printf(TEXT("pakchunk%d_s%d.txt"), Index, SubChunkIndex);
+				PakChunkFilename = FString::Printf(TEXT("pakchunk%d_s%d.txt"), PakchunkIndex, SubChunkIndex);
 			}
 
 			FString PakChunkOptions;
@@ -288,7 +292,8 @@ bool FAssetRegistryGenerator::GenerateStreamingInstallManifest(int64 InExtraFlav
 
 			if (bUseAssetManager)
 			{
-				FGuid Guid = UAssetManager::Get().GetChunkEncryptionKeyGuid(Index);
+				// For encryption chunks, PakchunkIndex equals ChunkID
+				FGuid Guid = UAssetManager::Get().GetChunkEncryptionKeyGuid(PakchunkIndex);
 				if (Guid.IsValid())
 				{
 					PakChunkOptions += TEXT(" encryptionkeyguid=") + Guid.ToString();
@@ -296,7 +301,8 @@ bool FAssetRegistryGenerator::GenerateStreamingInstallManifest(int64 InExtraFlav
 					// If this chunk has a seperate unique asset registry, add it to first subchunk's manifest here
 					if (SubChunkIndex == 0)
 					{
-						FName RegistryName = UAssetManager::Get().GetUniqueAssetRegistryName(Index);
+						// For chunks with unique asset registry name, pakchunkIndex should equal chunkid
+						FName RegistryName = UAssetManager::Get().GetUniqueAssetRegistryName(PakchunkIndex);
 						if (RegistryName != NAME_None)
 						{
 							FString AssetRegistryFilename = FString::Printf(TEXT("%s%sAssetRegistry%s.bin"), *InSandboxFile->GetSandboxDirectory(), *InSandboxFile->GetGameSandboxDirectoryName(), *RegistryName.ToString());
@@ -307,7 +313,7 @@ bool FAssetRegistryGenerator::GenerateStreamingInstallManifest(int64 InExtraFlav
 			}
 
 			++SubChunkIndex;
-			FString PakListFilename = FString::Printf(TEXT("%s/%s"), *TmpPackagingDir, *PakChunkFilename, Index);
+			FString PakListFilename = FString::Printf(TEXT("%s/%s"), *TmpPackagingDir, *PakChunkFilename);
 			TUniquePtr<FArchive> PakListFile(IFileManager::Get().CreateFileWriter(*PakListFilename));
 
 			if (!PakListFile)
@@ -357,7 +363,7 @@ bool FAssetRegistryGenerator::GenerateStreamingInstallManifest(int64 InExtraFlav
 			PakChunkListFile->Serialize(TCHAR_TO_ANSI(*PakChunkListLine), PakChunkListLine.Len());
 
 			int32 TargetLayer = 0;
-			FGameDelegates::Get().GetAssignLayerChunkDelegate().ExecuteIfBound(FinalChunkManifests[Index], Platform, Index, TargetLayer);
+			FGameDelegates::Get().GetAssignLayerChunkDelegate().ExecuteIfBound(FinalChunkManifests[PakchunkIndex], Platform, PakchunkIndex, TargetLayer);
 
 			FString LayerString = FString::Printf(TEXT("%d\r\n"), TargetLayer);
 
@@ -529,30 +535,30 @@ bool FAssetRegistryGenerator::SaveManifests(FSandboxPlatformFile* InSandboxFile,
 		}
 
 		// Generate map for the platform abstraction
-		TMultiMap<FString, int32> ChunkMap;	// asset -> ChunkIDs map
-		TSet<int32> ChunkIDsInUse;
+		TMultiMap<FString, int32> PakchunkMap;	// asset -> ChunkIDs map
+		TSet<int32> PakchunkIndicesInUse;
 		const FString PlatformName = TargetPlatform->PlatformName();
 
 		// Collect all unique chunk indices and map all files to their chunks
-		for (int32 ChunkIndex = 0; ChunkIndex < FinalChunkManifests.Num(); ++ChunkIndex)
+		for (int32 PakchunkIndex = 0; PakchunkIndex < FinalChunkManifests.Num(); ++PakchunkIndex)
 		{
-			if (FinalChunkManifests[ChunkIndex] && FinalChunkManifests[ChunkIndex]->Num())
+			if (FinalChunkManifests[PakchunkIndex] && FinalChunkManifests[PakchunkIndex]->Num())
 			{
-				ChunkIDsInUse.Add(ChunkIndex);
-				for (auto& Filename : *FinalChunkManifests[ChunkIndex])
+				PakchunkIndicesInUse.Add(PakchunkIndex);
+				for (auto& Filename : *FinalChunkManifests[PakchunkIndex])
 				{
 					FString PlatFilename = Filename.Value.Replace(TEXT("[Platform]"), *PlatformName);
-					ChunkMap.Add(PlatFilename, ChunkIndex);
+					PakchunkMap.Add(PlatFilename, PakchunkIndex);
 				}
 			}
 		}
 
 		// Sort our chunk IDs and file paths
-		ChunkMap.KeySort(TLess<FString>());
-		ChunkIDsInUse.Sort(TLess<int32>());
+		PakchunkMap.KeySort(TLess<FString>());
+		PakchunkIndicesInUse.Sort(TLess<int32>());
 
 		// Platform abstraction will generate any required platform-specific files for the chunks
-		if (!TargetPlatform->GenerateStreamingInstallManifest(ChunkMap, ChunkIDsInUse))
+		if (!TargetPlatform->GenerateStreamingInstallManifest(PakchunkMap, PakchunkIndicesInUse))
 		{
 			return false;
 		}
@@ -936,9 +942,9 @@ bool FAssetRegistryGenerator::SaveAssetRegistry(const FString& SandboxPath, bool
 
 		// Pass over all chunks and build a mapping of chunk index to asset registry name. All chunks that don't have a unique registry are assigned to the "generic bucket"
 		// which will be written to the master asset registry in chunk 0
-		for (int32 ChunkID = 0; ChunkID < FinalChunkManifests.Num(); ++ChunkID)
+		for (int32 PakchunkIndex = 0; PakchunkIndex < FinalChunkManifests.Num(); ++PakchunkIndex)
 		{
-			FChunkPackageSet* Manifest = FinalChunkManifests[ChunkID];
+			FChunkPackageSet* Manifest = FinalChunkManifests[PakchunkIndex];
 			if (Manifest == nullptr)
 			{
 				continue;
@@ -948,18 +954,19 @@ bool FAssetRegistryGenerator::SaveAssetRegistry(const FString& SandboxPath, bool
 
 			if (bUseAssetManager)
 			{
-				FName RegistryName = UAssetManager::Get().GetUniqueAssetRegistryName(ChunkID);
+				// For chunks with unique asset registry name, pakchunkIndex should equal chunkid
+				FName RegistryName = UAssetManager::Get().GetUniqueAssetRegistryName(PakchunkIndex);
 				if (RegistryName != NAME_None)
 				{
-					ChunkBuckets.FindOrAdd(ChunkID).Add(ChunkID);
-					ChunkBucketNames.FindOrAdd(ChunkID) = RegistryName.ToString();
+					ChunkBuckets.FindOrAdd(PakchunkIndex).Add(PakchunkIndex);
+					ChunkBucketNames.FindOrAdd(PakchunkIndex) = RegistryName.ToString();
 					bAddToGenericBucket = false;
 				}
 			}
 
 			if (bAddToGenericBucket)
 			{
-				ChunkBuckets.FindOrAdd(GenericChunkBucket).Add(ChunkID);
+				ChunkBuckets.FindOrAdd(GenericChunkBucket).Add(PakchunkIndex);
 			}
 		}
 
@@ -1194,7 +1201,7 @@ bool FAssetRegistryGenerator::GenerateAssetChunkInformationCSV(const FString& Ou
 		return A.ObjectPath < B.ObjectPath;
 	});
 
-	for (int32 ChunkID = 0, ChunkNum = FinalChunkManifests.Num(); ChunkID < ChunkNum; ++ChunkID)
+	for (int32 PakchunkIndex = 0; PakchunkIndex < FinalChunkManifests.Num(); ++PakchunkIndex)
 	{
 		FString PerChunkManifestCSV = HeaderText;
 		for (const FAssetData* AssetDataPtr : AssetDataList)
@@ -1204,19 +1211,19 @@ bool FAssetRegistryGenerator::GenerateAssetChunkInformationCSV(const FString& Ou
 			if (AssetData.ChunkIDs.Num() > 0)
 			{
 				const FAssetPackageData* PackageData = State.GetAssetPackageData(AssetData.PackageName);
-				if (AssetData.ChunkIDs.Contains(ChunkID) && PackageData && PackageData->DiskSize >= 0)
+				if (AssetData.ChunkIDs.Contains(PakchunkIndex) && PackageData && PackageData->DiskSize >= 0)
 				{
 					int64 FileSize = PackageData->DiskSize;
 					FString SoftChain;
 					bool bHardChunk = false;
-					if (ChunkID < ChunkManifests.Num())
+					if (PakchunkIndex < ChunkManifests.Num())
 					{
-						bHardChunk = ChunkManifests[ChunkID] && ChunkManifests[ChunkID]->Contains(AssetData.PackageName);
+						bHardChunk = ChunkManifests[PakchunkIndex] && ChunkManifests[PakchunkIndex]->Contains(AssetData.PackageName);
 					
 						if (!bHardChunk)
 						{
 							//
-							SoftChain = GetShortestReferenceChain(AssetData.PackageName, ChunkID);
+							SoftChain = GetShortestReferenceChain(AssetData.PackageName, PakchunkIndex);
 						}
 					}
 					if (SoftChain.IsEmpty())
@@ -1224,7 +1231,7 @@ bool FAssetRegistryGenerator::GenerateAssetChunkInformationCSV(const FString& Ou
 						SoftChain = TEXT("Soft: Possibly Unassigned Asset");
 					}
 
-					TmpString = FString::Printf(TEXT("%d,%s,%s,%s,%lld,"), ChunkID, *AssetData.PackageName.ToString(), *AssetData.AssetClass.ToString(), bHardChunk ? TEXT("Hard") : *SoftChain, FileSize);
+					TmpString = FString::Printf(TEXT("%d,%s,%s,%s,%lld,"), PakchunkIndex, *AssetData.PackageName.ToString(), *AssetData.AssetClass.ToString(), bHardChunk ? TEXT("Hard") : *SoftChain, FileSize);
 					CSVString += TmpString;
 					PerChunkManifestCSV += TmpString;
 					if (AssetData.ChunkIDs.Num() == 1)
@@ -1236,7 +1243,7 @@ bool FAssetRegistryGenerator::GenerateAssetChunkInformationCSV(const FString& Ou
 					{
 						for (const auto& OtherChunk : AssetData.ChunkIDs)
 						{
-							if (OtherChunk != ChunkID)
+							if (OtherChunk != PakchunkIndex)
 							{
 								TmpString = FString::Printf(TEXT("%d "), OtherChunk);
 								CSVString += TmpString;
@@ -1252,7 +1259,7 @@ bool FAssetRegistryGenerator::GenerateAssetChunkInformationCSV(const FString& Ou
 
 		if (bWriteIndividualFiles)
 		{
-			FFileHelper::SaveStringToFile(PerChunkManifestCSV, *FPaths::Combine(*OutputPath, *FString::Printf(TEXT("Chunks%dInfo.csv"), ChunkID)));
+			FFileHelper::SaveStringToFile(PerChunkManifestCSV, *FPaths::Combine(*OutputPath, *FString::Printf(TEXT("Chunks%dInfo.csv"), PakchunkIndex)));
 		}
 	}
 
@@ -1261,15 +1268,18 @@ bool FAssetRegistryGenerator::GenerateAssetChunkInformationCSV(const FString& Ou
 
 void FAssetRegistryGenerator::AddPackageToManifest(const FString& PackageSandboxPath, FName PackageName, int32 ChunkId)
 {
-	while (ChunkId >= ChunkManifests.Num())
+	HighestChunkId = ChunkId > HighestChunkId ? ChunkId : HighestChunkId;
+	int32 PakchunkIndex = GetPakchunkIndex(ChunkId);
+
+	while (PakchunkIndex >= ChunkManifests.Num())
 	{
 		ChunkManifests.Add(nullptr);
 	}
-	if (!ChunkManifests[ChunkId])
+	if (!ChunkManifests[PakchunkIndex])
 	{
-		ChunkManifests[ChunkId] = new FChunkPackageSet();
+		ChunkManifests[PakchunkIndex] = new FChunkPackageSet();
 	}
-	ChunkManifests[ChunkId]->Add(PackageName, PackageSandboxPath);
+	ChunkManifests[PakchunkIndex]->Add(PackageName, PackageSandboxPath);
 	//Safety check, it the package happens to exist in the unassigned list remove it now.
 	UnassignedPackageSet.Remove(PackageName);
 }
@@ -1277,9 +1287,11 @@ void FAssetRegistryGenerator::AddPackageToManifest(const FString& PackageSandbox
 
 void FAssetRegistryGenerator::RemovePackageFromManifest(FName PackageName, int32 ChunkId)
 {
-	if (ChunkManifests[ChunkId])
+	int32 PakchunkIndex = GetPakchunkIndex(ChunkId);
+
+	if (ChunkManifests[PakchunkIndex])
 	{
-		ChunkManifests[ChunkId]->Remove(PackageName);
+		ChunkManifests[PakchunkIndex]->Remove(PackageName);
 	}
 }
 
@@ -1350,9 +1362,9 @@ bool FAssetRegistryGenerator::CheckChunkAssetsAreNotInChild(const FChunkDependen
 	return true;
 }
 
-void FAssetRegistryGenerator::AddPackageAndDependenciesToChunk(FChunkPackageSet* ThisPackageSet, FName InPkgName, const FString& InSandboxFile, int32 ChunkID, FSandboxPlatformFile* SandboxPlatformFile)
+void FAssetRegistryGenerator::AddPackageAndDependenciesToChunk(FChunkPackageSet* ThisPackageSet, FName InPkgName, const FString& InSandboxFile, int32 PakchunkIndex, FSandboxPlatformFile* SandboxPlatformFile)
 {
-	FChunkPackageSet* InitialPackageSetForThisChunk = ChunkManifests.IsValidIndex(ChunkID) ? ChunkManifests[ChunkID] : nullptr;
+	FChunkPackageSet* InitialPackageSetForThisChunk = ChunkManifests.IsValidIndex(PakchunkIndex) ? ChunkManifests[PakchunkIndex] : nullptr;
 
 	//Add this asset
 	ThisPackageSet->Add(InPkgName, InSandboxFile);
@@ -1370,7 +1382,7 @@ void FAssetRegistryGenerator::AddPackageAndDependenciesToChunk(FChunkPackageSet*
 		for (const auto& PkgName : DependentPackageNames)
 		{
 			bool bSkip = false;
-			if (ChunkID != 0 && FinalChunkManifests[0])
+			if (PakchunkIndex != 0 && FinalChunkManifests[0])
 			{
 				// Do not add if this asset was assigned to the 0 chunk. These assets always exist on disk
 				bSkip = FinalChunkManifests[0]->Contains(PkgName);
@@ -1394,7 +1406,7 @@ void FAssetRegistryGenerator::AddPackageAndDependenciesToChunk(FChunkPackageSet*
 						if (UE_LOG_ACTIVE(LogAssetRegistryGenerator, Verbose))
 						{
 							// It was not assigned to this chunk and we're forcing it to be dragged in, let the user known
-							UE_LOG(LogAssetRegistryGenerator, Verbose, TEXT("Adding %s to chunk %i because %s depends on it."), *FilteredPackageName.ToString(), ChunkID, *InPkgName.ToString());
+							UE_LOG(LogAssetRegistryGenerator, Verbose, TEXT("Adding %s to chunk %i because %s depends on it."), *FilteredPackageName.ToString(), PakchunkIndex, *InPkgName.ToString());
 
 							TSet<FName> VisitedPackages;
 							TArray<FName> DependencyChain;
@@ -1425,21 +1437,27 @@ void FAssetRegistryGenerator::FixupPackageDependenciesForChunks(FSandboxPlatform
 	}
 	FinalChunkManifests.Empty();
 
-	for (int32 ChunkID = 0, MaxChunk = ChunkManifests.Num(); ChunkID < MaxChunk; ++ChunkID)
+	for (int32 PakchunkIndex = 0, MaxPakchunk = ChunkManifests.Num(); PakchunkIndex < MaxPakchunk; ++PakchunkIndex)
 	{
 		FinalChunkManifests.Add(nullptr);
-		if (!ChunkManifests[ChunkID])
+		if (!ChunkManifests[PakchunkIndex])
 		{
 			continue;
 		}
-		FinalChunkManifests[ChunkID] = new FChunkPackageSet();
-		for (auto It = ChunkManifests[ChunkID]->CreateConstIterator(); It; ++It)
+		FinalChunkManifests[PakchunkIndex] = new FChunkPackageSet();
+		for (auto It = ChunkManifests[PakchunkIndex]->CreateConstIterator(); It; ++It)
 		{
-			AddPackageAndDependenciesToChunk(FinalChunkManifests[ChunkID], It.Key(), It.Value(), ChunkID, InSandboxFile);
+			AddPackageAndDependenciesToChunk(FinalChunkManifests[PakchunkIndex], It.Key(), It.Value(), PakchunkIndex, InSandboxFile);
 		}
 	}
 
-	const FChunkDependencyTreeNode* ChunkDepGraph = DependencyInfo->GetOrBuildChunkDependencyGraph(ChunkManifests.Num() - 1);
+	FConfigFile PlatformIniFile;
+	FConfigCacheIni::LoadLocalIniFile(PlatformIniFile, TEXT("Engine"), true, *TargetPlatform->IniPlatformName());
+	bool bSkipResolveChunkDependencyGraph = false;
+	PlatformIniFile.GetBool(TEXT("Script/UnrealEd.ChunkDependencyInfo"), TEXT("bSkipResolveChunkDependencyGraph"), bSkipResolveChunkDependencyGraph);
+
+	const FChunkDependencyTreeNode* ChunkDepGraph = DependencyInfo->GetOrBuildChunkDependencyGraph(!bSkipResolveChunkDependencyGraph ? HighestChunkId : 0);
+
 	//Once complete, Add any remaining assets (that are not assigned to a chunk) to the first chunk.
 	if (FinalChunkManifests.Num() == 0)
 	{
@@ -1467,52 +1485,51 @@ void FAssetRegistryGenerator::FixupPackageDependenciesForChunks(FSandboxPlatform
 	//Finally, if the previous step may added any extra packages to the 0 chunk. Pull them out of other chunks and save space
 	ResolveChunkDependencyGraph(*ChunkDepGraph, FChunkPackageSet(), PackagesRemovedFromChunks);
 
-	for (int32 i = 0; i < ChunkManifests.Num(); ++i)
+	for (int32 PakchunkIndex = 0; PakchunkIndex < ChunkManifests.Num(); ++PakchunkIndex)
 	{
 		if (!bUseAssetManager)
 		{
-			FName CollectionName(*FString::Printf(TEXT("PackagesRemovedFromChunk%i"), i));
+			FName CollectionName(*FString::Printf(TEXT("PackagesRemovedFromChunk%i"), PakchunkIndex));
 			if (CreateOrEmptyCollection(CollectionName))
 			{
-				WriteCollection(CollectionName, PackagesRemovedFromChunks[i]);
+				WriteCollection(CollectionName, PackagesRemovedFromChunks[PakchunkIndex]);
 			}
 		}
 	}
 
-	for (int32 ChunkID = 0, MaxChunk = ChunkManifests.Num(); ChunkID < MaxChunk; ++ChunkID)
+	for (int32 PakchunkIndex = 0, MaxPakchunk = ChunkManifests.Num(); PakchunkIndex < MaxPakchunk; ++PakchunkIndex)
 	{
-		const int32 ChunkManifestNum = ChunkManifests[ChunkID] ? ChunkManifests[ChunkID]->Num() : 0;
-		const int32 FinalChunkManifestNum = FinalChunkManifests[ChunkID] ? FinalChunkManifests[ChunkID]->Num() : 0;
-		UE_LOG(LogAssetRegistryGenerator, Log, TEXT("Chunk: %i, Started with %i packages, Final after dependency resolve: %i"), ChunkID, ChunkManifestNum, FinalChunkManifestNum);
+		const int32 ChunkManifestNum = ChunkManifests[PakchunkIndex] ? ChunkManifests[PakchunkIndex]->Num() : 0;
+		const int32 FinalChunkManifestNum = FinalChunkManifests[PakchunkIndex] ? FinalChunkManifests[PakchunkIndex]->Num() : 0;
+		UE_LOG(LogAssetRegistryGenerator, Log, TEXT("Chunk: %i, Started with %i packages, Final after dependency resolve: %i"), PakchunkIndex, ChunkManifestNum, FinalChunkManifestNum);
 	}
 	
 	// Fix up the asset registry to reflect this chunk layout
-	for (int32 ChunkID = 0, MaxChunk = FinalChunkManifests.Num(); ChunkID < MaxChunk; ++ChunkID)
+	for (int32 PakchunkIndex = 0 ; PakchunkIndex < FinalChunkManifests.Num(); ++PakchunkIndex)
 	{
-		if (!FinalChunkManifests[ChunkID])
+		if (PakchunkIndex >= FinalChunkManifests.Num() || !FinalChunkManifests[PakchunkIndex])
 		{
 			continue;
 		}
-		for (const TPair<FName, FString>& Asset : *FinalChunkManifests[ChunkID])
+		for (const TPair<FName, FString>& Asset : *FinalChunkManifests[PakchunkIndex])
 		{
 			const TArray<const FAssetData*> AssetIndexArray = State.GetAssetsByPackageName(Asset.Key);
 			for (const FAssetData* AssetData : AssetIndexArray)
 			{
 				// Chunk Ids are safe to modify in place
-				const_cast<FAssetData*>(AssetData)->ChunkIDs.AddUnique(ChunkID);
+				const_cast<FAssetData*>(AssetData)->ChunkIDs.AddUnique(PakchunkIndex);
 			}
 		}
 	}
 }
 
-
-void FAssetRegistryGenerator::FindShortestReferenceChain(TArray<FReferencePair> PackageNames, int32 ChunkID, uint32& OutParentIndex, FString& OutChainPath)
+void FAssetRegistryGenerator::FindShortestReferenceChain(TArray<FReferencePair> PackageNames, int32 PakchunkIndex, uint32& OutParentIndex, FString& OutChainPath)
 {
 	TArray<FReferencePair> ReferencesToCheck;
 	uint32 Index = 0;
 	for (const auto& Pkg : PackageNames)
 	{
-		if (ChunkManifests[ChunkID] && ChunkManifests[ChunkID]->Contains(Pkg.PackageName))
+		if (ChunkManifests[PakchunkIndex] && ChunkManifests[PakchunkIndex]->Contains(Pkg.PackageName))
 		{
 			OutChainPath += TEXT("Soft: ");
 			OutChainPath += Pkg.PackageName.ToString();
@@ -1536,7 +1553,7 @@ void FAssetRegistryGenerator::FindShortestReferenceChain(TArray<FReferencePair>
 	if (ReferencesToCheck.Num() > 0)
 	{
 		uint32 ParentIndex = INDEX_NONE;
-		FindShortestReferenceChain(ReferencesToCheck, ChunkID, ParentIndex, OutChainPath);
+		FindShortestReferenceChain(ReferencesToCheck, PakchunkIndex, ParentIndex, OutChainPath);
 
 		if (ParentIndex < (uint32)PackageNames.Num())
 		{
@@ -1554,7 +1571,7 @@ void FAssetRegistryGenerator::FindShortestReferenceChain(TArray<FReferencePair>
 	}
 }
 
-FString FAssetRegistryGenerator::GetShortestReferenceChain(FName PackageName, int32 ChunkID)
+FString FAssetRegistryGenerator::GetShortestReferenceChain(FName PackageName, int32 PakchunkIndex)
 {
 	FString StringChain;
 	TArray<FReferencePair> ReferencesToCheck;
@@ -1562,7 +1579,7 @@ FString FAssetRegistryGenerator::GetShortestReferenceChain(FName PackageName, in
 	ReferencesToCheck.Add(FReferencePair(PackageName, 0));
 	InspectedNames.Empty();
 	InspectedNames.Add(PackageName);
-	FindShortestReferenceChain(ReferencesToCheck, ChunkID, ParentIndex, StringChain);
+	FindShortestReferenceChain(ReferencesToCheck, PakchunkIndex, ParentIndex, StringChain);
 
 	return StringChain;
 }
@@ -1614,4 +1631,38 @@ void FAssetRegistryGenerator::WriteCollection(FName CollectionName, const TArray
 	}
 }
 
+int32 FAssetRegistryGenerator::GetPakchunkIndex(int32 ChunkId)
+{
+	if (ChunkIdPakchunkIndexMapping.Contains(ChunkId))
+	{
+		int32 NewChunkId = ChunkIdPakchunkIndexMapping[ChunkId];
+		check(NewChunkId >= 0);
+		return NewChunkId;
+	}
+
+	return ChunkId;
+}
+
+void FAssetRegistryGenerator::InitializeChunkIdPakchunkIndexMapping()
+{
+	FConfigFile PlatformIniFile;
+	FConfigCacheIni::LoadLocalIniFile(PlatformIniFile, TEXT("Game"), true, *TargetPlatform->IniPlatformName());
+	TArray<FString> ChunkMapping;
+	PlatformIniFile.GetArray(TEXT("/Script/UnrealEd.ProjectPackagingSettings"), TEXT("ChunkIdPakchunkIndexMapping"), ChunkMapping);
+
+	FPlatformMisc::ParseChunkIdPakchunkIndexMapping(ChunkMapping, ChunkIdPakchunkIndexMapping);
+
+	// Validate ChunkIdPakchunkIndexMapping
+	TArray<int32> AllChunkIDs;
+	ChunkIdPakchunkIndexMapping.GetKeys(AllChunkIDs);
+	for (int32 ChunkID : AllChunkIDs)
+	{
+		if(UAssetManager::Get().GetChunkEncryptionKeyGuid(ChunkID).IsValid()
+			|| UAssetManager::Get().GetUniqueAssetRegistryName(ChunkID) != NAME_None)
+		{
+			UE_LOG(LogAssetRegistryGenerator, Error, TEXT("Chunks with encryption key guid or unique assetregistry name (Chunk %d) can not be mapped with ChunkIdPakchunkIndexMapping.  Mapping is removed."), ChunkID);
+			ChunkIdPakchunkIndexMapping.Remove(ChunkID);
+		}
+	}
+}
 #undef LOCTEXT_NAMESPACE

Engine/Source/Editor/UnrealEd/Public/Commandlets/AssetRegistryGenerator.h

@@ -176,6 +176,8 @@ private:
 	bool bGenerateChunks;
 	/** True if we should use the AssetManager, false to use the deprecated path */
 	bool bUseAssetManager;
+	/** Highest chunk id, being used for geneating dependency tree */
+	int32 HighestChunkId;
 	/** Array of Maps with chunks<->packages assignments */
 	TArray<FChunkPackageSet*>		ChunkManifests;
 	/** Map of packages that has not been assigned to chunks */
@@ -192,6 +194,9 @@ private:
 	/** Dependency type to follow when adding package dependencies to chunks.*/
 	EAssetRegistryDependencyType::Type DependencyType;
 
+	/** Mapping from chunk id to pakchunk file index. If not defined, Pakchunk index will be the same as chunk id by default */
+	TMap<int32, int32> ChunkIdPakchunkIndexMapping;
+
 	struct FReferencePair
 	{
 		FReferencePair() {}
@@ -228,6 +233,14 @@ private:
 	*/
 	void RemovePackageFromManifest(FName PackageName, int32 ChunkId);
 
+	/**
+	* Get pakchunk file index from ChunkID
+	*
+	* @param ChunkID
+	* @return Index of target pakchunk file
+	*/
+	int32 GetPakchunkIndex(int32 ChunkId);
+
 	/**
 	 * Walks the dependency graph of assets and assigns packages to correct chunks.
 	 * 
@@ -240,7 +253,7 @@ private:
 	 */
 	void InjectEncryptionData(FAssetRegistryState& TargetState);
 
-	void AddPackageAndDependenciesToChunk(FChunkPackageSet* ThisPackageSet, FName InPkgName, const FString& InSandboxFile, int32 ChunkID, FSandboxPlatformFile* SandboxPlatformFile);
+	void AddPackageAndDependenciesToChunk(FChunkPackageSet* ThisPackageSet, FName InPkgName, const FString& InSandboxFile, int32 PakchunkIndex, FSandboxPlatformFile* SandboxPlatformFile);
 
 	/**
 	 * Returns the path of the temporary packaging directory for the specified platform.
@@ -333,4 +346,6 @@ private:
 	/** Helper function to fill a given collection with a set of packages */
 	void WriteCollection(FName CollectionName, const TArray<FName>& PackageNames);
 	
+	/** Initialize ChunkIdPakchunkIndexMapping and PakchunkIndexChunkIdMapping. */
+	void InitializeChunkIdPakchunkIndexMapping();
 };

Engine/Source/Runtime/Core/Private/GenericPlatform/GenericPlatformMisc.cpp

@@ -1322,3 +1322,40 @@ FString FGenericPlatformMisc::LoadTextFileFromPlatformPackage(const FString& Rel
 	FFileHelper::LoadFileToString(Result, *Path);
 	return Result;
 }
+
+void FGenericPlatformMisc::ParseChunkIdPakchunkIndexMapping(TArray<FString> ChunkIndexMappingData, TMap<int32, int32>& OutMapping)
+{
+	OutMapping.Empty();
+
+	const TCHAR* PropertyOldChunkIndex = TEXT("Old");
+	const TCHAR* PropertyNewChunkIndex = TEXT("New");
+	for (const FString& Entry : ChunkIndexMappingData)
+	{
+		// Remove parentheses
+		FString EntryContent = Entry.Mid(1, Entry.Len() - 2);
+		TArray<FString> EntryProperties;
+		EntryContent.ParseIntoArray(EntryProperties, TEXT(","));
+
+		int32 ChunkId = -1;
+		int32 PakchunkIndex = -1;
+		for (const FString EntryProperty : EntryProperties)
+		{
+			TArray<FString> MappingKeyValue;
+			EntryProperty.ParseIntoArray(MappingKeyValue, TEXT("="));
+
+			if (MappingKeyValue[0].Equals(PropertyOldChunkIndex, ESearchCase::IgnoreCase))
+			{
+				LexTryParseString(ChunkId, *MappingKeyValue[1]);
+			}
+			else if (MappingKeyValue[0].Equals(PropertyNewChunkIndex, ESearchCase::IgnoreCase))
+			{
+				LexTryParseString(PakchunkIndex, *MappingKeyValue[1]);
+			}
+		}
+
+		if (ChunkId != -1 && PakchunkIndex != -1 && ChunkId != PakchunkIndex && !OutMapping.Contains(ChunkId))
+		{
+			OutMapping.Add(ChunkId, PakchunkIndex);
+		}
+	}
+}

Engine/Source/Runtime/Core/Public/GenericPlatform/GenericPlatformMisc.h

@@ -1279,6 +1279,8 @@ public:
 	*/
 	static FString LoadTextFileFromPlatformPackage(const FString& RelativePath);
 
+	static void ParseChunkIdPakchunkIndexMapping(TArray<FString> ChunkIndexRedirects, TMap<int32, int32>& OutMapping);
+
 #if !UE_BUILD_SHIPPING
 	/** 
 	 * Returns any platform specific warning messages we want printed on screen