Files
UnrealEngineUWP/Engine/Source/Programs/AutomationTool/Scripts/BuildCookRun.Automation.cs
Dan Thompson 3a6321167b Packaging Reference Chunk Database - "Lossless Patch Preventer"
Can now provide reference iostore containers to reuse compressed chunks from. If a match is found on *the decompressed data*, instead of recompressing the blocks, they are read off of disk. This allows tweaks of the compressor algorithm without introducing changes as the runtime still sees the exact same data. Additionally this allows for fairly dramatic staging speedups as nvme speeds are significantly faster than high effort compressions. This is distinct from the DDC compression because:
1) DDC compression ties in the compressor version/method
2) We are explicitly interested in chunks that are deployed to end users, not merely cached for speed.

To facilitate this, several changes were made to IoStore:

FIoStoreReader now directly reads from IFileHandles* instead of routing through the GenericPlatformFile async read system, as that system is sensitive to build #defines and can result in constant file opens under load (indeed, for anything not a .pak file, every read is an open/close).

Cold file cache read speed improvements from ~140MB/s to ~1 GB/s. Hot is more.

Additionally:
    FIoStoreReader switched to UE::Tasks from taskgraph for tasks in order to facilitate task retraction during waits as the previous ReadAsync call was trivial to deadlock when called from worker threads due to its use of TFuture<>.
    FIoStoreReader::ReadCompressed now returns the compressed blocks as they were on disk - padded to AES encryption block size.

#rb fabian.giesen
#rb jeff.roberts
#preflight 627586dcf77c9c2b543d4d8b

[CL 20086673 by Dan Thompson in ue5-main branch]
2022-05-06 18:22:44 -04:00

309 lines
9.0 KiB
C#

// Copyright Epic Games, Inc. All Rights Reserved.
using System;
using System.Collections.Generic;
using System.IO;
using System.Threading;
using System.Reflection;
using AutomationTool;
using AutomationScripts;
using UnrealBuildTool;
using EpicGames.Core;
[Help(@"Builds/Cooks/Runs a project.
For non-uprojects project targets are discovered by compiling target rule files found in the project folder.
If -map is not specified, the command looks for DefaultMap entry in the project's DefaultEngine.ini and if not found, in BaseEngine.ini.
If no DefaultMap can be found, the command falls back to /Engine/Maps/Entry.")]
[Help("project=Path", @"Project path (required), i.e: -project=QAGame, -project=Samples\BlackJack\BlackJack.uproject, -project=D:\Projects\MyProject.uproject")]
[Help("destsample", "Destination Sample name")]
[Help("foreigndest", "Foreign Destination")]
[Help(typeof(ProjectParams))]
[Help(typeof(UnrealBuild))]
[Help(typeof(CodeSign))]
public class BuildCookRun : BuildCommand
{
public override void ExecuteBuild()
{
var StartTime = DateTime.UtcNow;
// these need to be done first
var bForeign = ParseParam("foreign");
var bForeignCode = ParseParam("foreigncode");
if (bForeign)
{
MakeForeignSample();
}
else if (bForeignCode)
{
MakeForeignCodeSample();
}
var Params = SetupParams();
DoBuildCookRun(Params);
LogInformation("BuildCookRun time: {0:0.00} s", (DateTime.UtcNow - StartTime).TotalMilliseconds / 1000);
}
protected ProjectParams SetupParams()
{
LogInformation("Setting up ProjectParams for {0}", ProjectPath);
var Params = new ProjectParams
(
Command: this,
// Shared
RawProjectPath: ProjectPath
);
var DirectoriesToCook = ParseParamValue("cookdir");
if (!String.IsNullOrEmpty(DirectoriesToCook))
{
Params.DirectoriesToCook = new ParamList<string>(DirectoriesToCook.Split('+'));
}
var DDCGraph = ParseParamValue("ddc");
if (!String.IsNullOrEmpty(DDCGraph))
{
Params.DDCGraph = DDCGraph;
}
var InternationalizationPreset = ParseParamValue("i18npreset");
if (!String.IsNullOrEmpty(InternationalizationPreset))
{
Params.InternationalizationPreset = InternationalizationPreset;
}
var CulturesToCook = ParseParamValue("cookcultures");
if (!String.IsNullOrEmpty(CulturesToCook))
{
Params.CulturesToCook = new ParamList<string>(CulturesToCook.Split('+'));
}
var ReferenceContainerGlobalFileName = ParseParamValue("ReferenceContainerGlobalFileName");
if (!String.IsNullOrEmpty(ReferenceContainerGlobalFileName))
{
Params.ReferenceContainerGlobalFileName = ReferenceContainerGlobalFileName;
}
var ReferenceContainerCryptoKeys = ParseParamValue("ReferenceContainerCryptoKeys");
if (!String.IsNullOrEmpty(ReferenceContainerCryptoKeys))
{
Params.ReferenceContainerCryptoKeys = ReferenceContainerCryptoKeys;
}
if (Params.DedicatedServer)
{
foreach (var ServerPlatformInstance in Params.ServerTargetPlatformInstances)
{
ServerPlatformInstance.PlatformSetupParams(ref Params);
}
}
else
{
foreach (var ClientPlatformInstance in Params.ClientTargetPlatformInstances)
{
ClientPlatformInstance.PlatformSetupParams(ref Params);
}
}
Params.ValidateAndLog();
return Params;
}
/// <summary>
/// In case the command line specified multiple map names with a '+', selects the first map from the list.
/// </summary>
/// <param name="Maps">Map(s) specified in the commandline.</param>
/// <returns>First map or an empty string.</returns>
private static string GetFirstMap(string Maps)
{
string Map = String.Empty;
if (!String.IsNullOrEmpty(Maps))
{
var AllMaps = Maps.Split(new char[] { '+' }, StringSplitOptions.RemoveEmptyEntries);
if (!IsNullOrEmpty(AllMaps))
{
Map = AllMaps[0];
}
}
return Map;
}
private string GetTargetName(Type TargetRulesType)
{
const string TargetPostfix = "Target";
var Name = TargetRulesType.Name;
if (Name.EndsWith(TargetPostfix, StringComparison.InvariantCultureIgnoreCase))
{
Name = Name.Substring(0, Name.Length - TargetPostfix.Length);
}
return Name;
}
private string GetDefaultMap(ProjectParams Params)
{
const string EngineEntryMap = "/Engine/Maps/Entry";
LogInformation("Trying to find DefaultMap in ini files");
string DefaultMap = null;
var ProjectFolder = GetDirectoryName(Params.RawProjectPath.FullName);
var DefaultGameEngineConfig = CombinePaths(ProjectFolder, "Config", "DefaultEngine.ini");
if (FileExists(DefaultGameEngineConfig))
{
LogInformation("Looking for DefaultMap in {0}", DefaultGameEngineConfig);
DefaultMap = GetDefaultMapFromIni(DefaultGameEngineConfig, Params.DedicatedServer);
if (DefaultMap == null && Params.DedicatedServer)
{
DefaultMap = GetDefaultMapFromIni(DefaultGameEngineConfig, false);
}
}
else
{
var BaseEngineConfig = CombinePaths(CmdEnv.LocalRoot, "Config", "BaseEngine.ini");
if (FileExists(BaseEngineConfig))
{
LogInformation("Looking for DefaultMap in {0}", BaseEngineConfig);
DefaultMap = GetDefaultMapFromIni(BaseEngineConfig, Params.DedicatedServer);
if (DefaultMap == null && Params.DedicatedServer)
{
DefaultMap = GetDefaultMapFromIni(BaseEngineConfig, false);
}
}
}
// We check for null here becase null == not found
if (DefaultMap == null)
{
LogInformation("No DefaultMap found, assuming: {0}", EngineEntryMap);
DefaultMap = EngineEntryMap;
}
else
{
LogInformation("Found DefaultMap={0}", DefaultMap);
}
return DefaultMap;
}
private string GetDefaultMapFromIni(string IniFilename, bool DedicatedServer)
{
var IniLines = ReadAllLines(IniFilename);
string DefaultMap = null;
string ConfigKeyStr = "GameDefaultMap";
if (DedicatedServer)
{
ConfigKeyStr = "ServerDefaultMap";
}
foreach (var Line in IniLines)
{
if (Line.StartsWith(ConfigKeyStr, StringComparison.InvariantCultureIgnoreCase))
{
var DefaultMapPair = Line.Split('=');
DefaultMap = DefaultMapPair[1].Trim();
}
if (DefaultMap != null)
{
break;
}
}
return DefaultMap;
}
protected void DoBuildCookRun(ProjectParams Params)
{
int WorkingCL = -1;
if (P4Enabled && GlobalCommandLine.Submit && AllowSubmit)
{
WorkingCL = P4.CreateChange(P4Env.Client, String.Format("{0} build from changelist {1}", Params.ShortProjectName, P4Env.Changelist));
}
Project.Build(this, Params, WorkingCL, ProjectBuildTargets.All);
Project.Cook(Params);
Project.CopyBuildToStagingDirectory(Params);
Project.Package(Params, WorkingCL);
Project.Archive(Params);
Project.Deploy(Params);
PrintRunTime();
Project.Run(Params);
Project.GetFile(Params);
// Check everything in!
if (WorkingCL != -1)
{
int SubmittedCL;
P4.Submit(WorkingCL, out SubmittedCL, true, true);
}
}
private void MakeForeignSample()
{
string Sample = "BlankProject";
var DestSample = ParseParamValue("DestSample", "CopiedBlankProject");
var Src = CombinePaths(CmdEnv.LocalRoot, "Samples", "SampleGames", Sample);
if (!DirectoryExists(Src))
{
throw new AutomationException("Can't find source directory to make foreign sample {0}.", Src);
}
var Dest = ParseParamValue("ForeignDest", CombinePaths(@"C:\testue\foreign\", DestSample + "_ _Dir"));
LogInformation("Make a foreign sample {0} -> {1}", Src, Dest);
CloneDirectory(Src, Dest);
DeleteDirectory_NoExceptions(CombinePaths(Dest, "Intermediate"));
DeleteDirectory_NoExceptions(CombinePaths(Dest, "Saved"));
RenameFile(CombinePaths(Dest, Sample + ".uproject"), CombinePaths(Dest, DestSample + ".uproject"));
var IniFile = CombinePaths(Dest, "Config", "DefaultEngine.ini");
var Ini = new VersionFileUpdater(new FileReference(IniFile));
Ini.ReplaceLine("GameName=", DestSample);
Ini.Commit();
}
private void MakeForeignCodeSample()
{
string Sample = "PlatformerGame";
string DestSample = "PlatformerGame";
var Src = CombinePaths(CmdEnv.LocalRoot, Sample);
if (!DirectoryExists(Src))
{
throw new AutomationException("Can't find source directory to make foreign sample {0}.", Src);
}
var Dest = ParseParamValue("ForeignDest", CombinePaths(@"C:\testue\foreign\", DestSample + "_ _Dir"));
LogInformation("Make a foreign sample {0} -> {1}", Src, Dest);
CloneDirectory(Src, Dest);
DeleteDirectory_NoExceptions(CombinePaths(Dest, "Intermediate"));
DeleteDirectory_NoExceptions(CombinePaths(Dest, "Saved"));
DeleteDirectory_NoExceptions(CombinePaths(Dest, "Plugins", "FootIK", "Intermediate"));
//RenameFile(CombinePaths(Dest, Sample + ".uproject"), CombinePaths(Dest, DestSample + ".uproject"));
var IniFile = CombinePaths(Dest, "Config", "DefaultEngine.ini");
var Ini = new VersionFileUpdater(new FileReference(IniFile));
Ini.ReplaceLine("GameName=", DestSample);
Ini.Commit();
}
private FileReference ProjectFullPath;
public virtual FileReference ProjectPath
{
get
{
if (ProjectFullPath == null)
{
ProjectFullPath = ParseProjectParam();
if (ProjectFullPath == null)
{
throw new AutomationException("No project file specified. Use -project=<project>.");
}
}
return ProjectFullPath;
}
}
}