// Copyright 1998-2015 Epic Games, Inc. All Rights Reserved.
using System;
using System.Collections.Generic;
using System.IO;
using System.Threading;
using System.Reflection;
using AutomationTool;
using UnrealBuildTool;
public enum StagedFileType
{
UFS,
NonUFS,
DebugNonUFS
}
public class DeploymentContext //: ProjectParams
{
///
/// Full path where the project exists (For uprojects this should include the uproj filename, otherwise just project folder)
///
public string RawProjectPath;
///
/// true if we should stage crash reporter
///
public bool bStageCrashReporter;
///
/// CookPlatform, where to get the cooked data from and use for sandboxes
///
public string CookPlatform;
///
/// FinalCookPlatform, directory to stage and archive the final result to
///
public string FinalCookPlatform;
///
/// Source platform to get the cooked data from
///
public Platform CookSourcePlatform;
///
/// Target platform used for sandboxes and stage directory names
///
public Platform StageTargetPlatform;
///
/// Configurations to stage. Used to determine which ThirdParty configurations to copy.
///
public List StageTargetConfigurations;
///
/// Receipts for the build targets that should be staged
///
public List StageTargetReceipts;
///
/// this is the root directory that contains the engine: d:\a\UE4\
///
public string LocalRoot;
///
/// this is the directory that contains the "game": d:\a\UE4\ShooterGame
///
public string ProjectRoot;
///
/// raw name used for platform subdirectories Win32
///
public string PlatformDir;
///
/// this is the directory that contains the "game", staged: d:\stagedir\WindowsNoEditor\ShooterGame
///
public string StageProjectRoot;
///
/// Directory to put all of the files in: d:\stagedir\WindowsNoEditor
///
public string StageDirectory;
///
/// The relative staged project root, what would be tacked on to StageDirectory
///
public string RelativeProjectRootForStage;
///
/// The relative source project root, this will be the short project name for foreign projects, otherwise it is ProjectRoot minus the LocalRoot prefix
///
public string SourceRelativeProjectRoot;
///
/// The relative staged project root, used inside the pak file list (different slash convention than above)
///
public string RelativeProjectRootForUnrealPak;
///
/// This is what you use to test the engine which uproject you want. Many cases.
///
public string ProjectArgForCommandLines;
///
/// This is the root that the cook source platform would run from
///
public string CookSourceRuntimeRootDir;
///
/// This is the root that we are going to run from. Many cases.
///
public string RuntimeRootDir;
///
/// This is the project root that we are going to run from. Many cases.
///
public string RuntimeProjectRootDir;
///
/// This is the executable we are going to run and stage. Filled in by the platform abstraction.
///
public string RuntimeExecutable = "";
///
/// List of executables we are going to stage
///
public List StageExecutables;
///
/// Probably going away, used to construct ProjectArgForCommandLines in the case that we are running staged
///
public string UProjectCommandLineArgInternalRoot = "../../../";
///
/// Probably going away, used to construct the pak file list
///
public string PakFileInternalRoot = "../../../";
///
/// Probably going away, and currently unused, this would be used to build the list of files for the UFS server
///
public string UnrealFileServerInternalRoot = "../../../";
///
/// After staging, this is a map from source file to relative file in the stage
/// These file are binaries, etc and can't go into a pak file
///
public Dictionary NonUFSStagingFiles = new Dictionary();
///
/// After staging, this is a map from source file to relative file in the stage
/// These file are debug, and can't go into a pak file
///
public Dictionary NonUFSStagingFilesDebug = new Dictionary();
///
/// After staging, this is a map from source file to relative file in the stage
/// These file are content, and can go into a pak file
///
public Dictionary UFSStagingFiles = new Dictionary();
///
/// List of files to be archived
///
public Dictionary ArchivedFiles = new Dictionary();
///
/// Directory to archive all of the files in: d:\archivedir\WindowsNoEditor
///
public string ArchiveDirectory;
///
/// Filename for the manifest of file changes for iterative deployment.
///
static public readonly string UFSDeployDeltaFileName = "Manifest_DeltaUFSFiles.txt";
static public readonly string NonUFSDeployDeltaFileName = "Manifest_DeltaNonUFSFiles.txt";
///
/// Filename for the manifest of files currently deployed on a device.
///
static public readonly string UFSDeployedManifestFileName = "Manifest_UFSFiles.txt";
static public readonly string NonUFSDeployedManifestFileName = "Manifest_NonUFSFiles.txt";
///
/// The client connects to dedicated server to get data
///
public bool DedicatedServer;
///
/// True if this build is staged
///
public bool Stage;
///
/// True if this build is archived
///
public bool Archive;
///
/// True if this project has code
///
public bool IsCodeBasedProject;
///
/// Project name (name of the uproject file without extension or directory name where the project is localed)
///
public string ShortProjectName;
///
/// If true, multiple platforms are being merged together - some behavior needs to change (but not much)
///
public bool bIsCombiningMultiplePlatforms = false;
public DeploymentContext(
string RawProjectPathOrName,
string InLocalRoot,
string BaseStageDirectory,
string BaseArchiveDirectory,
string CookFlavor,
Platform InSourcePlatform,
Platform InTargetPlatform,
List InTargetConfigurations,
IEnumerable InStageTargetReceipts,
List InStageExecutables,
bool InServer,
bool InCooked,
bool InStageCrashReporter,
bool InStage,
bool InCookOnTheFly,
bool InArchive,
bool InProgram,
bool bHasDedicatedServerAndClient
)
{
bStageCrashReporter = InStageCrashReporter;
RawProjectPath = RawProjectPathOrName;
DedicatedServer = InServer;
LocalRoot = CommandUtils.CombinePaths(InLocalRoot);
CookSourcePlatform = InSourcePlatform;
StageTargetPlatform = InTargetPlatform;
StageTargetConfigurations = new List(InTargetConfigurations);
StageTargetReceipts = new List(InStageTargetReceipts);
StageExecutables = InStageExecutables;
IsCodeBasedProject = ProjectUtils.IsCodeBasedUProjectFile(RawProjectPath);
ShortProjectName = ProjectUtils.GetShortProjectName(RawProjectPath);
Stage = InStage;
Archive = InArchive;
if (CookSourcePlatform != null && InCooked)
{
CookPlatform = CookSourcePlatform.GetCookPlatform(DedicatedServer, bHasDedicatedServerAndClient, CookFlavor);
}
else if (CookSourcePlatform != null && InProgram)
{
CookPlatform = CookSourcePlatform.GetCookPlatform(false, false, "");
}
else
{
CookPlatform = "";
}
if (StageTargetPlatform != null && InCooked)
{
FinalCookPlatform = StageTargetPlatform.GetCookPlatform(DedicatedServer, bHasDedicatedServerAndClient, CookFlavor);
}
else if (StageTargetPlatform != null && InProgram)
{
FinalCookPlatform = StageTargetPlatform.GetCookPlatform(false, false, "");
}
else
{
FinalCookPlatform = "";
}
PlatformDir = StageTargetPlatform.PlatformType.ToString();
StageDirectory = CommandUtils.CombinePaths(BaseStageDirectory, FinalCookPlatform);
ArchiveDirectory = CommandUtils.CombinePaths(BaseArchiveDirectory, FinalCookPlatform);
if (!CommandUtils.FileExists(RawProjectPath))
{
throw new AutomationException("Can't find uproject file {0}.", RawProjectPathOrName);
}
ProjectRoot = CommandUtils.CombinePaths(CommandUtils.GetDirectoryName(Path.GetFullPath(RawProjectPath)));
if (!CommandUtils.DirectoryExists(ProjectRoot))
{
throw new AutomationException("Project Directory {0} doesn't exist.", ProjectRoot);
}
RelativeProjectRootForStage = ShortProjectName;
ProjectArgForCommandLines = CommandUtils.MakePathSafeToUseWithCommandLine(RawProjectPath);
CookSourceRuntimeRootDir = RuntimeRootDir = LocalRoot;
RuntimeProjectRootDir = ProjectRoot;
RelativeProjectRootForUnrealPak = CommandUtils.CombinePaths(RelativeProjectRootForStage).Replace("\\", "/");
if (RelativeProjectRootForUnrealPak.StartsWith("/"))
{
RelativeProjectRootForUnrealPak = RelativeProjectRootForUnrealPak.Substring(1);
RelativeProjectRootForStage = RelativeProjectRootForStage.Substring(1);
}
SourceRelativeProjectRoot = RelativeProjectRootForStage; // for foreign projects this doesn't make much sense, but it turns into a noop on staging files
if (ProjectRoot.StartsWith(LocalRoot, StringComparison.InvariantCultureIgnoreCase))
{
SourceRelativeProjectRoot = ProjectRoot.Substring(LocalRoot.Length);
}
if (SourceRelativeProjectRoot.StartsWith("/") || SourceRelativeProjectRoot.StartsWith("\\"))
{
SourceRelativeProjectRoot = SourceRelativeProjectRoot.Substring(1);
}
if (Stage)
{
CommandUtils.CreateDirectory(StageDirectory);
StageProjectRoot = CommandUtils.CombinePaths(StageDirectory, RelativeProjectRootForStage);
RuntimeRootDir = StageDirectory;
CookSourceRuntimeRootDir = CommandUtils.CombinePaths(BaseStageDirectory, CookPlatform);
RuntimeProjectRootDir = StageProjectRoot;
ProjectArgForCommandLines = CommandUtils.MakePathSafeToUseWithCommandLine(UProjectCommandLineArgInternalRoot + RelativeProjectRootForStage + "/" + ShortProjectName + ".uproject");
}
if (Archive)
{
CommandUtils.CreateDirectory(ArchiveDirectory);
}
ProjectArgForCommandLines = ProjectArgForCommandLines.Replace("\\", "/");
}
public void StageFile(StagedFileType FileType, string InputPath, string OutputPath = null, bool bRemap = true)
{
InputPath = InputPath.Replace(Path.AltDirectorySeparatorChar, Path.DirectorySeparatorChar);
if(OutputPath == null)
{
if(InputPath.StartsWith(ProjectRoot, StringComparison.InvariantCultureIgnoreCase))
{
OutputPath = CommandUtils.CombinePaths(RelativeProjectRootForStage, InputPath.Substring(ProjectRoot.Length).TrimStart('/', '\\'));
}
else if (InputPath.StartsWith(LocalRoot, StringComparison.InvariantCultureIgnoreCase))
{
OutputPath = CommandUtils.CombinePaths(InputPath.Substring(LocalRoot.Length).TrimStart('/', '\\'));
}
else
{
throw new AutomationException("Can't deploy {0} because it doesn't start with {1} or {2}", InputPath, ProjectRoot, LocalRoot);
}
}
if(bRemap)
{
OutputPath = StageTargetPlatform.Remap(OutputPath);
}
if (FileType == StagedFileType.UFS)
{
AddUniqueStagingFile(UFSStagingFiles, InputPath, OutputPath);
}
else if (FileType == StagedFileType.NonUFS)
{
AddUniqueStagingFile(NonUFSStagingFiles, InputPath, OutputPath);
}
else if (FileType == StagedFileType.DebugNonUFS)
{
AddUniqueStagingFile(NonUFSStagingFilesDebug, InputPath, OutputPath);
}
}
public void StageFilesInReceipt(BuildReceipt Receipt)
{
// Stage all the build products needed at runtime
foreach(BuildProduct BuildProduct in Receipt.BuildProducts)
{
// allow missing files if needed
if (Receipt.bRequireDependenciesToExist == false && File.Exists(BuildProduct.Path) == false)
{
continue;
}
if(BuildProduct.Type == BuildProductType.Executable || BuildProduct.Type == BuildProductType.DynamicLibrary || BuildProduct.Type == BuildProductType.RequiredResource)
{
StageFile(StagedFileType.NonUFS, BuildProduct.Path);
}
else if(BuildProduct.Type == BuildProductType.SymbolFile)
{
StageFile(StagedFileType.DebugNonUFS, BuildProduct.Path);
}
}
// Also stage any additional runtime dependencies, like ThirdParty DLLs
foreach(RuntimeDependency RuntimeDependency in Receipt.RuntimeDependencies)
{
// allow missing files if needed
if (Receipt.bRequireDependenciesToExist == false && File.Exists(RuntimeDependency.Path) == false)
{
continue;
}
StageFile(StagedFileType.NonUFS, RuntimeDependency.Path, RuntimeDependency.StagePath);
}
}
public int StageFiles(StagedFileType FileType, string InPath, string Wildcard = "*", bool bRecursive = true, string[] ExcludeWildcard = null, string NewPath = null, bool bAllowNone = false, bool bRemap = true, string NewName = null, bool bAllowNotForLicenseesFiles = true, bool bStripFilesForOtherPlatforms = true)
{
int FilesAdded = 0;
// make sure any ..'s are removed
Utils.CollapseRelativeDirectories(ref InPath);
if (CommandUtils.DirectoryExists(InPath))
{
var All = CommandUtils.FindFiles(Wildcard, bRecursive, InPath);
var Exclude = new HashSet();
if (ExcludeWildcard != null)
{
foreach (var Excl in ExcludeWildcard)
{
var Remove = CommandUtils.FindFiles(Excl, bRecursive, InPath);
foreach (var File in Remove)
{
Exclude.Add(CommandUtils.CombinePaths(File));
}
}
}
foreach (var AllFile in All)
{
var FileToCopy = CommandUtils.CombinePaths(AllFile);
if (Exclude.Contains(FileToCopy))
{
continue;
}
if (!bAllowNotForLicenseesFiles && (FileToCopy.Contains("NotForLicensees") || FileToCopy.Contains("NoRedist")))
{
continue;
}
if (bStripFilesForOtherPlatforms && !bIsCombiningMultiplePlatforms)
{
bool OtherPlatform = false;
foreach (UnrealTargetPlatform Plat in Enum.GetValues(typeof(UnrealTargetPlatform)))
{
if (Plat != StageTargetPlatform.PlatformType && Plat != UnrealTargetPlatform.Unknown)
{
var Search = FileToCopy;
if (Search.StartsWith(LocalRoot, StringComparison.InvariantCultureIgnoreCase))
{
if (LocalRoot.EndsWith("\\") || LocalRoot.EndsWith("/"))
{
Search = Search.Substring(LocalRoot.Length - 1);
}
else
{
Search = Search.Substring(LocalRoot.Length);
}
}
if (Search.StartsWith(ProjectRoot, StringComparison.InvariantCultureIgnoreCase))
{
if (ProjectRoot.EndsWith("\\") || ProjectRoot.EndsWith("/"))
{
Search = Search.Substring(ProjectRoot.Length - 1);
}
else
{
Search = Search.Substring(ProjectRoot.Length);
}
}
if (Search.IndexOf(CommandUtils.CombinePaths("/" + Plat.ToString() + "/"), 0, StringComparison.InvariantCultureIgnoreCase) >= 0)
{
OtherPlatform = true;
break;
}
}
}
if (OtherPlatform)
{
continue;
}
}
string Dest;
if (!FileToCopy.StartsWith(InPath))
{
throw new AutomationException("Can't deploy {0}; it was supposed to start with {1}", FileToCopy, InPath);
}
string FileToRemap = FileToCopy;
// If the specified a new directory, first we deal with that, then apply the other things
// this is used to collapse the sandbox, among other things
if (NewPath != null)
{
Dest = FileToRemap.Substring(InPath.Length);
if (Dest.StartsWith("/") || Dest.StartsWith("\\"))
{
Dest = Dest.Substring(1);
}
Dest = CommandUtils.CombinePaths(NewPath, Dest);
#if false // if the cooker rebases, I don't think we need to ever rebase while staging to a new path
if (Dest.StartsWith("/") || Dest.StartsWith("\\"))
{
Dest = Dest.Substring(1);
}
// project relative stuff in a collapsed sandbox
if (Dest.StartsWith(SourceRelativeProjectRoot, StringComparison.InvariantCultureIgnoreCase))
{
Dest = Dest.Substring(SourceRelativeProjectRoot.Length);
if (Dest.StartsWith("/") || Dest.StartsWith("\\"))
{
Dest = Dest.Substring(1);
}
Dest = CommandUtils.CombinePaths(RelativeProjectRootForStage, Dest);
}
#endif
}
// project relative file
else if (FileToRemap.StartsWith(ProjectRoot, StringComparison.InvariantCultureIgnoreCase))
{
Dest = FileToRemap.Substring(ProjectRoot.Length);
if (Dest.StartsWith("/") || Dest.StartsWith("\\"))
{
Dest = Dest.Substring(1);
}
Dest = CommandUtils.CombinePaths(RelativeProjectRootForStage, Dest);
}
// engine relative file
else if (FileToRemap.StartsWith(LocalRoot, StringComparison.InvariantCultureIgnoreCase))
{
Dest = CommandUtils.CombinePaths(FileToRemap.Substring(LocalRoot.Length));
}
else
{
throw new AutomationException("Can't deploy {0} because it doesn't start with {1} or {2}", FileToRemap, ProjectRoot, LocalRoot);
}
if (Dest.StartsWith("/") || Dest.StartsWith("\\"))
{
Dest = Dest.Substring(1);
}
if (NewName != null)
{
Dest = CommandUtils.CombinePaths(Path.GetDirectoryName(Dest), NewName);
}
if (bRemap)
{
Dest = StageTargetPlatform.Remap(Dest);
}
if (FileType == StagedFileType.UFS)
{
AddUniqueStagingFile(UFSStagingFiles, FileToCopy, Dest);
}
else if (FileType == StagedFileType.NonUFS)
{
AddUniqueStagingFile(NonUFSStagingFiles, FileToCopy, Dest);
}
else if (FileType == StagedFileType.DebugNonUFS)
{
AddUniqueStagingFile(NonUFSStagingFilesDebug, FileToCopy, Dest);
}
FilesAdded++;
}
}
if (FilesAdded == 0 && !bAllowNone && !bIsCombiningMultiplePlatforms)
{
AutomationTool.ErrorReporter.Error(String.Format("No files found to deploy for {0} with wildcard {1} and exclusions {2}", InPath, Wildcard, ExcludeWildcard), (int)AutomationTool.ErrorCodes.Error_StageMissingFile);
throw new AutomationException("No files found to deploy for {0} with wildcard {1} and exclusions {2}", InPath, Wildcard, ExcludeWildcard);
}
return FilesAdded;
}
private void AddUniqueStagingFile(Dictionary FilesToStage, string FileToCopy, string Dest)
{
string ExistingDest;
if (FilesToStage.TryGetValue(FileToCopy, out ExistingDest))
{
// If the file already exists, it must have the same dest
if (String.Compare(ExistingDest, Dest, true) != 0)
{
throw new AutomationException("Attempting to add \"{0}\" to staging map but it already exists with a different destination (existing: \"{1}\", new: \"{2}\"",
FileToCopy, ExistingDest, Dest);
}
}
else
{
FilesToStage.Add(FileToCopy, Dest);
}
}
public int ArchiveFiles(string InPath, string Wildcard = "*", bool bRecursive = true, string[] ExcludeWildcard = null)
{
int FilesAdded = 0;
if (CommandUtils.DirectoryExists(InPath))
{
var All = CommandUtils.FindFiles(Wildcard, bRecursive, InPath);
var Exclude = new HashSet();
if (ExcludeWildcard != null)
{
foreach (var Excl in ExcludeWildcard)
{
var Remove = CommandUtils.FindFiles(Excl, bRecursive, InPath);
foreach (var File in Remove)
{
Exclude.Add(CommandUtils.CombinePaths(File));
}
}
}
foreach (var AllFile in All)
{
var FileToCopy = CommandUtils.CombinePaths(AllFile);
if (Exclude.Contains(FileToCopy))
{
continue;
}
if (!bIsCombiningMultiplePlatforms)
{
bool OtherPlatform = false;
foreach (UnrealTargetPlatform Plat in Enum.GetValues(typeof(UnrealTargetPlatform)))
{
if (Plat != StageTargetPlatform.PlatformType && Plat != UnrealTargetPlatform.Unknown && FileToCopy.IndexOf(CommandUtils.CombinePaths("/" + Plat.ToString() + "/"), 0, StringComparison.InvariantCultureIgnoreCase) >= 0)
{
OtherPlatform = true;
break;
}
}
if (OtherPlatform)
{
continue;
}
}
string Dest;
if (!FileToCopy.StartsWith(InPath))
{
throw new AutomationException("Can't archive {0}; it was supposed to start with {1}", FileToCopy, InPath);
}
Dest = FileToCopy.Substring(InPath.Length);
if (Dest.StartsWith("/") || Dest.StartsWith("\\"))
{
Dest = Dest.Substring(1);
}
if (ArchivedFiles.ContainsKey(FileToCopy))
{
if (ArchivedFiles[FileToCopy] != Dest)
{
throw new AutomationException("Can't archive {0}: it was already in the files to archive with a different destination '{1}'", FileToCopy, Dest);
}
}
else
{
ArchivedFiles.Add(FileToCopy, Dest);
}
FilesAdded++;
}
}
return FilesAdded;
}
public String GetUFSDeploymentDeltaPath()
{
return Path.Combine(StageDirectory, UFSDeployDeltaFileName);
}
public String GetNonUFSDeploymentDeltaPath()
{
return Path.Combine(StageDirectory, NonUFSDeployDeltaFileName);
}
}