// Copyright 1998-2014 Epic Games, Inc. All Rights Reserved. using System; using System.Collections; using System.Collections.Generic; using System.Diagnostics; using System.IO; using System.Net; using System.Runtime.Serialization; using System.Runtime.Serialization.Json; using System.Text; using System.Threading; using Tools.DotNETCommon.XmlHandler; using UnrealBuildTool; using System.Runtime.CompilerServices; using System.Linq; namespace AutomationTool { #region ParamList /// /// Wrapper around List with support for multi parameter constructor, i.e: /// var Maps = new ParamList("Map1", "Map2"); /// /// public class ParamList : List { public ParamList(params T[] Args) { AddRange(Args); } public ParamList(ICollection Collection) : base(Collection != null ? Collection : new T[] {}) { } public override string ToString() { var Text = ""; for (int Index = 0; Index < Count; ++Index) { if (Index > 0) { Text += ", "; } Text += this[Index].ToString(); } return Text; } } #endregion #region PathSeparator public enum PathSeparator { Default = 0, Slash, Backslash, Depot, Local } #endregion /// /// Base utility function for script commands. /// public partial class CommandUtils { #region Environment Setup static private CommandEnvironment CmdEnvironment; /// /// BuildEnvironment to use for this buildcommand. This is initialized by InitBuildEnvironment. As soon /// as the script execution in ExecuteBuild begins, the BuildEnv is set up and ready to use. /// static public CommandEnvironment CmdEnv { get { if (CmdEnvironment == null) { throw new AutomationException("Attempt to use CommandEnvironment before it was initialized."); } return CmdEnvironment; } } /// /// Initializes build environment. If the build command needs a specific env-var mapping or /// has an extended BuildEnvironment, it must implement this method accordingly. /// /// Initialized and ready to use BuildEnvironment static internal void InitCommandEnvironment() { CmdEnvironment = Automation.IsBuildMachine ? new CommandEnvironment() : new LocalCommandEnvironment(); ; } #endregion #region Logging /// /// Returns a formatted string for the specified exception (including inner exception if any). /// /// /// Formatted exception string. public static string ExceptionToString(Exception Ex) { return LogUtils.FormatException(Ex); } /// /// Writes formatted text to log (with TraceEventType.Log). /// /// Format string /// Parameters [MethodImplAttribute(MethodImplOptions.NoInlining)] public static void Log(string Format, params object[] Args) { UnrealBuildTool.Log.WriteLine(1, TraceEventType.Information, Format, Args); } /// /// Writes formatted text to log (with TraceEventType.Log). /// /// Text [MethodImplAttribute(MethodImplOptions.NoInlining)] public static void Log(string Message) { UnrealBuildTool.Log.WriteLine(1, TraceEventType.Information, Message); } /// /// Writes formatted text to log (with TraceEventType.Error). /// /// Format string /// Parameters [MethodImplAttribute(MethodImplOptions.NoInlining)] public static void LogError(string Format, params object[] Args) { UnrealBuildTool.Log.WriteLine(1, TraceEventType.Error, Format, Args); } /// /// Writes formatted text to log (with TraceEventType.Error). /// /// Text [MethodImplAttribute(MethodImplOptions.NoInlining)] public static void LogError(string Message) { UnrealBuildTool.Log.WriteLine(1, TraceEventType.Error, Message); } /// /// Writes formatted text to log (with TraceEventType.Warning). /// /// Format string /// Parameters [MethodImplAttribute(MethodImplOptions.NoInlining)] public static void LogWarning(string Format, params object[] Args) { UnrealBuildTool.Log.WriteLine(1, TraceEventType.Warning, Format, Args); } /// /// Writes a message to log (with TraceEventType.Warning). /// /// Text [MethodImplAttribute(MethodImplOptions.NoInlining)] public static void LogWarning(string Message) { UnrealBuildTool.Log.WriteLine(1, TraceEventType.Warning, Message); } /// /// Writes formatted text to log (with TraceEventType.Verbose). /// /// Format string /// Arguments [MethodImplAttribute(MethodImplOptions.NoInlining)] public static void LogVerbose(string Format, params object[] Args) { UnrealBuildTool.Log.WriteLine(1, TraceEventType.Verbose, Format, Args); } /// /// Writes formatted text to log (with TraceEventType.Verbose). /// /// Text [MethodImplAttribute(MethodImplOptions.NoInlining)] public static void LogVerbose(string Message) { UnrealBuildTool.Log.WriteLine(1, TraceEventType.Verbose, Message); } /// /// Writes formatted text to log. /// /// Verbosity /// Format string /// Arguments [MethodImplAttribute(MethodImplOptions.NoInlining)] public static void Log(TraceEventType Verbosity, string Format, params object[] Args) { UnrealBuildTool.Log.WriteLine(1, Verbosity, Format, Args); } /// /// Writes formatted text to log. /// /// Verbosity /// Text [MethodImplAttribute(MethodImplOptions.NoInlining)] public static void Log(TraceEventType Verbosity, string Message) { UnrealBuildTool.Log.WriteLine(1, Verbosity, Message); } /// /// Dumps exception to log. /// /// Verbosity /// Exception [MethodImplAttribute(MethodImplOptions.NoInlining)] public static void Log(TraceEventType Verbosity, Exception Ex) { UnrealBuildTool.Log.WriteLine(1, Verbosity, LogUtils.FormatException(Ex)); } #endregion #region IO /// /// Finds files in specified paths. /// /// Pattern /// Recursive search /// Paths to search /// An array of files found in the specified paths public static string[] FindFiles(string SearchPattern, bool Recursive, params string[] Paths) { List FoundFiles = new List(); foreach (var PathToSearch in Paths) { var NormalizedPath = ConvertSeparators(PathSeparator.Default, PathToSearch); if (DirectoryExists(NormalizedPath)) { var FoundInPath = InternalUtils.SafeFindFiles(NormalizedPath, SearchPattern, Recursive); if (FoundInPath == null) { throw new AutomationException(String.Format("Failed to find files in '{0}'", NormalizedPath)); } FoundFiles.AddRange(FoundInPath); } } return FoundFiles.ToArray(); } /// /// Finds files in specified paths. /// /// Pattern /// Recursive search /// Paths to search /// An array of files found in the specified paths public static string[] FindFiles_NoExceptions(string SearchPattern, bool Recursive, params string[] Paths) { List FoundFiles = new List(); foreach (var PathToSearch in Paths) { var NormalizedPath = ConvertSeparators(PathSeparator.Default, PathToSearch); if (DirectoryExists(NormalizedPath)) { var FoundInPath = InternalUtils.SafeFindFiles(NormalizedPath, SearchPattern, Recursive); if (FoundInPath != null) { FoundFiles.AddRange(FoundInPath); } } } return FoundFiles.ToArray(); } /// /// Finds files in specified paths. /// /// Pattern /// Recursive search /// Paths to search /// An array of files found in the specified paths public static string[] FindFiles_NoExceptions(bool bQuiet, string SearchPattern, bool Recursive, params string[] Paths) { List FoundFiles = new List(); foreach (var PathToSearch in Paths) { var NormalizedPath = ConvertSeparators(PathSeparator.Default, PathToSearch); if (DirectoryExists(NormalizedPath)) { var FoundInPath = InternalUtils.SafeFindFiles(NormalizedPath, SearchPattern, Recursive, bQuiet); if (FoundInPath != null) { FoundFiles.AddRange(FoundInPath); } } } return FoundFiles.ToArray(); } /// /// Finds files in specified paths. /// /// Pattern /// Recursive search /// Paths to search /// An array of files found in the specified paths public static string[] FindDirectories(bool bQuiet, string SearchPattern, bool Recursive, params string[] Paths) { List FoundDirs = new List(); foreach (var PathToSearch in Paths) { var NormalizedPath = ConvertSeparators(PathSeparator.Default, PathToSearch); if (DirectoryExists(NormalizedPath)) { var FoundInPath = InternalUtils.SafeFindDirectories(NormalizedPath, SearchPattern, Recursive, bQuiet); if (FoundInPath == null) { throw new AutomationException(String.Format("Failed to find directories in '{0}'", NormalizedPath)); } FoundDirs.AddRange(FoundInPath); } } return FoundDirs.ToArray(); } /// /// Finds Directories in specified paths. /// /// Pattern /// Recursive search /// Paths to search /// An array of files found in the specified paths public static string[] FindDirectories_NoExceptions(bool bQuiet, string SearchPattern, bool Recursive, params string[] Paths) { List FoundDirs = new List(); foreach (var PathToSearch in Paths) { var NormalizedPath = ConvertSeparators(PathSeparator.Default, PathToSearch); if (DirectoryExists(NormalizedPath)) { var FoundInPath = InternalUtils.SafeFindDirectories(NormalizedPath, SearchPattern, Recursive, bQuiet); if (FoundInPath != null) { FoundDirs.AddRange(FoundInPath); } } } return FoundDirs.ToArray(); } /// /// Deletes a file(s). /// If the file does not exist, silently succeeds. /// If the deletion of the file fails, this function throws an Exception. /// /// Filename public static void DeleteFile(params string[] Filenames) { foreach (var Filename in Filenames) { var NormalizedFilename = ConvertSeparators(PathSeparator.Default, Filename); if (!InternalUtils.SafeDeleteFile(NormalizedFilename)) { throw new AutomationException(String.Format("Failed to delete file '{0}'", NormalizedFilename)); } } } /// /// Deletes a file(s). /// If the file does not exist, silently succeeds. /// If the deletion of the file fails, this function throws an Exception. /// /// Filename public static void DeleteFile(bool bQuiet, params string[] Filenames) { foreach (var Filename in Filenames) { var NormalizedFilename = ConvertSeparators(PathSeparator.Default, Filename); if (!InternalUtils.SafeDeleteFile(NormalizedFilename, bQuiet)) { throw new AutomationException(String.Format("Failed to delete file '{0}'", NormalizedFilename)); } } } /// /// Deletes a file(s). /// If the deletion of the file fails, prints a warning. /// /// Filename public static bool DeleteFile_NoExceptions(params string[] Filenames) { bool Result = true; foreach (var Filename in Filenames) { var NormalizedFilename = ConvertSeparators(PathSeparator.Default, Filename); if (!InternalUtils.SafeDeleteFile(NormalizedFilename)) { Log(TraceEventType.Warning, "Failed to delete file '{0}'", NormalizedFilename); Result = false; } } return Result; } /// /// Deletes a file(s). /// If the deletion of the file fails, prints a warning. /// /// Filename /// if true, then don't retry and don't print much. public static bool DeleteFile_NoExceptions(string Filename, bool bQuiet = false) { bool Result = true; var NormalizedFilename = ConvertSeparators(PathSeparator.Default, Filename); if (!InternalUtils.SafeDeleteFile(NormalizedFilename, bQuiet)) { Log(bQuiet ? TraceEventType.Information : TraceEventType.Warning, "Failed to delete file '{0}'", NormalizedFilename); Result = false; } return Result; } /// /// Deletes a directory(or directories) including its contents (recursively, will delete read-only files). /// If the deletion of the directory fails, this function throws an Exception. /// /// Suppresses log output if true /// Directory public static void DeleteDirectory(bool bQuiet, params string[] Directories) { foreach (var Directory in Directories) { var NormalizedDirectory = ConvertSeparators(PathSeparator.Default, Directory); if (!InternalUtils.SafeDeleteDirectory(NormalizedDirectory, bQuiet)) { throw new AutomationException(String.Format("Failed to delete directory '{0}'", NormalizedDirectory)); } } } /// /// Deletes a directory(or directories) including its contents (recursively, will delete read-only files). /// If the deletion of the directory fails, this function throws an Exception. /// /// Directory public static void DeleteDirectory(params string[] Directories) { DeleteDirectory(false, Directories); } /// /// Deletes a directory(or directories) including its contents (recursively, will delete read-only files). /// If the deletion of the directory fails, prints a warning. /// /// Suppresses log output if true /// Directory public static bool DeleteDirectory_NoExceptions(bool bQuiet, params string[] Directories) { bool Result = true; foreach (var Directory in Directories) { var NormalizedDirectory = ConvertSeparators(PathSeparator.Default, Directory); try { if (!InternalUtils.SafeDeleteDirectory(NormalizedDirectory, bQuiet)) { Log(TraceEventType.Warning, "Failed to delete directory '{0}'", NormalizedDirectory); Result = false; } } catch (Exception Ex) { if (!bQuiet) { Log(TraceEventType.Warning, "Failed to delete directory, exception '{0}'", NormalizedDirectory); Log(TraceEventType.Warning, Ex); } Result = false; } } return Result; } /// /// Deletes a directory(or directories) including its contents (recursively, will delete read-only files). /// If the deletion of the directory fails, prints a warning. /// /// Directory public static bool DeleteDirectory_NoExceptions(params string[] Directories) { return DeleteDirectory_NoExceptions(false, Directories); } /// /// Attempts to delete a directory, if that fails deletes all files and folder from the specified directory. /// This works around the issue when the user has a file open in a notepad from that directory. Somehow deleting the file works but /// deleting the directory with the file that's open, doesn't. /// /// public static void DeleteDirectoryContents(string DirectoryName) { Log("DeleteDirectoryContents({0})", DirectoryName); const bool bQuiet = true; var Files = CommandUtils.FindFiles_NoExceptions(bQuiet, "*", false, DirectoryName); foreach (var Filename in Files) { CommandUtils.DeleteFile_NoExceptions(Filename); } var Directories = CommandUtils.FindDirectories_NoExceptions(bQuiet, "*", false, DirectoryName); foreach (var SubDirectoryName in Directories) { CommandUtils.DeleteDirectory_NoExceptions(bQuiet, SubDirectoryName); } } /// /// Checks if a directory(or directories) exists. /// /// Directory /// True if the directory exists, false otherwise. public static bool DirectoryExists(params string[] Directories) { bool bExists = Directories.Length > 0; foreach (var DirectoryName in Directories) { var NormalizedDirectory = ConvertSeparators(PathSeparator.Default, DirectoryName); bExists = System.IO.Directory.Exists(NormalizedDirectory) && bExists; } return bExists; } /// /// Checks if a directory(or directories) exists. /// /// Directory /// True if the directory exists, false otherwise. public static bool DirectoryExists_NoExceptions(params string[] Directories) { bool bExists = Directories.Length > 0; foreach (var DirectoryName in Directories) { var NormalizedDirectory = ConvertSeparators(PathSeparator.Default, DirectoryName); try { bExists = System.IO.Directory.Exists(NormalizedDirectory) && bExists; } catch (Exception Ex) { Log(TraceEventType.Warning, "Unable to check if directory exists: {0}", NormalizedDirectory); Log(TraceEventType.Warning, Ex); bExists = false; break; } } return bExists; } /// /// Creates a directory(or directories). /// If the creation of the directory fails, this function throws an Exception. /// /// Directory public static void CreateDirectory(params string[] Directories) { foreach (var DirectoryName in Directories) { var NormalizedDirectory = ConvertSeparators(PathSeparator.Default, DirectoryName); if (!InternalUtils.SafeCreateDirectory(NormalizedDirectory)) { throw new AutomationException(String.Format("Failed to create directory '{0}'", NormalizedDirectory)); } } } /// /// Creates a directory(or directories). /// If the creation of the directory fails, this function throws an Exception. /// /// Directory public static void CreateDirectory(bool bQuiet, params string[] Directories) { foreach (var DirectoryName in Directories) { var NormalizedDirectory = ConvertSeparators(PathSeparator.Default, DirectoryName); if (!InternalUtils.SafeCreateDirectory(NormalizedDirectory, bQuiet)) { throw new AutomationException(String.Format("Failed to create directory '{0}'", NormalizedDirectory)); } } } /// /// Creates a directory (or directories). /// If the creation of the directory fails, this function prints a warning. /// /// Directory public static bool CreateDirectory_NoExceptions(params string[] Directories) { bool Result = true; foreach (var DirectoryName in Directories) { var NormalizedDirectory = ConvertSeparators(PathSeparator.Default, DirectoryName); if (!InternalUtils.SafeCreateDirectory(NormalizedDirectory)) { Log(TraceEventType.Warning, "Failed to create directory '{0}'", NormalizedDirectory); Result = false; } } return Result; } /// /// Renames/moves a file. /// If the rename of the file fails, this function throws an Exception. /// /// Old name /// new name public static void RenameFile(string OldName, string NewName, bool bQuiet = false) { var OldNormalized = ConvertSeparators(PathSeparator.Default, OldName); var NewNormalized = ConvertSeparators(PathSeparator.Default, NewName); if (!InternalUtils.SafeRenameFile(OldNormalized, NewNormalized, bQuiet)) { throw new AutomationException(String.Format("Failed to rename/move file '{0}' to '{1}'", OldNormalized, NewNormalized)); } } /// /// Renames/moves a file. /// If the rename of the file fails, this function prints a warning. /// /// Old name /// new name public static bool RenameFile_NoExceptions(string OldName, string NewName) { var OldNormalized = ConvertSeparators(PathSeparator.Default, OldName); var NewNormalized = ConvertSeparators(PathSeparator.Default, NewName); var Result = InternalUtils.SafeRenameFile(OldNormalized, NewNormalized); if (!Result) { Log(TraceEventType.Warning, "Failed to rename/move file '{0}' to '{1}'", OldName, NewName); } return Result; } /// /// Checks if a file(s) exists. /// /// Filename. /// True if the file exists, false otherwise. public static bool FileExists(params string[] Filenames) { bool bExists = Filenames.Length > 0; foreach (var Filename in Filenames) { var NormalizedFilename = ConvertSeparators(PathSeparator.Default, Filename); bExists = InternalUtils.SafeFileExists(NormalizedFilename) && bExists; } return bExists; } /// /// Checks if a file(s) exists. /// /// Filename. /// True if the file exists, false otherwise. public static bool FileExists_NoExceptions(params string[] Filenames) { // Standard version doesn't throw, but keep this function for consistency. return FileExists(Filenames); } /// /// Checks if a file(s) exists. /// /// Filename. /// True if the file exists, false otherwise. public static bool FileExists(bool bQuiet, params string[] Filenames) { bool bExists = Filenames.Length > 0; foreach (var Filename in Filenames) { var NormalizedFilename = ConvertSeparators(PathSeparator.Default, Filename); bExists = InternalUtils.SafeFileExists(NormalizedFilename, bQuiet) && bExists; } return bExists; } /// /// Checks if a file(s) exists. /// /// Filename. /// True if the file exists, false otherwise. public static bool FileExists_NoExceptions(bool bQuiet, params string[] Filenames) { // Standard version doesn't throw, but keep this function for consistency. return FileExists(bQuiet, Filenames); } static Stack WorkingDirectoryStack = new Stack(); /// /// Pushes the current working directory onto a stack and sets CWD to a new value. /// /// New working direcotry. public static void PushDir(string WorkingDirectory) { string OrigCurrentDirectory = Environment.CurrentDirectory; WorkingDirectory = ConvertSeparators(PathSeparator.Default, WorkingDirectory); try { Environment.CurrentDirectory = WorkingDirectory; } catch (Exception Ex) { throw new AutomationException(String.Format("Unable to change current directory to {0}", WorkingDirectory), Ex); } WorkingDirectoryStack.Push(OrigCurrentDirectory); } /// /// Pushes the current working directory onto a stack and sets CWD to a new value. /// /// New working direcotry. public static bool PushDir_NoExceptions(string WorkingDirectory) { bool Result = true; WorkingDirectory = ConvertSeparators(PathSeparator.Default, WorkingDirectory); try { Environment.CurrentDirectory = WorkingDirectory; WorkingDirectoryStack.Push(Environment.CurrentDirectory); } catch { Log(TraceEventType.Warning, "Unable to change current directory to {0}", WorkingDirectory); Result = false; } return Result; } /// /// Pops the last working directory from a stack and sets it as the current working directory. /// public static void PopDir() { if (WorkingDirectoryStack.Count > 0) { Environment.CurrentDirectory = WorkingDirectoryStack.Pop(); } else { throw new AutomationException("Unable to PopDir. WorkingDirectoryStack is empty."); } } /// /// Pops the last working directory from a stack and sets it as the current working directory. /// public static bool PopDir_NoExceptions() { bool Result = true; if (WorkingDirectoryStack.Count > 0) { Environment.CurrentDirectory = WorkingDirectoryStack.Pop(); } else { Log(TraceEventType.Warning, "Unable to PopDir. WorkingDirectoryStack is empty."); Result = false; } return Result; } /// /// Clears the directory stack /// public static void ClearDirStack() { while (WorkingDirectoryStack.Count > 0) { PopDir(); } } /// /// Changes the current working directory. /// /// New working directory. public static void ChDir(string WorkingDirectory) { WorkingDirectory = ConvertSeparators(PathSeparator.Default, WorkingDirectory); try { Environment.CurrentDirectory = WorkingDirectory; } catch (Exception Ex) { throw new ArgumentException(String.Format("Unable to change current directory to {0}", WorkingDirectory), Ex); } } /// /// Changes the current working directory. /// /// New working directory. public static bool ChDir_NoExceptions(string WorkingDirectory) { bool Result = true; WorkingDirectory = ConvertSeparators(PathSeparator.Default, WorkingDirectory); try { Environment.CurrentDirectory = WorkingDirectory; } catch { Log(TraceEventType.Warning, "Unable to change current directory to {0}", WorkingDirectory); Result = false; } return Result; } /// /// Sets file attributes. Will not change attributes that have not been specified. /// /// Filename /// Read-only attribute /// Hidden attribute. /// Archive attribute. public static void SetFileAttributes(string Filename, bool? ReadOnly = null, bool? Hidden = null, bool? Archive = null) { Filename = ConvertSeparators(PathSeparator.Default, Filename); if (!File.Exists(Filename)) { throw new AutomationException("Unable to set attributes for a non-exisiting file.", new FileNotFoundException("File not found.", Filename)); } FileAttributes Attributes = File.GetAttributes(Filename); Attributes = InternalSetAttributes(ReadOnly, Hidden, Archive, Attributes); File.SetAttributes(Filename, Attributes); } /// /// Sets file attributes. Will not change attributes that have not been specified. /// /// Filename /// Read-only attribute /// Hidden attribute. /// Archive attribute. public static bool SetFileAttributes_NoExceptions(string Filename, bool? ReadOnly = null, bool? Hidden = null, bool? Archive = null) { Filename = ConvertSeparators(PathSeparator.Default, Filename); if (!File.Exists(Filename)) { Log(TraceEventType.Warning, "Unable to set attributes for a non-exisiting file ({0})", Filename); return false; } bool Result = true; try { FileAttributes Attributes = File.GetAttributes(Filename); Attributes = InternalSetAttributes(ReadOnly, Hidden, Archive, Attributes); File.SetAttributes(Filename, Attributes); } catch (Exception Ex) { Log(TraceEventType.Warning, "Error trying to set file attributes for: {0}", Filename); Log(TraceEventType.Warning, Ex); Result = false; } return Result; } private static FileAttributes InternalSetAttributes(bool? ReadOnly, bool? Hidden, bool? Archive, FileAttributes Attributes) { if (ReadOnly != null) { if ((bool)ReadOnly) { Attributes |= FileAttributes.ReadOnly; } else { Attributes &= ~FileAttributes.ReadOnly; } } if (Hidden != null) { if ((bool)Hidden) { Attributes |= FileAttributes.Hidden; } else { Attributes &= ~FileAttributes.Hidden; } } if (Archive != null) { if ((bool)Archive) { Attributes |= FileAttributes.Archive; } else { Attributes &= ~FileAttributes.Archive; } } return Attributes; } /// /// Writes a line of formatted string to a file. Creates the file if it does not exists. /// If the file does exists, appends a new line. /// /// Filename /// Format string /// Arguments public static void WriteToFile(string Filename, string Format, params object[] Args) { WriteToFile(Filename, String.Format(Format, Args)); } /// /// Writes a line of formatted string to a file. Creates the file if it does not exists. /// If the file does exists, appends a new line. /// /// Filename /// Text to write public static void WriteToFile(string Filename, string Text) { try { Filename = ConvertSeparators(PathSeparator.Default, Filename); FileStream Stream; if (File.Exists(Filename)) { Stream = new FileStream(Filename, FileMode.Append, FileAccess.Write, FileShare.Read); } else { Stream = new FileStream(Filename, FileMode.OpenOrCreate, FileAccess.Write, FileShare.Read); } if (Stream != null) { using (StreamWriter Writer = new StreamWriter(Stream)) { Writer.WriteLine(Text); Writer.Flush(); Writer.Close(); } Stream.Dispose(); } } catch (Exception Ex) { throw new AutomationException(String.Format("Failed to Write to file {0}", Filename), Ex); } } /// /// Reads all text lines from a file. /// /// Filename /// Array of lines of text read from the file. null if the file did not exist or could not be read. public static string[] ReadAllLines(string Filename) { Filename = ConvertSeparators(PathSeparator.Default, Filename); return InternalUtils.SafeReadAllLines(Filename); } /// /// Reads all text from a file. /// /// Filename /// All text read from the file. null if the file did not exist or could not be read. public static string ReadAllText(string Filename) { Filename = ConvertSeparators(PathSeparator.Default, Filename); return InternalUtils.SafeReadAllText(Filename); } /// /// Writes lines to a file. /// /// Filename /// Text public static void WriteAllLines(string Filename, string[] Lines) { Filename = ConvertSeparators(PathSeparator.Default, Filename); if (!InternalUtils.SafeWriteAllLines(Filename, Lines)) { throw new AutomationException("Unable to write to file: {0}", Filename); } } /// /// Writes lines to a file. /// /// Filename /// Text public static bool WriteAllLines_NoExceptions(string Filename, string[] Lines) { Filename = ConvertSeparators(PathSeparator.Default, Filename); return InternalUtils.SafeWriteAllLines(Filename, Lines); } /// /// Writes text to a file. /// /// Filename /// Text public static void WriteAllText(string Filename, string Text) { Filename = ConvertSeparators(PathSeparator.Default, Filename); if (!InternalUtils.SafeWriteAllText(Filename, Text)) { throw new AutomationException("Unable to write to file: {0}", Filename); } } /// /// Writes text to a file. /// /// Filename /// Text public static bool WriteAllText_NoExceptions(string Filename, string Text) { Filename = ConvertSeparators(PathSeparator.Default, Filename); return InternalUtils.SafeWriteAllText(Filename, Text); } /// /// Writes byte array to a file. /// /// Filename /// Byte array public static void WriteAllBytes(string Filename, byte[] Bytes) { Filename = ConvertSeparators(PathSeparator.Default, Filename); if (!InternalUtils.SafeWriteAllBytes(Filename, Bytes)) { throw new AutomationException("Unable to write to file: {0}", Filename); } } /// /// Writes byte array to a file. /// /// Filename /// Byte array public static bool WriteAllBytes_NoExceptions(string Filename, byte[] Bytes) { Filename = ConvertSeparators(PathSeparator.Default, Filename); return InternalUtils.SafeWriteAllBytes(Filename, Bytes); } /// /// Gets a character representing the specified separator type. /// /// Separator type. /// Separator character public static char GetPathSeparatorChar(PathSeparator SeparatorType) { char Separator; switch (SeparatorType) { case PathSeparator.Slash: case PathSeparator.Depot: Separator = '/'; break; case PathSeparator.Backslash: Separator = '\\'; break; default: Separator = Path.DirectorySeparatorChar; break; } return Separator; } /// /// Checks if the character is one of the two sperator types ('\' or '/') /// /// Character to check. /// True if the character is a separator, false otherwise. public static bool IsPathSeparator(char Character) { return (Character == '/' || Character == '\\'); } /// /// Combines paths and replaces all path separators with the system default separator. /// /// /// Combined Path public static string CombinePaths(params string[] Paths) { return CombinePaths(PathSeparator.Default, Paths); } /// /// Combines paths and replaces all path separators wth the system specified separator. /// /// Type of separartor to use when combining paths. /// /// Combined Path public static string CombinePaths(PathSeparator SeparatorType, params string[] Paths) { // Pick a separator to use. var SeparatorToUse = GetPathSeparatorChar(SeparatorType); var SeparatorToReplace = SeparatorToUse == '/' ? '\\' : '/'; // Allocate string builder int CombinePathMaxLength = 0; foreach (var PathPart in Paths) { CombinePathMaxLength += (PathPart != null) ? PathPart.Length : 0; } CombinePathMaxLength += Paths.Length; var CombinedPath = new StringBuilder(CombinePathMaxLength); // Combine all paths CombinedPath.Append(Paths[0]); for (int PathIndex = 1; PathIndex < Paths.Length; ++PathIndex) { var NextPath = Paths[PathIndex]; if (String.IsNullOrEmpty(NextPath) == false) { int NextPathStartIndex = 0; if (CombinedPath.Length != 0) { var LastChar = CombinedPath[CombinedPath.Length - 1]; var NextChar = NextPath[0]; var IsLastCharPathSeparator = IsPathSeparator(LastChar); var IsNextCharPathSeparator = IsPathSeparator(NextChar); // Check if a separator between paths is required if (!IsLastCharPathSeparator && !IsNextCharPathSeparator) { CombinedPath.Append(SeparatorToUse); } // Check if one of the saprators needs to be skipped. else if (IsLastCharPathSeparator && IsNextCharPathSeparator) { NextPathStartIndex = 1; } } CombinedPath.Append(NextPath, NextPathStartIndex, NextPath.Length - NextPathStartIndex); } } // Make sure there's only one separator type used. CombinedPath.Replace(SeparatorToReplace, SeparatorToUse); return CombinedPath.ToString(); } /// /// Converts all separators in path to the specified separator type. /// /// Desired separator type. /// Path /// Path where all separators have been converted to the specified type. public static string ConvertSeparators(PathSeparator ToSperatorType, string PathToConvert) { return CombinePaths(ToSperatorType, PathToConvert); } /// /// Copies a file. /// /// /// /// True if the operation was successful, false otherwise. public static void CopyFile(string Source, string Dest, bool bQuiet = false) { Source = ConvertSeparators(PathSeparator.Default, Source); Dest = ConvertSeparators(PathSeparator.Default, Dest); if (InternalUtils.SafeFileExists(Dest, true)) { InternalUtils.SafeDeleteFile(Dest, bQuiet); } else if (!InternalUtils.SafeDirectoryExists(Path.GetDirectoryName(Dest), true)) { if (!InternalUtils.SafeCreateDirectory(Path.GetDirectoryName(Dest), bQuiet)) { throw new AutomationException("Failed to create directory {0} for copy", Path.GetDirectoryName(Dest)); } } if (InternalUtils.SafeFileExists(Dest, true)) { throw new AutomationException("Failed to delete {0} for copy", Dest); } if (!InternalUtils.SafeCopyFile(Source, Dest, bQuiet)) { throw new AutomationException("Failed to copy {0} to {1}", Source, Dest); } } /// /// Copies a file. Does not throw exceptions. /// /// /// /// True if the operation was successful, false otherwise. public static bool CopyFile_NoExceptions(string Source, string Dest, bool bQuiet = false) { Source = ConvertSeparators(PathSeparator.Default, Source); Dest = ConvertSeparators(PathSeparator.Default, Dest); if (InternalUtils.SafeFileExists(Dest, true)) { InternalUtils.SafeDeleteFile(Dest, bQuiet); } else if (!InternalUtils.SafeDirectoryExists(Path.GetDirectoryName(Dest), true)) { if (!InternalUtils.SafeCreateDirectory(Path.GetDirectoryName(Dest))) { return false; } } if (InternalUtils.SafeFileExists(Dest, true)) { return false; } return InternalUtils.SafeCopyFile(Source, Dest, bQuiet); } /// /// Copies a file if the dest doesn't exist or the dest timestamp is different; after a copy, copies the timestamp /// /// /// /// True if the operation was successful, false otherwise. public static void CopyFileIncremental(string Source, string Dest) { Source = ConvertSeparators(PathSeparator.Default, Source); Dest = ConvertSeparators(PathSeparator.Default, Dest); if (InternalUtils.SafeFileExists(Dest, true)) { TimeSpan Diff = File.GetLastWriteTimeUtc(Dest) - File.GetLastWriteTimeUtc(Source); if (Diff.TotalSeconds > -1 && Diff.TotalSeconds < 1) { Log("CopyFileIncremental Skipping {0}, up to date.", Dest); return; } InternalUtils.SafeDeleteFile(Dest); } else if (!InternalUtils.SafeDirectoryExists(Path.GetDirectoryName(Dest), true)) { if (!InternalUtils.SafeCreateDirectory(Path.GetDirectoryName(Dest))) { throw new AutomationException("Failed to create directory {0} for copy", Path.GetDirectoryName(Dest)); } } if (InternalUtils.SafeFileExists(Dest, true)) { throw new AutomationException("Failed to delete {0} for copy", Dest); } if (!InternalUtils.SafeCopyFile(Source, Dest)) { throw new AutomationException("Failed to copy {0} to {1}", Source, Dest); } FileAttributes Attributes = File.GetAttributes(Dest); if ((Attributes & FileAttributes.ReadOnly) != 0) { File.SetAttributes(Dest, Attributes & ~FileAttributes.ReadOnly); } File.SetLastWriteTimeUtc(Dest, File.GetLastWriteTimeUtc(Source)); } /// /// Returns directory name without filename. /// The difference between this and Path.GetDirectoryName is that this /// function will not throw away the last name if it doesn't have an extension, for example: /// D:\Project\Data\Asset -> D:\Project\Data\Asset /// D:\Project\Data\Asset.ussset -> D:\Project\Data /// /// /// public static string GetDirectoryName(string FilePath) { var LastSeparatorIndex = Math.Max(FilePath.LastIndexOf('/'), FilePath.LastIndexOf('\\')); var ExtensionIndex = FilePath.LastIndexOf('.'); if (ExtensionIndex > LastSeparatorIndex || LastSeparatorIndex == (FilePath.Length - 1)) { return FilePath.Substring(0, LastSeparatorIndex); } else { return FilePath; } } /// /// Returns the last directory name in the path string. /// For example: D:\Temp\Project\File.txt -> Project, Data\Samples -> Samples /// /// /// public static string GetLastDirectoryName(string FilePath) { var LastDir = GetDirectoryName(FilePath); var LastSeparatorIndex = Math.Max(LastDir.LastIndexOf('/'), LastDir.LastIndexOf('\\')); if (LastSeparatorIndex >= 0) { LastDir = LastDir.Substring(LastSeparatorIndex + 1); } return LastDir; } /// /// Removes multi-dot extensions from a filename (i.e. *.automation.csproj) /// /// Filename to remove the extensions from /// Clean filename. public static string GetFilenameWithoutAnyExtensions(string Filename) { do { Filename = Path.GetFileNameWithoutExtension(Filename); } while (Filename.IndexOf('.') >= 0); return Filename; } /// /// A container for a binary files (dll, exe) with its associated debug info. /// public class FileManifest { /// /// Items /// public readonly List FileManifestItems = new List(); /// /// Constructor /// public FileManifest() { } } /// /// Reads a file manifest and returns it /// /// ManifestName /// public static FileManifest ReadManifest(string ManifestName) { return XmlHandler.ReadXml(ManifestName); } private static void CloneDirectoryRecursiveWorker(string SourcePathBase, string TargetPathBase, List ClonedFiles) { if (!InternalUtils.SafeCreateDirectory(TargetPathBase)) { throw new AutomationException("Failed to create directory {0} for copy", TargetPathBase); } DirectoryInfo SourceDirectory = new DirectoryInfo(SourcePathBase); DirectoryInfo[] SourceSubdirectories = SourceDirectory.GetDirectories(); // Copy the files FileInfo[] SourceFiles = SourceDirectory.GetFiles(); foreach (FileInfo SourceFI in SourceFiles) { string TargetFilename = CommandUtils.CombinePaths(TargetPathBase, SourceFI.Name); SourceFI.CopyTo(TargetFilename); if (ClonedFiles != null) { ClonedFiles.Add(TargetFilename); } } // Recurse into subfolders foreach (DirectoryInfo SourceSubdir in SourceSubdirectories) { string NewSourcePath = CommandUtils.CombinePaths(SourcePathBase, SourceSubdir.Name); string NewTargetPath = CommandUtils.CombinePaths(TargetPathBase, SourceSubdir.Name); CloneDirectoryRecursiveWorker(NewSourcePath, NewTargetPath, ClonedFiles); } } /// /// Clones a directory. /// Warning: Will delete all of the existing files in TargetPath /// This is recursive, copying subfolders too. /// /// Source directory. /// Target directory. /// List of cloned files. public static void CloneDirectory(string SourcePath, string TargetPath, List ClonedFiles = null) { DeleteDirectory_NoExceptions(TargetPath); CloneDirectoryRecursiveWorker(SourcePath, TargetPath, ClonedFiles); } #endregion #region Threaded Copy /// /// Single batch of files to copy on a thread /// class CopyRequest { public string[] Source; public string[] Dest; public Exception Result; public Thread Worker; } /// /// Main thread procedure for copying files /// private static void CopyThreadProc(object Request) { const bool bQuiet = true; var FileToCopy = (CopyRequest)Request; try { for (int Index = 0; Index < FileToCopy.Source.Length; ++Index) { CopyFile(FileToCopy.Source[Index], FileToCopy.Dest[Index], bQuiet); } } catch (Exception Ex) { FileToCopy.Result = Ex; } } /// /// Copies files using miltiple threads /// /// /// /// public static void ThreadedCopyFiles(string SourceDirectory, string DestDirectory, int MaxThreads = 64) { CreateDirectory(DestDirectory); var SourceFiles = Directory.GetFiles(SourceDirectory, "*", SearchOption.AllDirectories); var SourceBase = GetDirectoryName(SourceDirectory) + GetPathSeparatorChar(PathSeparator.Default); var DestBase = GetDirectoryName(DestDirectory) + GetPathSeparatorChar(PathSeparator.Default); var DestFiles = new string[SourceFiles.Length]; for (int Index = 0; Index < SourceFiles.Length; ++Index) { DestFiles[Index] = SourceFiles[Index].Replace(SourceBase, DestBase); } ThreadedCopyFiles(SourceFiles, DestFiles, MaxThreads); } /// /// Copies files using miltiple threads /// /// /// /// public static void ThreadedCopyFiles(string[] Source, string[] Dest, int MaxThreads = 64) { Log("Copying {0} file(s) using max {1} thread(s)", Source.Length, MaxThreads); if (Source.Length != Dest.Length) { throw new AutomationException("Source count ({0}) does not match Dest count ({1})", Source.Length, Dest.Length); } const int MinFilesPerThread = 10; if (MaxThreads > 1 && Source.Length > MinFilesPerThread) { // Split evenly across the threads int FilesPerThread = Math.Max(Source.Length / MaxThreads, MinFilesPerThread); int ThreadCount = Math.Min(Source.Length / FilesPerThread + 1, MaxThreads); FilesPerThread = Source.Length / ThreadCount + 1; // Divide and copy var WorkerThreads = new List(ThreadCount); var FilesToCopy = Source.Length; var FirstFileIndex = 0; while (FilesToCopy > 0) { var Request = new CopyRequest(); WorkerThreads.Add(Request); Request.Source = new string[Math.Min(FilesToCopy, FilesPerThread)]; Request.Dest = new string[Request.Source.Length]; Array.Copy(Source, FirstFileIndex, Request.Source, 0, Request.Source.Length); Array.Copy(Dest, FirstFileIndex, Request.Dest, 0, Request.Dest.Length); Request.Worker = new Thread(new ParameterizedThreadStart(CopyThreadProc)); Request.Worker.Start(Request); FirstFileIndex += Request.Source.Length; FilesToCopy -= Request.Source.Length; } // Wait for completion foreach (var Request in WorkerThreads) { if (Request.Worker.IsAlive) { Request.Worker.Join(); } } } else { const bool bQuiet = true; for (int Index = 0; Index < Source.Length; ++Index) { CopyFile(Source[Index], Dest[Index], bQuiet); } } } #endregion #region Environment variables /// /// Gets environment variable value. /// /// Name of the environment variable /// Environment variable value as string. public static string GetEnvVar(string Name) { return InternalUtils.GetEnvironmentVariable(Name, ""); } /// /// Gets environment variable value. /// /// Name of the environment variable /// Default value of the environment variable if the variable is not set. /// Environment variable value as string. public static string GetEnvVar(string Name, string DefaultValue) { return InternalUtils.GetEnvironmentVariable(Name, DefaultValue); } /// /// Sets environment variable. /// /// Variable name. /// Variable value. /// True if the value has been set, false otherwise. public static void SetEnvVar(string Name, object Value) { try { Log("SetEnvVar {0}={1}", Name, Value); Environment.SetEnvironmentVariable(Name, Value.ToString()); } catch (Exception Ex) { throw new AutomationException(String.Format("Failed to set environment variable {0} to {1}", Name, Value), Ex); } } /// /// Sets the environment variable if it hasn't been already. /// /// Environment variable name /// New value public static void ConditionallySetEnvVar(string VarName, string Value) { if (String.IsNullOrEmpty(CommandUtils.GetEnvVar(VarName))) { Environment.SetEnvironmentVariable(VarName, Value); } } /// /// Gets environment variables set by a batch file. /// /// Filename that sets the environment variables /// True if found variables should be automatically set withing this process. /// Dictionary of environment variables set by the batch file. public static CaselessDictionary GetEnvironmentVariablesFromBatchFile(string BatchFileName, bool AlsoSet = false) { if (File.Exists(BatchFileName)) { // Create a wrapper batch file that echoes environment variables to a text file var EnvOutputFileName = CommandUtils.CombinePaths(Path.GetTempPath(), "HarvestEnvVars.txt"); var EnvReaderBatchFileName = CommandUtils.CombinePaths(Path.GetTempPath(), "HarvestEnvVars.bat"); { var EnvReaderBatchFileContent = new List(); // Run 'vcvars32.bat' (or similar x64 version) to set environment variables EnvReaderBatchFileContent.Add(String.Format("call \"{0}\"", BatchFileName)); // Pipe all environment variables to a file where we can read them in EnvReaderBatchFileContent.Add(String.Format("set >\"{0}\"", EnvOutputFileName)); File.WriteAllLines(EnvReaderBatchFileName, EnvReaderBatchFileContent); } InternalUtils.SafeDeleteFile(EnvOutputFileName); CommandUtils.Run(EnvReaderBatchFileName, ""); var Result = new CaselessDictionary(); // Load environment variables var EnvStringsFromFile = File.ReadAllLines(EnvOutputFileName); foreach (var EnvString in EnvStringsFromFile) { // Parse the environment variable name and value from the string ("name=value") int EqualSignPos = EnvString.IndexOf('='); var EnvironmentVariableName = EnvString.Substring(0, EqualSignPos); var EnvironmentVariableValue = EnvString.Substring(EqualSignPos + 1); if (Environment.GetEnvironmentVariable(EnvironmentVariableName) != EnvironmentVariableValue) { Result.Add(EnvironmentVariableName, EnvironmentVariableValue); } if (AlsoSet) { // Set the environment variable Environment.SetEnvironmentVariable(EnvironmentVariableName, EnvironmentVariableValue); } } return Result; } else { throw new AutomationException("BatchFile {0} does not exist!", BatchFileName); } } #endregion #region CommandLine /// /// Converts a list of arguments to a string where each argument is separated with a space character. /// /// Arguments /// Single string containing all arguments separated with a space. public static string ArgsToCommandLine(params object[] Args) { string Arguments = ""; if (Args != null) { for (int Index = 0; Index < Args.Length; ++Index) { Arguments += Args[Index].ToString(); if (Index < (Args.Length - 1)) { Arguments += " "; } } } return Arguments; } /// /// Parses the argument list for a parameter and returns whether it is defined or not. /// /// Argument list. /// Param to check for. /// True if param was found, false otherwise. public static bool ParseParam(object[] ArgList, string Param) { foreach (object Arg in ArgList) { if (Arg.ToString().Equals(Param, StringComparison.InvariantCultureIgnoreCase)) { return true; } } return false; } /// /// Parses the command's Params list for a parameter and returns whether it is defined or not. /// /// Param to check for. /// True if param was found, false otherwise. public bool ParseParam(string Param) { return ParseParam(Params, Param); } /// /// Parses the argument list for a parameter and reads its value. /// Ex. ParseParamValue(Args, "map=") /// /// Argument list. /// Param to read its value. /// Returns the value or Default if the parameter was not found. public static string ParseParamValue(object[] ArgList, string Param, string Default = null) { if (!Param.EndsWith("=")) { Param += "="; } foreach (object Arg in ArgList) { string ArgStr = Arg.ToString(); if (ArgStr.StartsWith(Param, StringComparison.InvariantCultureIgnoreCase)) { return ArgStr.Substring(Param.Length); } } return Default; } /// /// Parses the command's Params list for a parameter and reads its value. /// Ex. ParseParamValue(Args, "map=") /// /// Param to read its value. /// Returns the value or Default if the parameter was not found. public string ParseParamValue(string Param, string Default = null) { return ParseParamValue(Params, Param, Default); } /// /// Parses the command's Params list for a parameter and reads its value. /// Ex. ParseParamValue(Args, "map=") /// /// Param to read its value. /// Returns the value or Default if the parameter was not found. public int ParseParamInt(string Param, int Default = 0) { string num = ParseParamValue(Params, Param, Default.ToString()); return int.Parse(num); } /// /// Makes sure path can be used as a command line param (adds quotes if it contains spaces) /// /// Path to convert /// public static string MakePathSafeToUseWithCommandLine(string InPath) { if (InPath.Contains(' ') && InPath[0] != '\"') { InPath = "\"" + InPath + "\""; } return InPath; } #endregion #region Other public static string EscapePath(string InPath) { return InPath.Replace(":", "").Replace("/", "+").Replace("\\", "+").Replace(" ", "+"); } /// /// Checks if collection is either null or empty. /// /// Collection to check. /// True if the collection is either nur or empty. public static bool IsNullOrEmpty(ICollection Collection) { return Collection == null || Collection.Count == 0; } /// /// List of available target platforms. /// public static UnrealBuildTool.UnrealTargetPlatform[] KnownTargetPlatforms { get { if (UBTTargetPlatforms == null || UBTTargetPlatforms.Length == 0) { UBTTargetPlatforms = new UnrealBuildTool.UnrealTargetPlatform[UnrealBuildTool.UEBuildPlatform.BuildPlatformDictionary.Count]; int Index = 0; foreach (var Platform in UnrealBuildTool.UEBuildPlatform.BuildPlatformDictionary) { UBTTargetPlatforms[Index++] = Platform.Key; } } return UBTTargetPlatforms; } } private static UnrealBuildTool.UnrealTargetPlatform[] UBTTargetPlatforms; /// /// Uploads the contents of a given directory to a series of MCP endpoints /// /// The directory to upload /// Username for the MCP service permission /// Password for the MCP service permission /// List of MCP service URL enpodints public static void UploadToEMS(string Location, string Username, string Password, List Services) { //@todo add logging //@todo refactor for more general usability //@todo this information should be grabbed from Jenkins via an env var. See AWSContext if (Location == null) { throw new AutomationException("Location for EMS upload is not set."); } if (Username == null) { throw new AutomationException("Username for EMS upload is not set."); } if (Password == null) { throw new AutomationException("Password for EMS upload is not set."); } if (Services == null || Services.Count == 0) { throw new AutomationException("No services found EMS upload."); } string EnumerateUrl = "/api/cloudstorage/system"; //iterate through files in the directory and store them //do this here rather than at upload time because they might be uploaded to multiple services List Files = new List(); string[] FileNames = Directory.GetFiles(Location); foreach (String FileName in FileNames) { string[] Parts = FileName.Split('\\'); EMSFileInfo Info = new EMSFileInfo(); Info.Bytes = System.IO.File.ReadAllBytes(FileName); Info.FileName = Parts[Parts.Length - 1]; Files.Add(Info); } //auth header WebHeaderCollection Headers = new WebHeaderCollection(); string BasicAuthString = System.Convert.ToBase64String(Encoding.UTF8.GetBytes(Username + ":" + Password)); Headers.Add("Authorization", "Basic " + BasicAuthString); DataContractJsonSerializer Serializer = new DataContractJsonSerializer(typeof(List)); foreach (string BaseUrl in Services) { string Url = BaseUrl + EnumerateUrl; //list all the files WebRequest Enumerate = WebRequest.Create(Url); Enumerate.Method = "GET"; Enumerate.ContentType = "application/json"; Enumerate.Headers = Headers; HttpWebResponse Response = (HttpWebResponse)Enumerate.GetResponse(); //if there are any, iterate through and delete if (Response.StatusCode == HttpStatusCode.OK) { List FoundFiles = (List) Serializer.ReadObject(Response.GetResponseStream()); foreach (EnumerationResponse File in FoundFiles) { WebRequest Delete = WebRequest.Create(Url + "/" + File.UniqueFilename); Delete.Method = "DELETE"; Delete.Headers = Headers; Delete.GetResponse(); } } //upload the files foreach (EMSFileInfo File in Files) { WebRequest Upload = WebRequest.Create(Url + "?filename=" + File.FileName); Upload.Method = "POST"; Upload.Headers = Headers; Upload.ContentType = "text/plain"; Upload.ContentLength = File.Bytes.Length; using (var payload = Upload.GetRequestStream()) { payload.Write(File.Bytes, 0, File.Bytes.Length); } using (var response = (HttpWebResponse)Upload.GetResponse()) { } } } } #endregion #region Properties /// /// Command line parameters for this command (empty by non-null by default) /// private object[] CommandLineParams = new object[0]; public object[] Params { get { return CommandLineParams; } set { CommandLineParams = value; } } /// /// Path to the AutomationTool executable. /// public static string ExeFilename { get { return InternalUtils.ExecutingAssemblyLocation; } } /// /// Directory where the AutomationTool executable sits. /// public static string ExeDirectory { get { return InternalUtils.ExecutingAssemblyDirectory; } } /// /// Current directory. /// public static string CurrentDirectory { get { return Environment.CurrentDirectory; } set { ChDir(value); } } /// /// Checks if this command is running on a build machine. /// public static bool IsBuildMachine { get { return Automation.IsBuildMachine; } } #endregion public static string RootSharedTempStorageDirectory() { string StorageDirectory = ""; if (UnrealBuildTool.Utils.IsRunningOnMono) { StorageDirectory = "/Volumes/Builds"; } else { StorageDirectory = CombinePaths("P:", "Builds"); } return StorageDirectory; } static bool DirectoryExistsAndIsWritable_NoExceptions(string Dir) { try { if (!DirectoryExists_NoExceptions(Dir)) { return false; } var TestGUID = Guid.NewGuid(); var Filename = CombinePaths(Dir, TestGUID.ToString() + ".Temp.txt"); WriteAllText_NoExceptions(Filename, "Test"); if (FileExists_NoExceptions(true, Filename)) { DeleteFile_NoExceptions(Filename, true); //Log(System.Diagnostics.TraceEventType.Information, "Resolved shared dir {0}", Dir); return true; } Log(System.Diagnostics.TraceEventType.Warning, "Shared dir {0} is not writable", Dir); } catch (Exception Ex) { Log(System.Diagnostics.TraceEventType.Warning, "Failed to resolve shared dir {0}", Dir); Log(System.Diagnostics.TraceEventType.Warning, LogUtils.FormatException(Ex)); } return false; } static Dictionary ResolveCache = new Dictionary(); public static string ResolveSharedBuildDirectory(string GameFolder) { if (ResolveCache.ContainsKey(GameFolder)) { return ResolveCache[GameFolder]; } string Root = RootSharedTempStorageDirectory(); string Result = CombinePaths(Root, GameFolder); if (String.IsNullOrEmpty(GameFolder) || !DirectoryExistsAndIsWritable_NoExceptions(Result)) { string GameStr = "Game"; bool HadGame = false; if (GameFolder.EndsWith(GameStr, StringComparison.InvariantCultureIgnoreCase)) { string ShortFolder = GameFolder.Substring(0, GameFolder.Length - GameStr.Length); Result = CombinePaths(Root, ShortFolder); HadGame = true; } if (!HadGame || !DirectoryExistsAndIsWritable_NoExceptions(Result)) { Result = CombinePaths(Root, "UE4"); if (!DirectoryExistsAndIsWritable_NoExceptions(Result)) { throw new AutomationException("Could not find an appropriate shared temp folder {0}", Result); } } } ResolveCache.Add(GameFolder, Result); return Result; } public static void CleanFormalBuilds(string DirectoryForThisBuild, string CLString = "", int MaximumDaysToKeepTempStorage = 4) { if (CLString == "" && (!IsBuildMachine || !DirectoryForThisBuild.StartsWith(RootSharedTempStorageDirectory()) || !P4Enabled)) { return; } try { if (P4Enabled && CLString == "") { CLString = P4Env.ChangelistString; } string ParentDir = Path.GetDirectoryName(CombinePaths(DirectoryForThisBuild)); if (!DirectoryExists_NoExceptions(ParentDir)) { throw new AutomationException("Not cleaning formal builds, because the parent directory {0} does not exist.", ParentDir); } string MyDir = Path.GetFileName(CombinePaths(DirectoryForThisBuild)); int CLStart = MyDir.IndexOf(CLString); if (CLStart < 0) { throw new AutomationException("Not cleaning formal builds, because the directory {0} does not contain the CL {1}.", DirectoryForThisBuild, CLString); } string StartString = MyDir.Substring(0, CLStart); string EndString = MyDir.Substring(CLStart + CLString.Length); DirectoryInfo DirInfo = new DirectoryInfo(ParentDir); var TopLevelDirs = DirInfo.GetDirectories(); Log("Looking for directories to delete in {0} {1} dirs", ParentDir, TopLevelDirs.Length); foreach (var TopLevelDir in TopLevelDirs) { if (DirectoryExists_NoExceptions(TopLevelDir.FullName)) { var JustDir = Path.GetFileName(CombinePaths(TopLevelDir.FullName)); if (JustDir.StartsWith(StartString, StringComparison.InvariantCultureIgnoreCase) && (String.IsNullOrEmpty(EndString) || JustDir.EndsWith(EndString, StringComparison.InvariantCultureIgnoreCase))) { string CLPart = JustDir.Substring(StartString.Length, JustDir.Length - StartString.Length - EndString.Length); if (!CLPart.Contains("-") && !CLPart.Contains("+")) { DirectoryInfo ThisDirInfo = new DirectoryInfo(TopLevelDir.FullName); bool bOld = false; if ((DateTime.UtcNow - ThisDirInfo.CreationTimeUtc).TotalDays > MaximumDaysToKeepTempStorage) { bOld = true; } if (bOld) { Log("Deleting temp storage directory {0}, because it is more than {1} days old.", TopLevelDir.FullName, MaximumDaysToKeepTempStorage); DeleteDirectory_NoExceptions(true, TopLevelDir.FullName); } else { Log("Not Deleteing temp storage directory {0}, because it is less than {1} days old.", TopLevelDir.FullName, MaximumDaysToKeepTempStorage); } } else { Log("skipping {0}, because the CL part {1} had weird characters", JustDir, CLPart); } } else { Log("skipping {0}, because it didn't start with {1} or end with {2}", JustDir, StartString, EndString); } } } } catch (Exception Ex) { Log(System.Diagnostics.TraceEventType.Warning, "Unable to Clean Directory with DirectoryForThisBuild {0}", DirectoryForThisBuild); Log(System.Diagnostics.TraceEventType.Warning, " Exception was {0}", LogUtils.FormatException(Ex)); } } } /// /// Use with "using" syntax to push and pop directories in a convenient, exception-safe way /// public class PushedDirectory : IDisposable { public PushedDirectory(string DirectoryName) { CommandUtils.PushDir(DirectoryName); } public void Dispose() { CommandUtils.PopDir(); GC.SuppressFinalize(this); } } /// /// Helper class to associate a file and its contents /// public class EMSFileInfo { public string FileName { get; set; } public byte[] Bytes { get; set; } } /// /// Wrapper class for the enumerate files JSON response from MCP /// [DataContract] public sealed class EnumerationResponse { [DataMember(Name = "doNotCache", IsRequired = true)] public Boolean DoNotCache { get; set; } [DataMember(Name = "uniqueFilename", IsRequired = true)] public string UniqueFilename { get; set; } [DataMember(Name = "filename", IsRequired = true)] public string Filename { get; set; } [DataMember(Name = "hash", IsRequired = true)] public string Hash { get; set; } [DataMember(Name = "length", IsRequired = true)] public long Length { get; set; } [DataMember(Name = "uploaded", IsRequired = true)] public string Uploaded { get; set; } } /// /// Code signing /// [Help("NoSign", "Skips signing of code/content files.")] public class CodeSign { /// /// If so, what is the signing identity to search for? /// public static string SigningIdentity = "Epic Games"; /// /// Should we use the machine store? /// public static bool bUseMachineStoreInsteadOfUserStore = false; /// /// How long to keep re-trying code signing for /// public static TimeSpan CodeSignTimeOut = new TimeSpan(0, 3, 0); // Keep trying to sign one file for up to 3 minutes /// /// Code signs the specified file /// public static void SignSingleExecutableIfEXEOrDLL(string Filename, bool bIgnoreExtension = false) { if (UnrealBuildTool.Utils.IsRunningOnMono) { CommandUtils.Log(TraceEventType.Information, String.Format("Can't sign '{0}', we are running under mono.", Filename)); return; } if (!CommandUtils.FileExists(Filename)) { throw new AutomationException("Can't sign '{0}', file does not exist.", Filename); } // Make sure the file isn't read-only FileInfo TargetFileInfo = new FileInfo(Filename); // Executable extensions List Extensions = new List(); Extensions.Add(".dll"); Extensions.Add(".exe"); bool IsExecutable = bIgnoreExtension; foreach (var Ext in Extensions) { if (TargetFileInfo.FullName.EndsWith(Ext, StringComparison.InvariantCultureIgnoreCase)) { IsExecutable = true; break; } } if (!IsExecutable) { CommandUtils.Log(TraceEventType.Verbose, String.Format("Won't sign '{0}', not an executable.", TargetFileInfo.FullName)); return; } string SignToolName = null; if (WindowsPlatform.Compiler == WindowsCompiler.VisualStudio2013) { SignToolName = "C:/Program Files (x86)/Windows Kits/8.1/bin/x86/SignTool.exe"; } else if (WindowsPlatform.Compiler == WindowsCompiler.VisualStudio2012) { SignToolName = "C:/Program Files (x86)/Windows Kits/8.0/bin/x86/SignTool.exe"; } if (!File.Exists(SignToolName)) { throw new AutomationException("SignTool not found at '{0}' (are you missing the Windows SDK?)", SignToolName); } TargetFileInfo.IsReadOnly = false; // Code sign the executable string TimestampServer = "http://timestamp.verisign.com/scripts/timestamp.dll"; string SpecificStoreArg = bUseMachineStoreInsteadOfUserStore ? " /sm" : ""; //@TODO: Verbosity choosing // /v will spew lots of info // /q does nothing on success and minimal output on failure string CodeSignArgs = String.Format("sign{0} /a /n \"{1}\" /t {2} /v {3}", SpecificStoreArg, SigningIdentity, TimestampServer, TargetFileInfo.FullName); DateTime StartTime = DateTime.Now; int NumTrials = 0; for (; ; ) { ProcessResult Result = CommandUtils.Run(SignToolName, CodeSignArgs, null, CommandUtils.ERunOptions.AllowSpew); ++NumTrials; if (Result.ExitCode != 1) { if (Result.ExitCode == 2) { CommandUtils.Log(TraceEventType.Error, String.Format("Signtool returned a warning.")); } // Success! break; } else { // Keep retrying until we run out of time TimeSpan RunTime = DateTime.Now - StartTime; if (RunTime > CodeSignTimeOut) { throw new AutomationException("Failed to sign executable '{0}' {1} times over a period of {2}", TargetFileInfo.FullName, NumTrials, RunTime); } } } } /// /// Code signs the specified file or folder /// public static void SignMacFileOrFolder(string InPath, bool bIgnoreExtension = false) { bool bExists = CommandUtils.FileExists(InPath) || CommandUtils.DirectoryExists(InPath); if (!bExists) { throw new AutomationException("Can't sign '{0}', file or folder does not exist.", InPath); } // Executable extensions List Extensions = new List(); Extensions.Add(".dylib"); Extensions.Add(".app"); bool IsExecutable = bIgnoreExtension || (Path.GetExtension(InPath) == "" && !InPath.EndsWith("PkgInfo")); foreach (var Ext in Extensions) { if (InPath.EndsWith(Ext, StringComparison.InvariantCultureIgnoreCase)) { IsExecutable = true; break; } } if (!IsExecutable) { CommandUtils.Log(TraceEventType.Verbose, String.Format("Won't sign '{0}', not an executable.", InPath)); return; } string SignToolName = "codesign"; string CodeSignArgs = String.Format("-f --deep -s \"{0}\" -v \"{1}\"", "Developer ID Application", InPath); DateTime StartTime = DateTime.Now; int NumTrials = 0; for (; ; ) { ProcessResult Result = CommandUtils.Run(SignToolName, CodeSignArgs, null, CommandUtils.ERunOptions.AllowSpew); int ExitCode = Result.ExitCode; ++NumTrials; if (ExitCode == 0) { // Success! break; } else { // Keep retrying until we run out of time TimeSpan RunTime = DateTime.Now - StartTime; if (RunTime > CodeSignTimeOut) { throw new AutomationException("Failed to sign '{0}' {1} times over a period of {2}", InPath, NumTrials, RunTime); } } } } /// /// Codesigns multiple files, but skips anything that's not an EXE or DLL file /// Will automatically skip signing if -NoSign is specified in the command line. /// /// List of files to sign public static void SignMultipleIfEXEOrDLL(BuildCommand Command, List Files) { if (!Command.ParseParam("NoSign")) { CommandUtils.Log("Signing up to {0} files...", Files.Count); UnrealBuildTool.UnrealTargetPlatform TargetPlatform = UnrealBuildTool.ExternalExecution.GetRuntimePlatform(); if (TargetPlatform == UnrealBuildTool.UnrealTargetPlatform.Mac) { foreach (var File in Files) { SignMacFileOrFolder(File); } } else { foreach (var File in Files) { SignSingleExecutableIfEXEOrDLL(File); } } } else { CommandUtils.Log("Skipping signing {0} files due to -nosign.", Files.Count); } } } }