From ea34b09df362e7707cf156bd42febac98aaa90e8 Mon Sep 17 00:00:00 2001 From: Ben Marsh Date: Wed, 28 Apr 2021 11:27:19 -0400 Subject: [PATCH] BuildGraph: Add tasks for running Git and Docker, and add support for deleting directories from tasks. [CL 16144334 by Ben Marsh in ue5-main branch] --- .../AutomationUtils/CommandUtils.cs | 34 +++++- .../AutomationUtils/ProcessUtils.cs | 11 +- .../BuildGraph/Tasks/DeleteTask.cs | 66 ++++++++---- .../BuildGraph/Tasks/DockerTask.cs | 102 ++++++++++++++++++ .../BuildGraph/Tasks/GitTask.cs | 102 ++++++++++++++++++ 5 files changed, 288 insertions(+), 27 deletions(-) create mode 100644 Engine/Source/Programs/AutomationTool/BuildGraph/Tasks/DockerTask.cs create mode 100644 Engine/Source/Programs/AutomationTool/BuildGraph/Tasks/GitTask.cs diff --git a/Engine/Source/Programs/AutomationTool/AutomationUtils/CommandUtils.cs b/Engine/Source/Programs/AutomationTool/AutomationUtils/CommandUtils.cs index 7c3adbb97716..a1311d9eecad 100644 --- a/Engine/Source/Programs/AutomationTool/AutomationUtils/CommandUtils.cs +++ b/Engine/Source/Programs/AutomationTool/AutomationUtils/CommandUtils.cs @@ -2466,6 +2466,38 @@ namespace AutomationTool Callback(); } } + + public static FileReference FindToolInPath(string ToolName) + { + string PathVariable = Environment.GetEnvironmentVariable("PATH"); + foreach (string PathEntry in PathVariable.Split(Path.PathSeparator)) + { + try + { + DirectoryReference PathDir = new DirectoryReference(PathEntry); + if (HostPlatform.Current.HostEditorPlatform == UnrealTargetPlatform.Win64) + { + FileReference ToolFile = FileReference.Combine(PathDir, $"{ToolName}.exe"); + if (FileReference.Exists(ToolFile)) + { + return ToolFile; + } + } + else + { + FileReference ToolFile = FileReference.Combine(PathDir, ToolName); + if (FileReference.Exists(ToolFile)) + { + return ToolFile; + } + } + } + catch + { + } + } + return null; + } } /// @@ -3127,7 +3159,7 @@ namespace AutomationTool { FinalFiles.Add(new FileReference(TargetFileInfo)); } - } + } CodeSignWindows.Sign(FinalFiles, CodeSignWindows.SignatureType.SHA1); CodeSignWindows.Sign(FinalFiles.Where(x => !x.HasExtension(".msi")).ToList(), CodeSignWindows.SignatureType.SHA256); // MSI files can only have one signature; prefer SHA1 for compatibility } diff --git a/Engine/Source/Programs/AutomationTool/AutomationUtils/ProcessUtils.cs b/Engine/Source/Programs/AutomationTool/AutomationUtils/ProcessUtils.cs index 38ff5f6aed74..f8325c3b8b3a 100644 --- a/Engine/Source/Programs/AutomationTool/AutomationUtils/ProcessUtils.cs +++ b/Engine/Source/Programs/AutomationTool/AutomationUtils/ProcessUtils.cs @@ -54,7 +54,7 @@ namespace AutomationTool /// Creates a new process and adds it to the tracking list. /// /// New Process objects - public static IProcessResult CreateProcess(string AppName, bool bAllowSpew, bool bCaptureSpew, Dictionary Env = null, LogEventType SpewVerbosity = LogEventType.Console, ProcessResult.SpewFilterCallbackType SpewFilterCallback = null) + public static IProcessResult CreateProcess(string AppName, bool bAllowSpew, bool bCaptureSpew, Dictionary Env = null, LogEventType SpewVerbosity = LogEventType.Console, ProcessResult.SpewFilterCallbackType SpewFilterCallback = null, string WorkingDir = null) { var NewProcess = HostPlatform.Current.CreateProcess(AppName); if (Env != null) @@ -71,6 +71,11 @@ namespace AutomationTool } } } + if (WorkingDir != null) + { + NewProcess.StartInfo.WorkingDirectory = WorkingDir; + } + var Result = new ProcessResult(AppName, NewProcess, bAllowSpew, bCaptureSpew, SpewVerbosity: SpewVerbosity, InSpewFilterCallback: SpewFilterCallback); AddProcess(Result); return Result; @@ -810,7 +815,7 @@ namespace AutomationTool /// Environment to pass to program. /// Callback to filter log spew before output. /// Object containing the exit code of the program as well as it's stdout output. - public static IProcessResult Run(string App, string CommandLine = null, string Input = null, ERunOptions Options = ERunOptions.Default, Dictionary Env = null, ProcessResult.SpewFilterCallbackType SpewFilterCallback = null, string Identifier = null) + public static IProcessResult Run(string App, string CommandLine = null, string Input = null, ERunOptions Options = ERunOptions.Default, Dictionary Env = null, ProcessResult.SpewFilterCallbackType SpewFilterCallback = null, string Identifier = null, string WorkingDir = null) { App = ConvertSeparators(PathSeparator.Default, App); @@ -840,7 +845,7 @@ namespace AutomationTool LogWithVerbosity(SpewVerbosity,"Running: " + App + " " + (String.IsNullOrEmpty(CommandLine) ? "" : CommandLine)); } - IProcessResult Result = ProcessManager.CreateProcess(App, Options.HasFlag(ERunOptions.AllowSpew), !Options.HasFlag(ERunOptions.NoStdOutCapture), Env, SpewVerbosity: SpewVerbosity, SpewFilterCallback: SpewFilterCallback); + IProcessResult Result = ProcessManager.CreateProcess(App, Options.HasFlag(ERunOptions.AllowSpew), !Options.HasFlag(ERunOptions.NoStdOutCapture), Env, SpewVerbosity: SpewVerbosity, SpewFilterCallback: SpewFilterCallback, WorkingDir: WorkingDir); using (LogIndentScope Scope = Options.HasFlag(ERunOptions.AllowSpew) ? new LogIndentScope(" ") : null) { Process Proc = Result.ProcessObject; diff --git a/Engine/Source/Programs/AutomationTool/BuildGraph/Tasks/DeleteTask.cs b/Engine/Source/Programs/AutomationTool/BuildGraph/Tasks/DeleteTask.cs index 4b99f64fc902..9392fa93b532 100644 --- a/Engine/Source/Programs/AutomationTool/BuildGraph/Tasks/DeleteTask.cs +++ b/Engine/Source/Programs/AutomationTool/BuildGraph/Tasks/DeleteTask.cs @@ -21,9 +21,15 @@ namespace BuildGraph.Tasks /// /// List of file specifications separated by semicolons (for example, *.cpp;Engine/.../*.bat), or the name of a tag set /// - [TaskParameter(ValidationType = TaskParameterValidationType.FileSpec)] + [TaskParameter(Optional = true, ValidationType = TaskParameterValidationType.FileSpec)] public string Files; + /// + /// List of directory names + /// + [TaskParameter(Optional = true)] + public string Directories; + /// /// Whether to delete empty directories after deleting the files. Defaults to true. /// @@ -59,35 +65,49 @@ namespace BuildGraph.Tasks /// Mapping from tag names to the set of files they include public override void Execute(JobContext Job, HashSet BuildProducts, Dictionary> TagNameToFileSet) { - // Find all the referenced files and delete them - HashSet Files = ResolveFilespec(CommandUtils.RootDirectory, Parameters.Files, TagNameToFileSet); - foreach(FileReference File in Files) + if (Parameters.Files != null) { - if (!InternalUtils.SafeDeleteFile(File.FullName)) + // Find all the referenced files and delete them + HashSet Files = ResolveFilespec(CommandUtils.RootDirectory, Parameters.Files, TagNameToFileSet); + foreach (FileReference File in Files) { - CommandUtils.LogWarning("Couldn't delete file {0}", File.FullName); + if (!InternalUtils.SafeDeleteFile(File.FullName)) + { + CommandUtils.LogWarning("Couldn't delete file {0}", File.FullName); + } + } + + // Try to delete all the parent directories. Keep track of the directories we've already deleted to avoid hitting the disk. + if (Parameters.DeleteEmptyDirectories) + { + // Find all the directories that we're touching + HashSet ParentDirectories = new HashSet(); + foreach (FileReference File in Files) + { + ParentDirectories.Add(File.Directory); + } + + // Recurse back up from each of those directories to the root folder + foreach (DirectoryReference ParentDirectory in ParentDirectories) + { + for (DirectoryReference CurrentDirectory = ParentDirectory; CurrentDirectory != CommandUtils.RootDirectory; CurrentDirectory = CurrentDirectory.ParentDirectory) + { + if (!TryDeleteEmptyDirectory(CurrentDirectory)) + { + break; + } + } + } } } - - // Try to delete all the parent directories. Keep track of the directories we've already deleted to avoid hitting the disk. - if(Parameters.DeleteEmptyDirectories) + if (Parameters.Directories != null) { - // Find all the directories that we're touching - HashSet ParentDirectories = new HashSet(); - foreach(FileReference File in Files) + foreach (string Directory in Parameters.Directories.Split(';')) { - ParentDirectories.Add(File.Directory); - } - - // Recurse back up from each of those directories to the root folder - foreach(DirectoryReference ParentDirectory in ParentDirectories) - { - for(DirectoryReference CurrentDirectory = ParentDirectory; CurrentDirectory != CommandUtils.RootDirectory; CurrentDirectory = CurrentDirectory.ParentDirectory) + if (!String.IsNullOrEmpty(Directory)) { - if(!TryDeleteEmptyDirectory(CurrentDirectory)) - { - break; - } + DirectoryReference FullDir = new DirectoryReference(Directory); + FileUtils.ForceDeleteDirectory(FullDir); } } } diff --git a/Engine/Source/Programs/AutomationTool/BuildGraph/Tasks/DockerTask.cs b/Engine/Source/Programs/AutomationTool/BuildGraph/Tasks/DockerTask.cs new file mode 100644 index 000000000000..4db3aa909a02 --- /dev/null +++ b/Engine/Source/Programs/AutomationTool/BuildGraph/Tasks/DockerTask.cs @@ -0,0 +1,102 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +using EpicGames.Core; +using System; +using System.Collections.Generic; +using System.Text; +using System.Xml; + +namespace AutomationTool.Tasks +{ + /// + /// Parameters for a Docker task + /// + public class DockerTaskParameters + { + /// + /// Docker command line arguments + /// + [TaskParameter(Optional = true)] + public string Arguments; + + /// + /// Base directory for running the command + /// + [TaskParameter(Optional = true)] + public string BaseDir; + + /// + /// The minimum exit code, which is treated as an error. + /// + [TaskParameter(Optional = true)] + public int ErrorLevel = 1; + } + + /// + /// Spawns Docker and waits for it to complete. + /// + [TaskElement("Docker", typeof(DockerTaskParameters))] + public class DockerTask : CustomTask + { + /// + /// Parameters for this task + /// + DockerTaskParameters Parameters; + + /// + /// Construct a Docker task + /// + /// Parameters for the task + public DockerTask(DockerTaskParameters InParameters) + { + Parameters = InParameters; + } + + /// + /// Execute the task. + /// + /// Information about the current job + /// Set of build products produced by this node. + /// Mapping from tag names to the set of files they include + public override void Execute(JobContext Job, HashSet BuildProducts, Dictionary> TagNameToFileSet) + { + FileReference ToolFile = CommandUtils.FindToolInPath("docker"); + if(ToolFile == null) + { + throw new AutomationException("Unable to find path to Docker. Check you have it installed, and it is on your PATH."); + } + + IProcessResult Result = CommandUtils.Run(ToolFile.FullName, Parameters.Arguments, WorkingDir: Parameters.BaseDir); + if (Result.ExitCode < 0 || Result.ExitCode >= Parameters.ErrorLevel) + { + throw new AutomationException("Docker terminated with an exit code indicating an error ({0})", Result.ExitCode); + } + } + + /// + /// Output this task out to an XML writer. + /// + public override void Write(XmlWriter Writer) + { + Write(Writer, Parameters); + } + + /// + /// Find all the tags which are used as inputs to this task + /// + /// The tag names which are read by this task + public override IEnumerable FindConsumedTagNames() + { + yield break; + } + + /// + /// Find all the tags which are modified by this task + /// + /// The tag names which are modified by this task + public override IEnumerable FindProducedTagNames() + { + yield break; + } + } +} diff --git a/Engine/Source/Programs/AutomationTool/BuildGraph/Tasks/GitTask.cs b/Engine/Source/Programs/AutomationTool/BuildGraph/Tasks/GitTask.cs new file mode 100644 index 000000000000..b3fb3df76ad0 --- /dev/null +++ b/Engine/Source/Programs/AutomationTool/BuildGraph/Tasks/GitTask.cs @@ -0,0 +1,102 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +using EpicGames.Core; +using System; +using System.Collections.Generic; +using System.Text; +using System.Xml; + +namespace AutomationTool.Tasks +{ + /// + /// Parameters for a Git task + /// + public class GitTaskParameters + { + /// + /// Git command line arguments + /// + [TaskParameter(Optional = true)] + public string Arguments; + + /// + /// Base directory for running the command + /// + [TaskParameter(Optional = true)] + public string BaseDir; + + /// + /// The minimum exit code, which is treated as an error. + /// + [TaskParameter(Optional = true)] + public int ErrorLevel = 1; + } + + /// + /// Spawns Git and waits for it to complete. + /// + [TaskElement("Git", typeof(GitTaskParameters))] + public class GitTask : CustomTask + { + /// + /// Parameters for this task + /// + GitTaskParameters Parameters; + + /// + /// Construct a Git task + /// + /// Parameters for the task + public GitTask(GitTaskParameters InParameters) + { + Parameters = InParameters; + } + + /// + /// Execute the task. + /// + /// Information about the current job + /// Set of build products produced by this node. + /// Mapping from tag names to the set of files they include + public override void Execute(JobContext Job, HashSet BuildProducts, Dictionary> TagNameToFileSet) + { + FileReference ToolFile = CommandUtils.FindToolInPath("git"); + if(ToolFile == null) + { + throw new AutomationException("Unable to find path to Git. Check you have it installed, and it is on your PATH."); + } + + IProcessResult Result = CommandUtils.Run(ToolFile.FullName, Parameters.Arguments, WorkingDir: Parameters.BaseDir); + if (Result.ExitCode < 0 || Result.ExitCode >= Parameters.ErrorLevel) + { + throw new AutomationException("Git terminated with an exit code indicating an error ({0})", Result.ExitCode); + } + } + + /// + /// Output this task out to an XML writer. + /// + public override void Write(XmlWriter Writer) + { + Write(Writer, Parameters); + } + + /// + /// Find all the tags which are used as inputs to this task + /// + /// The tag names which are read by this task + public override IEnumerable FindConsumedTagNames() + { + yield break; + } + + /// + /// Find all the tags which are modified by this task + /// + /// The tag names which are modified by this task + public override IEnumerable FindProducedTagNames() + { + yield break; + } + } +}