// Copyright Epic Games, Inc. All Rights Reserved. // This software is provided "as-is," without any express or implied warranty. // In no event shall the author, nor Epic Games, Inc. be held liable for any damages arising from the use of this software. // This software will not be supported. // Use at your own risk. using System; using System.Linq; using System.Threading; using System.Threading.Tasks; using System.Diagnostics; using System.Reflection; using EpicGames.Core; using System.IO; using System.Collections.Generic; using System.Text; using Microsoft.Build.Execution; using Microsoft.Build.Framework.Profiler; using UnrealBuildBase; namespace AutomationToolDriver { public partial class Program { /// /// Parses command line parameter. /// /// Parameter index /// Command line /// Recently parsed command /// True if the parameter has been successfully parsed. private static void ParseParam(string CurrentParam, CommandInfo CurrentCommand) { if (AutomationToolCommandLine.IsParameterIgnored(CurrentParam)) { return; } bool bGlobalParam = AutomationToolCommandLine.TrySetGlobal(CurrentParam); // Global value parameters, handled explicitly string Option_ScriptsForProject = "-ScriptsForProject"; string Option_ScriptDir = "-ScriptDir"; string Option_Telemetry = "-Telemetry"; // The parameter was not found in the list of global parameters, continue looking... if (CurrentParam.StartsWith(Option_ScriptsForProject + "=", StringComparison.InvariantCultureIgnoreCase)) { if (AutomationToolCommandLine.IsSetUnchecked(Option_ScriptsForProject)) { throw new Exception("The -ScriptsForProject argument may only be specified once"); } string ProjectFileName = CurrentParam.Substring(CurrentParam.IndexOf('=') + 1).Replace("\"", ""); if (!File.Exists(ProjectFileName)) { throw new Exception($"Project '{ProjectFileName}' does not exist"); } AutomationToolCommandLine.SetUnchecked(Option_ScriptsForProject, Path.GetFullPath(ProjectFileName)); } else if (CurrentParam.StartsWith(Option_ScriptDir + "=", StringComparison.InvariantCultureIgnoreCase)) { string ScriptDir = CurrentParam.Substring(CurrentParam.IndexOf('=') + 1); if (Directory.Exists(ScriptDir)) { List OutAdditionalScriptDirectories = (List)AutomationToolCommandLine.GetValueUnchecked(Option_ScriptDir) ?? new List(); OutAdditionalScriptDirectories.Add(ScriptDir); AutomationToolCommandLine.SetUnchecked(Option_ScriptDir, OutAdditionalScriptDirectories); Log.TraceVerbose("Found additional script dir: {0}", ScriptDir); } else { throw new Exception($"Specified ScriptDir doesn't exist: {ScriptDir}"); } } else if (CurrentParam.StartsWith(Option_Telemetry + "=", StringComparison.InvariantCultureIgnoreCase)) { string TelemetryPath = CurrentParam.Substring(CurrentParam.IndexOf('=') + 1); AutomationToolCommandLine.SetUnchecked(Option_Telemetry, TelemetryPath); } else if (CurrentParam.StartsWith("-")) { if (CurrentCommand != null) { CurrentCommand.Arguments.Add(CurrentParam.Substring(1)); } else if (!bGlobalParam) { throw new Exception($"Unknown parameter {CurrentParam} in the command line that does not belong to any command."); } } else if (CurrentParam.Contains("=")) { // Environment variable int ValueStartIndex = CurrentParam.IndexOf('=') + 1; string EnvVarName = CurrentParam.Substring(0, ValueStartIndex - 1); if (String.IsNullOrEmpty(EnvVarName)) { throw new Exception($"Unable to parse environment variable that has no name. Error when parsing command line param {CurrentParam}"); } string EnvVarValue = CurrentParam.Substring(ValueStartIndex); Log.TraceLog($"SetEnvVar {EnvVarName}={EnvVarValue}"); Environment.SetEnvironmentVariable(EnvVarName, EnvVarValue); } } private static string ParseString(string Key, string Value) { if (!String.IsNullOrEmpty(Key)) { if (Value == "true" || Value == "false") { return "-" + Key; } else { string param = "-" + Key + "="; if (Value.Contains(" ")) { param += "\"" + Value + "\""; } else { param += Value; } return param; } } else { return Value; } } private static string ParseList(string Key, List Value) { string param = "-" + Key + "="; bool bStart = true; foreach (var Val in Value) { if (!bStart) { param += "+"; } param += Val as string; bStart = false; } return param; } private static void ParseDictionary(Dictionary Value, List Arguments) { foreach (var Pair in Value) { if ((Pair.Value as string) != null && !string.IsNullOrEmpty(Pair.Value as string)) { Arguments.Add(ParseString(Pair.Key, Pair.Value as string)); } else if (Pair.Value.GetType() == typeof(bool)) { if ((bool)Pair.Value) { Arguments.Add("-" + Pair.Key); } } else if ((Pair.Value as List) != null) { Arguments.Add(ParseList(Pair.Key, Pair.Value as List)); } else if ((Pair.Value as Dictionary) != null) { string param = "-" + Pair.Key + "=\""; List Args = new List(); ParseDictionary(Pair.Value as Dictionary, Args); bool bStart = true; foreach (var Arg in Args) { if (!bStart) { param += " "; } param += Arg.Replace("\"", "\'"); bStart = false; } param += "\""; Arguments.Add(param); } } } private static void ParseProfile(ref string[] CommandLine) { // find if there is a profile file to read string Profile = ""; List Arguments = new List(); for (int Index = 0; Index < CommandLine.Length; ++Index) { if (CommandLine[Index].StartsWith("-profile=")) { Profile = CommandLine[Index].Substring(CommandLine[Index].IndexOf('=') + 1); } else { Arguments.Add(CommandLine[Index]); } } if (!string.IsNullOrEmpty(Profile)) { if (File.Exists(Profile)) { // find if the command has been specified var text = File.ReadAllText(Profile); var RawObject = fastJSON.JSON.Instance.Parse(text) as Dictionary; var Params = RawObject["scripts"] as List; foreach (var Script in Params) { string ScriptName = (Script as Dictionary)["script"] as string; if (!string.IsNullOrEmpty(ScriptName) && !Arguments.Contains(ScriptName)) { Arguments.Add(ScriptName); } (Script as Dictionary).Remove("script"); ParseDictionary((Script as Dictionary), Arguments); } } } CommandLine = Arguments.ToArray(); } /// /// Parse the command line and create a list of commands to execute. /// /// Command line public static void ParseCommandLine(string[] Arguments) { AutomationToolCommandLine = new ParsedCommandLine( new Dictionary(StringComparer.InvariantCultureIgnoreCase) { {"-Verbose", "Enables verbose logging"}, {"-VeryVerbose", "Enables very verbose logging"}, {"-TimeStamps", ""}, {"-Submit", "Allows UAT command to submit changes"}, {"-NoSubmit", "Prevents any submit attempts"}, {"-NoP4", "Disables Perforce functionality {default if not run on a build machine}"}, {"-P4", "Enables Perforce functionality {default if run on a build machine}"}, {"-IgnoreDependencies", ""}, {"-Help", "Displays help"}, {"-List", "Lists all available commands"}, {"-NoKill", "Does not kill any spawned processes on exit"}, {"-UTF8Output", ""}, {"-AllowStdOutLogVerbosity", ""}, {"-NoAutoSDK", ""}, {"-Compile", "Force all script modules to be compiled"}, {"-NoCompile", "Do not attempt to compile any script modules - attempts to run with whatever is up to date" }, {"-IgnoreBuildRecords", "Ignore build records (Intermediate/ScriptModule/ProjectName.json) files when determining if script modules are up to date" }, {"-UseLocalBuildStorage", @"Allows you to use local storage for your root build storage dir {default of P:\Builds {on PC} is changed to Engine\Saved\LocalBuilds}. Used for local testing."}, {"-WaitForDebugger", "Waits for a debugger to be attached, and breaks once debugger successfully attached."}, {"-BuildMachine", "" }, {"-WaitForUATMutex", "" } }, new HashSet(){ "-msbuild-verbose" } ); ParseProfile(ref Arguments); Log.TraceInformation("Parsing command line: {0}", CommandLine.FormatCommandLine(Arguments)); CommandInfo CurrentCommand = null; for (int Index = 0; Index < Arguments.Length; ++Index) { // Guard against empty arguments passed as "" on the command line string Param = Arguments[Index]; if(Param.Length > 0) { if (Param.StartsWith("-") || Param.Contains("=")) { ParseParam(Arguments[Index], CurrentCommand); } else { CurrentCommand = new CommandInfo(Arguments[Index]); AutomationToolCommandLine.CommandsToExecute.Add(CurrentCommand); } } } // Validate var Result = AutomationToolCommandLine.CommandsToExecute.Count > 0 || AutomationToolCommandLine.IsSetGlobal("-Help") || AutomationToolCommandLine.IsSetGlobal("-List"); if (AutomationToolCommandLine.CommandsToExecute.Count > 0) { Log.TraceVerbose("Found {0} scripts to execute:", AutomationToolCommandLine.CommandsToExecute.Count); foreach (var Command in AutomationToolCommandLine.CommandsToExecute) { Log.TraceVerbose(" " + Command.ToString()); } } else if (!Result) { throw new Exception("Failed to find scripts to execute in the command line params."); } if (AutomationToolCommandLine.IsSetGlobal("-NoP4") && AutomationToolCommandLine.IsSetGlobal("-P4")) { throw new Exception("'-NoP4' and '-P4' can't be set simultaneously."); } if (AutomationToolCommandLine.IsSetGlobal("-NoSubmit") && AutomationToolCommandLine.IsSetGlobal("-Submit")) { throw new Exception("'-NoSubmit' and '-Submit' can't be set simultaneously."); } } static ParsedCommandLine AutomationToolCommandLine; static StartupTraceListener StartupListener = new StartupTraceListener(); // Do not add [STAThread] here. It will cause deadlocks in platform automation code. public static async Task Main(string[] Arguments) { // Initialize the log system, buffering the output until we can create the log file Log.AddTraceListener(StartupListener); // Populate AutomationToolCommandLine and CommandsToExecute try { ParseCommandLine(Arguments); } catch (Exception Ex) { Log.TraceError("Error: " + Ex.Message); return (int)ExitCode.Error_Arguments; } // Wait for a debugger to be attached if (AutomationToolCommandLine.IsSetGlobal("-WaitForDebugger")) { Console.WriteLine("Waiting for debugger to be attached..."); while (Debugger.IsAttached == false) { Thread.Sleep(100); } Debugger.Break(); } Stopwatch Timer = Stopwatch.StartNew(); // Ensure UTF8Output flag is respected, since we are initializing logging early in the program. if (AutomationToolCommandLine.IsSetGlobal("-UTF8Output")) { Console.OutputEncoding = new System.Text.UTF8Encoding(false, false); } // Parse the log level argument if (AutomationToolCommandLine.IsSetGlobal("-Verbose")) { Log.OutputLevel = LogEventType.Verbose; } if (AutomationToolCommandLine.IsSetGlobal("-VeryVerbose")) { Log.OutputLevel = LogEventType.VeryVerbose; } // Configure log timestamps Log.IncludeTimestamps = AutomationToolCommandLine.IsSetGlobal("-Timestamps"); // Enter the main program section ExitCode ReturnCode = ExitCode.Success; try { // Set the working directory to the Unreal root directory Environment.CurrentDirectory = Path.GetFullPath(Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().GetOriginalLocation()), "..", "..", "..", "..")); // Ensure we can resolve any external assemblies as necessary. string PathToBinariesDotNET = Path.GetDirectoryName(Assembly.GetEntryAssembly().GetOriginalLocation()); AssemblyUtils.InstallAssemblyResolver(PathToBinariesDotNET); AssemblyUtils.InstallRecursiveAssemblyResolver(PathToBinariesDotNET); // Log the operating environment. Since we usually compile to AnyCPU, we may be executed using different system paths under WOW64. Log.TraceVerbose("{2}: Running on {0} as a {1}-bit process.", RuntimePlatform.Current.ToString(), Environment.Is64BitProcess ? 64 : 32, DateTime.UtcNow.ToString("o")); // Log if we're running from the launcher string ExecutingAssemblyLocation = Assembly.GetExecutingAssembly().Location; if (string.Compare(ExecutingAssemblyLocation, Assembly.GetEntryAssembly().GetOriginalLocation(), StringComparison.OrdinalIgnoreCase) != 0) { Log.TraceVerbose("Executed from AutomationToolLauncher ({0})", ExecutingAssemblyLocation); } Log.TraceVerbose("CWD={0}", Environment.CurrentDirectory); // Log the application version FileVersionInfo Version = AssemblyUtils.ExecutableVersion; Log.TraceVerbose("{0} ver. {1}", Version.ProductName, Version.ProductVersion); bool bWaitForUATMutex = AutomationToolCommandLine.IsSetGlobal("-WaitForUATMutex"); // Don't allow simultaneous execution of AT (in the same branch) ReturnCode = await ProcessSingleton.RunSingleInstanceAsync(MainProc, bWaitForUATMutex); } catch (Exception Ex) { Log.TraceError(ExceptionUtils.FormatException(Ex)); } finally { // Write the exit code Log.TraceInformation("AutomationTool executed for {0}", Timer.Elapsed.ToString("h'h 'm'm 's's'")); Log.TraceInformation("AutomationTool exiting with ExitCode={0} ({1})", (int)ReturnCode, ReturnCode); // Can't use NoThrow here because the code logs exceptions. We're shutting down logging! Trace.Close(); } return (int)ReturnCode; } static async Task MainProc() { string ScriptsForProject = (string)AutomationToolCommandLine.GetValueUnchecked("-ScriptsForProject"); List AdditionalScriptDirs = (List) AutomationToolCommandLine.GetValueUnchecked("-ScriptDir"); bool bForceCompile = AutomationToolCommandLine.IsSetGlobal("-Compile"); bool bNoCompile = AutomationToolCommandLine.IsSetGlobal("-NoCompile"); bool bUseBuildRecords = !AutomationToolCommandLine.IsSetGlobal("-IgnoreBuildRecords"); List Commands = AutomationToolCommandLine.IsSetGlobal("-List") ? null : AutomationToolCommandLine.CommandsToExecute; bool bBuildSuccess; HashSet ScriptModuleAssemblyPaths = InitializeScriptModules( ScriptsForProject, AdditionalScriptDirs, bForceCompile, bNoCompile, bUseBuildRecords, out bBuildSuccess); if (!bBuildSuccess) { return ExitCode.Error_Unknown; } // Load AutomationUtils.Automation.dll FileReference AssemblyPath = ScriptModuleAssemblyPaths.FirstOrDefault(x => x.GetFileNameWithoutExtension().Contains("AutomationUtils.Automation")); Assembly AutomationUtilsAssembly = AssemblyPath != null ? Assembly.LoadFrom(AssemblyPath.FullName) : null; if (AutomationUtilsAssembly == null) { throw new Exception("Did not find an AutomationUtils.Automation.dll"); } // Call into AutomationTool.Automation.Process() Type AutomationTools_Automation = AutomationUtilsAssembly.GetType("AutomationTool.Automation"); MethodInfo Automation_Process = AutomationTools_Automation.GetMethod("ProcessAsync"); return await (Task) Automation_Process.Invoke(null, new object[] {AutomationToolCommandLine, StartupListener, ScriptModuleAssemblyPaths}); } } }