Files
UnrealEngineUWP/Engine/Source/Programs/AutomationTool/Program.cs

141 lines
5.2 KiB
C#
Raw Normal View History

// Copyright 1998-2017 Epic Games, Inc. All Rights Reserved.
// This software is provided "as-is," without any express or implied warranty.
// In no event shall the author, nor Epic Games, Inc. be held liable for any damages arising from the use of this software.
// This software will not be supported.
// Use at your own risk.
using System;
using System.Threading;
using System.Diagnostics;
using UnrealBuildTool;
using System.Reflection;
using Tools.DotNETCommon;
using System.IO;
namespace AutomationTool
{
public class Program
{
Copying //UE4/Dev-Build to //UE4/Dev-Main (Source: //UE4/Dev-Build @ 2972140) ========================== MAJOR FEATURES + CHANGES ========================== Change 2959679 on 2016/04/28 by Ben.Marsh UGS: Show the original author for changes with a #ROBOMERGE-AUTHOR tag. Change 2959695 on 2016/04/28 by Ben.Marsh UGS: Only filter out changes from by buildmachine that contain the string "CIS Counter". Change 2960798 on 2016/04/29 by Ben.Marsh Remove C++ version of ParallelExecutor. Now implemented in C# as part of UAT. Change 2960928 on 2016/04/29 by Ben.Marsh UGS: Change filter for buildmachine changes to only include rebuilt lightmaps. Change 2963214 on 2016/05/02 by Ben.Marsh BuildGraph: Allow specifying optional dependencies for a node, indicating that the build products from an upstream node are desired, but should not block the node from running. Change 2964454 on 2016/05/03 by Ben.Marsh Change PostBuildInfoTool to PostBadgeStatus, and add position-independent argument parsing. Change 2964533 on 2016/05/03 by Ben.Marsh BuildGraph: Add the ability to generate summary badges from BuildGraph scripts, which can be pushed into a separate database for consumption by UGS. Change 2964852 on 2016/05/03 by Ben.Marsh BuildGraph: Add a task which can submit a set of files to Perforce, optionally creating and using a different workspace to do so. Change 2966856 on 2016/05/04 by Ben.Marsh EC: Allow specifying a filter for the changes considered when looking for the most recent change. Allows filtering out content changes for UGS builds, code-only builds, etc... Change 2966867 on 2016/05/04 by Ben.Marsh EC: Restore code to always set time in CIS state; we never want large builds to trigger off their defined interval. Change 2967504 on 2016/05/05 by Ben.Marsh UAT: Make sure the intermediate directory exists before writing out the list of changes in StreamCopyDescription. Change 2967778 on 2016/05/05 by Ben.Marsh UAT: Detect the P4 environment by querying Perforce for the setting of P4PORT, rather than assuming it's set in an environment variable. Windows stores this setting in the registry rather than the environment, but it's also valid to be set via P4CONFIG. Change 2967815 on 2016/05/05 by Ben.Marsh EC: Copy the initial resource pool setting from the stream settings into an EC property Change 2967873 on 2016/05/05 by Ben.Marsh EC: Allow stream settings to be stored directly in /GUBP_V5/Streams/ rather than having to be in a child property sheet. Change 2969294 on 2016/05/06 by Ben.Marsh EC: Extend ConformResources command to allow updating the pools that resources are assigned to, and to limit the number of machines which are syncing at once. Also added new EC procedure to allow specifying these arguments. Change 2969371 on 2016/05/06 by Ben.Marsh EC: Allow overriding the stream and workspace identifier synced by the builders. Overriding the stream allows syncing a narrower view of files (eg. Dev-Main vs Main), and overriding the workspace identifier allows sharing a workspace between two streams. Change 2970623 on 2016/05/09 by Ben.Marsh UAT: Prevent Ctrl-C handler delegate from being garbage collected and failing to be triggered. Change 2970627 on 2016/05/09 by Ben.Marsh UAT: Don't limit the list of valid target platforms specified on the command line to just those that we have initialized. Ignoring the platform if the SDK is not installed is never what the user wants. Change 2972140 on 2016/05/10 by Ben.Marsh Change 'Engine, Tools and Monolithics' to include QAGame and Template editors, but exclude everything downstream of a trigger. #lockdown Nick.Penwarden [CL 2972146 by Ben Marsh in Main branch]
2016-05-10 08:49:41 -04:00
/// <summary>
/// Keep a persistent reference to the delegate for handling Ctrl-C events. Since it's passed to non-managed code, we have to prevent it from being garbage collected.
/// </summary>
static ProcessManager.CtrlHandlerDelegate CtrlHandlerDelegateInstance = CtrlHandler;
[STAThread]
public static int Main()
{
var CommandLine = SharedUtils.ParseCommandLine();
LogUtils.InitLogging(CommandLine);
ExitCode ReturnCode = ExitCode.Success;
try
{
// ensure we can resolve any external assemblies as necessary.
AssemblyUtils.InstallAssemblyResolver(Path.GetDirectoryName(Assembly.GetEntryAssembly().GetOriginalLocation()));
HostPlatform.Initialize();
Copying //UE4/Dev-Build to //UE4/Main ========================== MAJOR FEATURES + CHANGES ========================== Change 2790858 on 2015/12/04 by Ben.Marsh Fix ERRORLEVEL not being correctly returned by build.bat, causing failed builds to be treated as succeeded builds when launched from Visual Studio. #codereview Wes.Hunt Change 2792683 on 2015/12/07 by Matthew.Griffin Removed some usages of RunningRocket() function Checks for whether engine tools need rebuilding should be ignored in all installed engine builds. New Intermediate folders should be within project directory and engine's should not be removed when engine installed. Change 2794194 on 2015/12/08 by Matthew.Griffin Removed IsRocket function. Changed IsEngineInstalled so that -NotInstalledEngine will always disable installed engine behavior. Change 2801483 on 2015/12/14 by Matthew.Griffin Removed uses of UnrealBuildTool.RunningRocket() Mac Tool chain test was down to Engine libraries being pre-compiled Allow Third Party dependencies in any monolithic target, not just games All installed engine builds should use shared build environment Removed test no longer needed now that projects from installed builds have game targets Include engine source if it exists in any installed build to improve intellisense in VCProjects Change 2803276 on 2015/12/15 by Matthew.Griffin Removed Rocket switch from ProjectParams as it can always check Automation.RunningRocket() Removed bGeneratingRocketProjectFiles from ProjectFileGenerator as it can always check UnrealBuildTool.RunningRocket() Changed a few usages to IsEngineInstalled where obvious Change 2814190 on 2016/01/03 by Ben.Marsh Convert ParallelExecutor to managed code, so we can use it trivially from UAT without having to compile with UBT first. Should be moved into UBT at some point. Also add a support class for creating managed child processes, which are terminated automatically if the parent process is closed. Currently only implemented on Windows, using P/Invoke. Also allows setting priority level of the process, which could replace the hacky way that we determine the physical processor count in UBT. Change 2822503 on 2016/01/10 by Wes.Hunt AnalyticsET now uses the Data Router protocol. * Automatically detects configs that try to use the old endpoint and routes them to the new endpoint. This code is temp until internal games merge the code and have a chance to update their configs. * There is a new attribute on BeginSession called "LegacyURL" that is "true" if the code needs to have its configs updated, as well as a LogAnalytics warning. * Updated EngineAnalytics and AutomationAnalytics providers to use the new URL and specify any appropriate new, optional config values. * Update CrashReporter provider to use new URL and simplified it's provider configuration (now uses AnalyticsET config struct directly). Removed recently added GetUE4TypeOverride and moved it to the one place in Engine that needs it. #jira UE-21755 #lockdown nick.penwarden [CL 2822983 by Wes Hunt in Main branch]
2016-01-11 10:12:46 -05:00
Log.TraceVerbose("{2}: Running on {0} as a {1}-bit process.", HostPlatform.Current.GetType().Name, Environment.Is64BitProcess ? 64 : 32, DateTime.UtcNow.ToString("o"));
// Log if we're running from the launcher
var ExecutingAssemblyLocation = Assembly.GetExecutingAssembly().Location;
if (string.Compare(ExecutingAssemblyLocation, Assembly.GetEntryAssembly().GetOriginalLocation(), StringComparison.OrdinalIgnoreCase) != 0)
{
Log.TraceVerbose("Executed from AutomationToolLauncher ({0})", ExecutingAssemblyLocation);
}
Log.TraceVerbose("CWD={0}", Environment.CurrentDirectory);
// Hook up exit callbacks
var Domain = AppDomain.CurrentDomain;
Domain.ProcessExit += Domain_ProcessExit;
Domain.DomainUnload += Domain_ProcessExit;
Copying //UE4/Dev-Build to //UE4/Dev-Main (Source: //UE4/Dev-Build @ 2972140) ========================== MAJOR FEATURES + CHANGES ========================== Change 2959679 on 2016/04/28 by Ben.Marsh UGS: Show the original author for changes with a #ROBOMERGE-AUTHOR tag. Change 2959695 on 2016/04/28 by Ben.Marsh UGS: Only filter out changes from by buildmachine that contain the string "CIS Counter". Change 2960798 on 2016/04/29 by Ben.Marsh Remove C++ version of ParallelExecutor. Now implemented in C# as part of UAT. Change 2960928 on 2016/04/29 by Ben.Marsh UGS: Change filter for buildmachine changes to only include rebuilt lightmaps. Change 2963214 on 2016/05/02 by Ben.Marsh BuildGraph: Allow specifying optional dependencies for a node, indicating that the build products from an upstream node are desired, but should not block the node from running. Change 2964454 on 2016/05/03 by Ben.Marsh Change PostBuildInfoTool to PostBadgeStatus, and add position-independent argument parsing. Change 2964533 on 2016/05/03 by Ben.Marsh BuildGraph: Add the ability to generate summary badges from BuildGraph scripts, which can be pushed into a separate database for consumption by UGS. Change 2964852 on 2016/05/03 by Ben.Marsh BuildGraph: Add a task which can submit a set of files to Perforce, optionally creating and using a different workspace to do so. Change 2966856 on 2016/05/04 by Ben.Marsh EC: Allow specifying a filter for the changes considered when looking for the most recent change. Allows filtering out content changes for UGS builds, code-only builds, etc... Change 2966867 on 2016/05/04 by Ben.Marsh EC: Restore code to always set time in CIS state; we never want large builds to trigger off their defined interval. Change 2967504 on 2016/05/05 by Ben.Marsh UAT: Make sure the intermediate directory exists before writing out the list of changes in StreamCopyDescription. Change 2967778 on 2016/05/05 by Ben.Marsh UAT: Detect the P4 environment by querying Perforce for the setting of P4PORT, rather than assuming it's set in an environment variable. Windows stores this setting in the registry rather than the environment, but it's also valid to be set via P4CONFIG. Change 2967815 on 2016/05/05 by Ben.Marsh EC: Copy the initial resource pool setting from the stream settings into an EC property Change 2967873 on 2016/05/05 by Ben.Marsh EC: Allow stream settings to be stored directly in /GUBP_V5/Streams/ rather than having to be in a child property sheet. Change 2969294 on 2016/05/06 by Ben.Marsh EC: Extend ConformResources command to allow updating the pools that resources are assigned to, and to limit the number of machines which are syncing at once. Also added new EC procedure to allow specifying these arguments. Change 2969371 on 2016/05/06 by Ben.Marsh EC: Allow overriding the stream and workspace identifier synced by the builders. Overriding the stream allows syncing a narrower view of files (eg. Dev-Main vs Main), and overriding the workspace identifier allows sharing a workspace between two streams. Change 2970623 on 2016/05/09 by Ben.Marsh UAT: Prevent Ctrl-C handler delegate from being garbage collected and failing to be triggered. Change 2970627 on 2016/05/09 by Ben.Marsh UAT: Don't limit the list of valid target platforms specified on the command line to just those that we have initialized. Ignoring the platform if the SDK is not installed is never what the user wants. Change 2972140 on 2016/05/10 by Ben.Marsh Change 'Engine, Tools and Monolithics' to include QAGame and Template editors, but exclude everything downstream of a trigger. #lockdown Nick.Penwarden [CL 2972146 by Ben Marsh in Main branch]
2016-05-10 08:49:41 -04:00
HostPlatform.Current.SetConsoleCtrlHandler(CtrlHandlerDelegateInstance);
var Version = AssemblyUtils.ExecutableVersion;
Log.TraceVerbose("{0} ver. {1}", Version.ProductName, Version.ProductVersion);
// Don't allow simultaneous execution of AT (in the same branch)
ReturnCode = InternalUtils.RunSingleInstance(MainProc, CommandLine);
}
catch (AutomationException Ex)
{
Log.TraceError("AutomationTool terminated with exception: {0}", Ex);
ReturnCode = Ex.ErrorCode;
}
catch (Exception Ex)
{
// Catch all exceptions and propagate the ErrorCode if we are given one.
Log.TraceError("AutomationTool terminated with exception: {0}", Ex);
ReturnCode = ExitCode.Error_Unknown;
}
UBT Utils.cs (New logging system) * Allows us to use built-in Trace providers (console, file, etc) directly and still use our custom formatting. * Fat comments explaining why Trace.WriteXXX functions should not be used directly in our system. * Fixes thread safety by using Trace.WriteXXX under the hood after formatting, which uses a global lock (except on Mono, where a bug appears to be preventing this. Simulating the call on that platform). * No need for TraceEvent overloads, which saves us the extra parameter cruft. * Removed non-varargs overloads of Log functions (technically a bit slower, but these are already small messages). * No longer needed VerbosityFilter and ConsoleListener classes. * Avoid calling GetSource() if we aren't outputting the source. * Avoid formatting the string if it won't pass the verbosity level. * Consolidated all of UAT and UBT options into this class, so they could fully share the implementation. UBT BuildConfiguration.cs * Added LogFilename (and --log=<file> arg) that enables logging to a file. * Added static ctor guard that asserts if someone tries to read a config before we have loaded config files and parsed config-override commandlines. It's a poor man's hack, but better than nothing! UBT UEBuildConfiguration.cs * Same static ctor guard as above. UBT UnrealBuildTools.cs (initialization refactoring) * In general I tried to de-mystify some of the rationale behind our startup code via fat comments. * Broke main into 3 stages: 1. "early code" that should not try to read a config value. * Very little code here. Mostly setting the current directory. * Does an early init of logging to ensure logging is around, but config values won't be ready. 2. "Init Configuration code" that loads config files and parses command lines that may override them. * I isolated two locations in startup that parsed long sets of switches and moved ones that trivially affected BuildConfiguration and UEBuildConfiguration in here. Those two locations seemed to have mostly copies of the same switches, indicating serious param parsing issues at some point in time. * This allows switches to override config files more easily than the patchwork of re-parsing that was currently used (particularly for -verbose). * I did a cursory examination of later code that indicated this double (actually, triple) parsing was no longer necessary with the refactors above. Any insight into why things may have ended up this way would be helpful. 3. "Post Init code" that is actually the meat of UBT. * I left this code largely untouched. * Removed 2 of 3 different command line logging statements. * Removed two redundant parses of config overrides (ParseBuildConfigurationFlags). * Guarded all of main in a try/catch block to ensure no exceptions can leak from UBT without returning a valid error code. It ALMOST already did this, but only covered the part surrounded by the Mutex. * There was a perplexing bit that redundantly called XmlConfigLoader.Reset<> (line 683) that I struggled to understand. It turns out UEBuildConfiguration was sensitive to the current directory being set before files were loaded, and the old code called XmlConfigLoader.Init() super early, which required it to be called again after the current directory was set (see UEBuldConfiguration.UEThirdPartySourceDirectory for the cause). After my changes, I verified as best I could that these calls are no longer needed and removed them. XmlConfigLoader.cs * Add support for Properties in XmlConfigLoader. AutomationTool Program.cs * Guard logging shutdown code in try/finally so it can't be missed. AutomationTool Log.cs * Uses new logging system from UBT * Removed unnecessary classes (VerbosityFilter, AutomationConsoleTraceListener, and AutomationFileTraceListener) * Console trace logic is handled by UBT code now, moved UTF8Output handling to InitLogging. * A custom TraceListener for file logging was unnecessary. * Logic to handle creating the log file and retry loops was move into InitLogging, and the result passed to a regular TextFileTraceListener. * Logic to handle copying the log on shutdown was moved to a ShutdownLogging function. #codereview:robert.manuszewski,michael.trepka,kellan.carr [CL 2526245 by Wes Hunt in Main branch]
2015-04-26 18:19:28 -04:00
finally
{
// In all cases, do necessary shut down stuff, but don't let any additional exceptions leak out while trying to shut down.
// Make sure there's no directories on the stack.
NoThrow(() => CommandUtils.ClearDirStack(), "Clear Dir Stack");
// Try to kill process before app domain exits to leave the other KillAll call to extreme edge cases
NoThrow(() => { if (ShouldKillProcesses && !Utils.IsRunningOnMono) ProcessManager.KillAll(); }, "Kill All Processes");
Log.TraceInformation("AutomationTool exiting with ExitCode={0} ({1})", (int)ReturnCode, ReturnCode);
// Can't use NoThrow here because the code logs exceptions. We're shutting down logging!
UBT Utils.cs (New logging system) * Allows us to use built-in Trace providers (console, file, etc) directly and still use our custom formatting. * Fat comments explaining why Trace.WriteXXX functions should not be used directly in our system. * Fixes thread safety by using Trace.WriteXXX under the hood after formatting, which uses a global lock (except on Mono, where a bug appears to be preventing this. Simulating the call on that platform). * No need for TraceEvent overloads, which saves us the extra parameter cruft. * Removed non-varargs overloads of Log functions (technically a bit slower, but these are already small messages). * No longer needed VerbosityFilter and ConsoleListener classes. * Avoid calling GetSource() if we aren't outputting the source. * Avoid formatting the string if it won't pass the verbosity level. * Consolidated all of UAT and UBT options into this class, so they could fully share the implementation. UBT BuildConfiguration.cs * Added LogFilename (and --log=<file> arg) that enables logging to a file. * Added static ctor guard that asserts if someone tries to read a config before we have loaded config files and parsed config-override commandlines. It's a poor man's hack, but better than nothing! UBT UEBuildConfiguration.cs * Same static ctor guard as above. UBT UnrealBuildTools.cs (initialization refactoring) * In general I tried to de-mystify some of the rationale behind our startup code via fat comments. * Broke main into 3 stages: 1. "early code" that should not try to read a config value. * Very little code here. Mostly setting the current directory. * Does an early init of logging to ensure logging is around, but config values won't be ready. 2. "Init Configuration code" that loads config files and parses command lines that may override them. * I isolated two locations in startup that parsed long sets of switches and moved ones that trivially affected BuildConfiguration and UEBuildConfiguration in here. Those two locations seemed to have mostly copies of the same switches, indicating serious param parsing issues at some point in time. * This allows switches to override config files more easily than the patchwork of re-parsing that was currently used (particularly for -verbose). * I did a cursory examination of later code that indicated this double (actually, triple) parsing was no longer necessary with the refactors above. Any insight into why things may have ended up this way would be helpful. 3. "Post Init code" that is actually the meat of UBT. * I left this code largely untouched. * Removed 2 of 3 different command line logging statements. * Removed two redundant parses of config overrides (ParseBuildConfigurationFlags). * Guarded all of main in a try/catch block to ensure no exceptions can leak from UBT without returning a valid error code. It ALMOST already did this, but only covered the part surrounded by the Mutex. * There was a perplexing bit that redundantly called XmlConfigLoader.Reset<> (line 683) that I struggled to understand. It turns out UEBuildConfiguration was sensitive to the current directory being set before files were loaded, and the old code called XmlConfigLoader.Init() super early, which required it to be called again after the current directory was set (see UEBuldConfiguration.UEThirdPartySourceDirectory for the cause). After my changes, I verified as best I could that these calls are no longer needed and removed them. XmlConfigLoader.cs * Add support for Properties in XmlConfigLoader. AutomationTool Program.cs * Guard logging shutdown code in try/finally so it can't be missed. AutomationTool Log.cs * Uses new logging system from UBT * Removed unnecessary classes (VerbosityFilter, AutomationConsoleTraceListener, and AutomationFileTraceListener) * Console trace logic is handled by UBT code now, moved UTF8Output handling to InitLogging. * A custom TraceListener for file logging was unnecessary. * Logic to handle creating the log file and retry loops was move into InitLogging, and the result passed to a regular TextFileTraceListener. * Logic to handle copying the log on shutdown was moved to a ShutdownLogging function. #codereview:robert.manuszewski,michael.trepka,kellan.carr [CL 2526245 by Wes Hunt in Main branch]
2015-04-26 18:19:28 -04:00
LogUtils.ShutdownLogging();
}
// STOP: No code beyond the return statement should go beyond this point!
// Nothing should happen after the finally block above is finished.
return (int)ReturnCode;
}
/// <summary>
/// Wraps an action in an exception block.
/// Ensures individual actions can be performed and exceptions won't prevent further actions from being executed.
/// Useful for shutdown code where shutdown may be in several stages and it's important that all stages get a chance to run.
/// </summary>
/// <param name="Action"></param>
private static void NoThrow(System.Action Action, string ActionDesc)
{
try
{
Action();
}
catch (Exception Ex)
{
Log.TraceError("Exception performing nothrow action \"{0}\": {1}", ActionDesc, LogUtils.FormatException(Ex));
}
}
static bool CtrlHandler(CtrlTypes EventType)
{
Domain_ProcessExit(null, null);
if (EventType == CtrlTypes.CTRL_C_EVENT)
{
// Force exit
Environment.Exit(3);
}
return true;
}
static void Domain_ProcessExit(object sender, EventArgs e)
{
// Kill all spawned processes (Console instead of Log because logging is closed at this time anyway)
if (ShouldKillProcesses && !Utils.IsRunningOnMono)
{
ProcessManager.KillAll();
}
Trace.Close();
}
static ExitCode MainProc(object Param)
{
ExitCode Result = Automation.Process((string[])Param);
ShouldKillProcesses = Automation.ShouldKillProcesses;
return Result;
}
static bool ShouldKillProcesses = true;
}
}