Files
UnrealEngineUWP/Engine/Source/Programs/AutomationTool/Program.cs
Wes Hunt 585af51d15 TempStorage Refactor
GUBP High Level
* Temp Storage is zipped into a single archive per node now. This results in ~75% reduction in temp storage usage and network traffic, not to mention the per-file overhead.
* Temp Storage is in P:\\Builds\\{Game}\\TmpStore instead of P:\\Builds\\{Game}\\GUBP (to facilitate easier cleaning of this new structure).
* Temp Storage nodes are in subdirectories of {Branch}\\{CL}\\{NodeName} now instead of a flat directory structure that was hard to manually sift through.

GUBP Mid Level
* Removed -Store= and -StoreSuffix= test parameters.
* Added -NoZipTempStorage parameter to turn off temp storage zipping if necessary.
* Created GUBP.JobInfo class that collects info about the job as a whole to be passed around by GUBP. Mostly used by any code that need to interact with TempStorage.
* Created TempStorageNodeInfo that describes the necessary parameters to find the temp storage location for a node.
* Fully XML commented TempStorage.cs, and commented internals all major functions.
* Added a bunch of telemetry data for storing, retrieving, and cleaning shared temp storage.

UAT Mid Level
* Fixed a bug in Ionic.Zip that make ExtractAll() not work on Mono, checked in new DLLs.
* Added UAT parameter -UseLocalBuildStorage that allows you to test build storage stuff completely locally. Writes to Engine\\Saved\\LocalBuilds\\...

GUBP Low Level
* Refactored some GUBP startup code so temp vars would be limited in scope. Makes it easier to track the impact of refactoring these things.
* CullNodesForPreflight is only called for preflight builds.
* Refactored TempStorage.FindTempStorageManifests to use new TmpStore structure and harden the brittle string/path parsing it was doing. See the new TempStorage FindMatchingSharedTempStorageNodeCLs().
* Refactored TempStorage Saving and Loading to use XDocument instead of older XmlDocument. Removed a bunch of redundant checks.
* Use StripBaseDirectory and MakeRerootedFilePath to remove the brittle directory manipulation code. Directories no longer require a '/' at the end.
* Removed a few redundant caching layers in cleaning temp storage that try to ensure we don't clean a folder twice. None of them were necessary.
* Removed unused single-threaded copy code from temp storage.
* Updated Temp Storage unit test, and fully commented the logic behind it.

UAT Low Level
* UAT top level exception handler is now a single log line now to help parsers find the error.
* Removed several uses of FormatException as it doesn't display the entire exception chain, and is not as good as the default exception formatter.
* Removed ExceptionToString as it used FormatException, which was not a good precedent.
* Fixed several cases of exception propagation that was not properly chaining the inner exception.
* Refactored ThreadedCopyFiles to use Parallel.For because it was just as fast (if not faster) and much simpler to maintain.
* Removed the suffix from Robust_FileExists_NoExceptions because it's sole purpose in life WAS to throw exceptions!
* Added a bunch of XML doc comments to CommandUtils.
* Modernized some container manipulation and iteration to use IEnumerable and extension methods more appropriately.
* Added several @todos for other minor cleanup stuff that should happen eventually.
* Fixed some uses of String.Compare to use invariant culture.
#codereview:ben.marsh

[CL 2644846 by Wes Hunt in Main branch]
2015-08-05 10:22:11 -04:00

138 lines
5.1 KiB
C#

// Copyright 1998-2015 Epic Games, Inc. All Rights Reserved.
// This software is provided "as-is," without any express or implied warranty.
// In no event shall the author, nor Epic Games, Inc. be held liable for any damages arising from the use of this software.
// This software will not be supported.
// Use at your own risk.
using System;
using System.Threading;
using System.Diagnostics;
using UnrealBuildTool;
using System.Reflection;
using Tools.DotNETCommon;
namespace AutomationTool
{
public class Program
{
[STAThread]
public static int Main()
{
var CommandLine = SharedUtils.ParseCommandLine();
LogUtils.InitLogging(CommandLine);
ErrorCodes ReturnCode = ErrorCodes.Error_Success;
try
{
HostPlatform.Initialize();
Log.TraceVerbose("Running on {0} as a {1}-bit process.", HostPlatform.Current.GetType().Name, Environment.Is64BitProcess ? 64 : 32);
XmlConfigLoader.Init();
// Log if we're running from the launcher
var ExecutingAssemblyLocation = Assembly.GetExecutingAssembly().Location;
if (string.Compare(ExecutingAssemblyLocation, Assembly.GetEntryAssembly().GetOriginalLocation(), StringComparison.OrdinalIgnoreCase) != 0)
{
Log.TraceVerbose("Executed from AutomationToolLauncher ({0})", ExecutingAssemblyLocation);
}
Log.TraceVerbose("CWD={0}", Environment.CurrentDirectory);
// Hook up exit callbacks
var Domain = AppDomain.CurrentDomain;
Domain.ProcessExit += Domain_ProcessExit;
Domain.DomainUnload += Domain_ProcessExit;
HostPlatform.Current.SetConsoleCtrlHandler(CtrlHandler);
var Version = AssemblyUtils.ExecutableVersion;
Log.TraceVerbose("{0} ver. {1}", Version.ProductName, Version.ProductVersion);
// Don't allow simultaneous execution of AT (in the same branch)
InternalUtils.RunSingleInstance(MainProc, CommandLine);
}
catch (Exception Ex)
{
// Catch all exceptions and propagate the ErrorCode if we are given one.
Log.TraceError("AutomationTool terminated with exception: {0}", Ex);
// set the exit code of the process
if (Ex is AutomationException)
{
ReturnCode = (Ex as AutomationException).ErrorCode;
}
else
{
ReturnCode = ErrorCodes.Error_Unknown;
}
}
finally
{
// In all cases, do necessary shut down stuff, but don't let any additional exceptions leak out while trying to shut down.
// Make sure there's no directories on the stack.
NoThrow(() => CommandUtils.ClearDirStack(), "Clear Dir Stack");
// Try to kill process before app domain exits to leave the other KillAll call to extreme edge cases
NoThrow(() => { if (ShouldKillProcesses && !Utils.IsRunningOnMono) ProcessManager.KillAll(); }, "Kill All Processes");
Log.TraceInformation("AutomationTool exiting with ExitCode={0}", ReturnCode);
// Can't use NoThrow here because the code logs exceptions. We're shutting down logging!
LogUtils.ShutdownLogging();
}
// STOP: No code beyond the return statement should go beyond this point!
// Nothing should happen after the finally block above is finished.
return (int)ReturnCode;
}
/// <summary>
/// Wraps an action in an exception block.
/// Ensures individual actions can be performed and exceptions won't prevent further actions from being executed.
/// Useful for shutdown code where shutdown may be in several stages and it's important that all stages get a chance to run.
/// </summary>
/// <param name="Action"></param>
private static void NoThrow(System.Action Action, string ActionDesc)
{
try
{
Action();
}
catch (Exception Ex)
{
Log.TraceError("Exception performing nothrow action \"{0}\": {1}", ActionDesc, LogUtils.FormatException(Ex));
}
}
static bool CtrlHandler(CtrlTypes EventType)
{
Domain_ProcessExit(null, null);
if (EventType == CtrlTypes.CTRL_C_EVENT)
{
// Force exit
Environment.Exit(3);
}
return true;
}
static void Domain_ProcessExit(object sender, EventArgs e)
{
// Kill all spawned processes (Console instead of Log because logging is closed at this time anyway)
Console.WriteLine("Domain_ProcessExit");
if (ShouldKillProcesses && !Utils.IsRunningOnMono)
{
ProcessManager.KillAll();
}
Trace.Close();
}
static void MainProc(object Param)
{
Automation.Process((string[])Param);
ShouldKillProcesses = Automation.ShouldKillProcesses;
}
static bool ShouldKillProcesses = true;
}
}