Integrated scripts for doing simple benchmarking of build steps from Private-Profiling.

Also includes -noshaderddc option for emulating a cold DDC for shaders only.


#rb na
#ROBOMERGE-OWNER: andrew.grant
#ROBOMERGE-AUTHOR: andrew.grant
#ROBOMERGE-SOURCE: CL 11519411 via CL 11519529 via CL 11519553
#ROBOMERGE-BOT: (v654-11333218)

[CL 11524747 by andrew grant in Main branch]

Engine/Source/Programs/AutomationTool/AutomationUtils/UE4Build.cs

@@ -1,4 +1,4 @@
-// Copyright Epic Games, Inc. All Rights Reserved.
+// Copyright Epic Games, Inc. All Rights Reserved.
 using System;
 using System.Collections.Generic;
 using System.Text;
@@ -36,6 +36,11 @@ namespace AutomationTool
 	{
 		private BuildCommand OwnerCommand;
 
+		/// <summary>
+		/// If true we will let UBT build UHT
+		/// </summary>
+		public bool AlwaysBuildUHT { get; set; }
+
 		public bool HasBuildProduct(string InFile)
 		{
 			string File = CommandUtils.CombinePaths(InFile);
@@ -125,7 +130,9 @@ namespace AutomationTool
 
 			ClearExportedXGEXML();
 
-			CommandUtils.RunUBT(CommandUtils.CmdEnv, UBTExecutable: UBTExecutable, Project: UprojectPath, Target: TargetName, Platform: Platform, Config: Config, AdditionalArgs: String.Format("-Manifest={0} -nobuilduht -NoHotReload -xgeexport {1}", CommandUtils.MakePathSafeToUseWithCommandLine(ManifestFile.FullName), AddArgs));
+			string UHTArg = this.AlwaysBuildUHT ? "" : "-nobuilduht";
+
+			CommandUtils.RunUBT(CommandUtils.CmdEnv, UBTExecutable: UBTExecutable, Project: UprojectPath, Target: TargetName, Platform: Platform, Config: Config, AdditionalArgs: String.Format("-Manifest={0} {1} -NoHotReload -xgeexport {2}", CommandUtils.MakePathSafeToUseWithCommandLine(ManifestFile.FullName), UHTArg, AddArgs));
 
 			XGEItem Result = new XGEItem();
 			Result.Platform = Platform;

Engine/Source/Programs/AutomationTool/Scripts/AutomationScripts.Automation.csproj

@@ -61,6 +61,12 @@
     </Compile>
     <Compile Include="AnalyzeThirdPartyLibs.Automation.cs" />
     <Compile Include="ArchiveCommand.Automation.cs" />
+    <Compile Include="BenchmarkBuild\BenchmarkBuild.Automation.cs" />
+    <Compile Include="BenchmarkBuild\BenchmarkBuildTests.cs" />
+    <Compile Include="BenchmarkBuild\BenchmarkCookTask.cs" />
+    <Compile Include="BenchmarkBuild\BenchmarkNopCompileTask.cs" />
+    <Compile Include="BenchmarkBuild\BenchmarkSingleCompileTask.cs" />
+    <Compile Include="BenchmarkBuild\BenchmarkTaskBase.cs" />
     <Compile Include="FinalizeInstalledBuild.cs" />
     <Compile Include="BuildHlslcc.Automation.cs" />
     <Compile Include="ParseMsvcTimingInfo.cs" />

Engine/Source/Programs/AutomationTool/Scripts/BenchmarkBuild/BenchmarkBuild.Automation.cs

@@ -0,0 +1,431 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+using AutomationTool;
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text;
+using System.Threading;
+using System.Threading.Tasks;
+using Tools.DotNETCommon;
+using UnrealBuildTool;
+
+namespace AutomationTool.Benchmark
+{
+
+	[Help("Runs benchmarks and reports overall results")]
+	[Help("Example1: RunUAT BenchmarkBuild -all -project=UE4")]
+	[Help("Example2: RunUAT BenchmarkBuild -allcompile -project=UE4+EngineTest -platform=PS4")]
+	[Help("Example3: RunUAT BenchmarkBuild -editor -client -cook -cooknoshaderddc -cooknoddc -xge -noxge -singlecompile -nopcompile -project=UE4+QAGame+EngineTest -platform=WIn64+PS4+XboxOne+Switch -iterations=3")]
+	[Help("preview", "List everything that will run but don't do it")]
+	[Help("project=project", "Do tests on the specified projec(s)t. E.g. -project=FortniteGame+QAGame")]
+	[Help("all", "Run all the things (except noddc)")]
+	[Help("allcompile", "Run all the compile things")]
+	[Help("editor", "Include the editor for compile tests")]
+	[Help("client", "Include a client for comple tests")]	
+	[Help("platform=platform", "Specify the platform(s) to use for client compilation/cooking, if empty the local platform be used if -client or -cook is specified")]
+	[Help("xge", "Compile with XGE")]
+	[Help("noxge", "Compile without XGE")]
+	[Help("singlecompile", "Do single-file compile")]
+	[Help("nopcompile", "Do nothing-needs-compiled compile")]
+	[Help("cook", "Do a cook for the specified platform")]
+	[Help("cooknoshaderddc", "Do a cook test with no ddc for shaders")]
+	[Help("cooknoddc", "Do a cook test with nodcc (likely to take 10+ hours with cookfortnite)")]
+	[Help("iterations=n", "How many times to perform each step)")]
+	[Help("wait=n", "How many seconds to wait between each step)")]
+	[Help("warmcook", "Do a cook that doesn't count to make sure any remote DDC is full")]
+	[Help("noclean", "Don't build from clean. Mostly just to speed things up when testing")]
+	class BenchmarkBuild : BuildCommand
+	{
+		protected List<BenchmarkTaskBase> Tasks = new List<BenchmarkTaskBase>();
+
+		protected Dictionary<BenchmarkTaskBase, List<TimeSpan>> Results = new Dictionary<BenchmarkTaskBase, List<TimeSpan>>();
+
+		public BenchmarkBuild()
+		{ 
+		}
+
+		public override ExitCode Execute()
+		{
+			bool Preview = ParseParam("preview");
+
+			bool AllThings = ParseParam("all");
+
+			bool AllCompile =  AllThings | ParseParam("allcompile");
+
+			bool DoUE4 = AllCompile | ParseParam("ue4");
+			bool DoBuildEditorTests = AllCompile | ParseParam("editor");
+			bool DoBuildClientTests = AllCompile | ParseParam("client");
+			bool DoNoCompile = AllCompile | ParseParam("nopcompile");
+			bool DoSingleCompile = AllCompile | ParseParam("singlecompile");
+			bool DoXGE = AllCompile | ParseParam("xge");
+			bool DoNoXGE = AllCompile | ParseParam("noxge");
+
+			bool DoCookTests = AllThings | ParseParam("cook");
+			bool DoWarmCook = AllThings | ParseParam("warmcook");
+			bool DoNoShaderDDC = AllThings | ParseParam("cooknoshaderddc");
+			bool DoNoDDC = ParseParam("cooknoddc");
+
+			bool NoClean = ParseParam("noclean");
+			int TimeBetweenTasks = ParseParamInt("wait", 10);
+			int NumLoops = ParseParamInt("iterations", 1);
+
+			// We always build the editor for the platform we're running on
+			UnrealTargetPlatform EditorPlatform = BuildHostPlatform.Current.Platform;
+
+			List<UnrealTargetPlatform> ClientPlatforms = new List<UnrealTargetPlatform>();
+
+			string PlatformArg = ParseParamValue("platform", "");
+
+			if (!string.IsNullOrEmpty(PlatformArg))
+			{
+				var PlatformList = PlatformArg.Split(new[] { '+', ',' }, StringSplitOptions.RemoveEmptyEntries);
+
+				foreach (var Platform in PlatformList)
+				{
+					UnrealTargetPlatform PlatformEnum;
+					if (!UnrealTargetPlatform.TryParse(Platform, out PlatformEnum))
+					{
+						throw new AutomationException("{0} is not a valid Unreal Platform", Platform);
+					}
+
+					ClientPlatforms.Add(PlatformEnum);
+				}
+			}
+			else
+			{
+				ClientPlatforms.Add(EditorPlatform);
+			}
+
+			DoXGE = DoXGE && BenchmarkBuildTask.SupportsXGE;
+
+			// Set this based on whether the user specified -noclean
+			BuildOptions CleanFlag = NoClean ? BuildOptions.None : BuildOptions.Clean;
+
+			List<string> ProjectsToBenchmark = new List<string>();
+
+			string ProjectsArg = ParseParamValue("project", null);
+
+			// Look at the project argument and verify it's a valid uproject
+			if (!string.IsNullOrEmpty(ProjectsArg))
+			{
+				var ProjectList = ProjectsArg.Split(new[] { '+', ',' }, StringSplitOptions.RemoveEmptyEntries);
+
+				foreach (var Project in ProjectList)
+				{
+					if (!string.Equals(Project, "UE4", StringComparison.OrdinalIgnoreCase))
+					{
+						FileReference ProjectFile = ProjectUtils.FindProjectFileFromName(Project);
+
+						if (ProjectFile == null)
+						{
+							throw new AutomationException("Could not find project file for {0}", Project);
+						}
+					}
+
+					ProjectsToBenchmark.Add(Project);
+				}
+			}
+
+			foreach (var Project in ProjectsToBenchmark)
+			{
+				bool IsVanillaUE4 = string.Equals(Project, "UE4", StringComparison.OrdinalIgnoreCase);
+
+				if (DoBuildEditorTests)
+				{
+					if (DoXGE)
+					{
+						Tasks.Add(new BenchmarkBuildTask(Project, "Editor", EditorPlatform, CleanFlag));
+					}
+
+					if (DoNoXGE)
+					{
+						Tasks.Add(new BenchmarkBuildTask(Project, "Editor", EditorPlatform, CleanFlag | BuildOptions.NoXGE));
+					}
+
+					if (DoNoCompile)
+					{
+						
+						// note, don't clean since we build normally then build a single file
+						Tasks.Add(new BenchmarkNopCompileTask(Project, "Editor", EditorPlatform, BuildOptions.None));
+					}
+
+					if (DoSingleCompile)
+					{
+						FileReference SourceFile = FindProjectSourceFile(Project);
+
+						// note, don't clean since we build normally then build again
+						Tasks.Add(new BenchmarkSingleCompileTask(Project, "Editor", EditorPlatform, SourceFile, BuildOptions.None));
+					}
+				}
+
+				if (DoBuildClientTests)
+				{
+					// build a client if the project supports it
+					string TargetName = ProjectSupportsClientBuild(Project) ? "Client" : "Game";
+
+					foreach (var ClientPlatform in ClientPlatforms)
+					{
+						if (DoXGE)
+						{
+							Tasks.Add(new BenchmarkBuildTask(Project, TargetName, ClientPlatform, CleanFlag));
+						}
+
+						if (DoNoXGE)
+						{
+							Tasks.Add(new BenchmarkBuildTask(Project, TargetName, ClientPlatform, CleanFlag | BuildOptions.NoXGE));
+						}
+
+						if (DoNoCompile)
+						{
+							// note, don't clean since we build normally then build again
+							Tasks.Add(new BenchmarkNopCompileTask(Project, TargetName, ClientPlatform, BuildOptions.NoXGE));
+						}
+
+						if (DoSingleCompile)
+						{
+							FileReference SourceFile = FindProjectSourceFile(Project);
+
+							// note, don't clean since we build normally then build a single file
+							Tasks.Add(new BenchmarkSingleCompileTask(Project, TargetName, ClientPlatform, SourceFile, BuildOptions.None));
+						}
+					}
+				}
+
+				// Do cook tests if this is a project and not the engine
+				if (DoCookTests && !IsVanillaUE4)
+				{
+					// Cook a client if the project supports it
+					CookOptions ClientCookOptions = ProjectSupportsClientBuild(Project) ? CookOptions.Client : CookOptions.None;
+
+					foreach (var ClientPlatform in ClientPlatforms)
+					{
+						CookOptions TaskCookOptions = ClientCookOptions | CookOptions.Clean;
+
+						if (DoWarmCook)
+						{
+							TaskCookOptions |= CookOptions.WarmCook;
+						}
+
+						if (DoCookTests)
+						{
+							Tasks.Add(new BenchmarkCookTask(Project, ClientPlatform, TaskCookOptions));
+							TaskCookOptions = ClientCookOptions | CookOptions.Clean;
+						}
+
+						if (DoNoShaderDDC)
+						{
+							Tasks.Add(new BenchmarkCookTask(Project, ClientPlatform, TaskCookOptions | CookOptions.NoShaderDDC));
+							TaskCookOptions = ClientCookOptions | CookOptions.Clean;
+						}
+
+						if (DoNoDDC)
+						{
+							Tasks.Add(new BenchmarkCookTask(Project, ClientPlatform, TaskCookOptions | CookOptions.NoDDC));
+						}
+					}
+				}
+			}
+
+			Log.TraceInformation("Will execute tests:");
+
+			foreach (var Task in Tasks)
+			{
+				Log.TraceInformation("{0}", Task.GetTaskName());
+			}
+
+			if (!Preview)
+			{
+				// create results lists
+				foreach (var Task in Tasks)
+				{
+					Results.Add(Task, new List<TimeSpan>());
+				}
+
+				DateTime StartTime = DateTime.Now;
+
+				for (int i = 0; i < NumLoops; i++)
+				{
+					foreach (var Task in Tasks)
+					{
+						Log.TraceInformation("Starting task {0} (Pass {1})", Task.GetTaskName(), i+1);
+
+						Task.Run();
+
+						Log.TraceInformation("Task {0} took {1}", Task.GetTaskName(), Task.TaskTime.ToString(@"hh\:mm\:ss"));
+
+						Results[Task].Add(Task.TaskTime);
+
+						WriteCSVResults();
+
+						Thread.Sleep(TimeBetweenTasks * 1000);
+					}
+				}
+
+				Log.TraceInformation("**********************************************************************");
+				Log.TraceInformation("Test Results:");
+				foreach (var Task in Tasks)
+				{
+					string TimeString = "";
+
+					IEnumerable<TimeSpan> TaskTimes = Results[Task];
+
+					foreach (var TaskTime in TaskTimes)
+					{
+						if (TimeString.Length > 0)
+						{
+							TimeString += ", ";
+						}
+
+						if (TaskTime == TimeSpan.Zero)
+						{
+							TimeString += "Failed";
+						}
+						else
+						{
+							TimeString += TaskTime.ToString(@"hh\:mm\:ss");
+						}
+					}
+
+					var AvgTimeString = "";
+
+					if (TaskTimes.Count() > 1)
+					{
+						var AvgTime = new TimeSpan(TaskTimes.Sum(T => T.Ticks) / TaskTimes.Count());
+
+						AvgTimeString = string.Format(" (Avg: {0})", AvgTime.ToString(@"hh\:mm\:ss"));
+					}					
+
+					Log.TraceInformation("Task {0}:\t{1}{2}", Task.GetTaskName(), TimeString, AvgTimeString);
+				}
+				Log.TraceInformation("**********************************************************************");
+
+				TimeSpan Elapsed = DateTime.Now - StartTime;
+
+				Log.TraceInformation("Total benchmark time: {0}",  Elapsed.ToString(@"hh\:mm\:ss"));
+
+				WriteCSVResults();
+			}
+
+			return ExitCode.Success;
+		}
+
+		/// <summary>
+		/// Writes our current result to a CSV file. It's expected that this function is called multiple times so results are
+		/// updated as we go
+		/// </summary>
+		void WriteCSVResults()
+		{
+			FileReference FileName = FileReference.Combine(CommandUtils.EngineDirectory, "..", string.Format("{0}_Benchmark.csv", Environment.MachineName));
+
+			Log.TraceInformation("Writing results to {0}", FileName);
+
+			try
+			{
+				List<string> Lines = new List<string>();
+
+				// first line is machine name,,Iteration 1, Iteration 2 etc
+				string FirstLine = string.Format("{0},", Environment.MachineName);
+
+				if (Tasks.Count() > 0)
+				{
+					int Iterations = Results[Tasks.First()].Count();
+
+					if (Iterations > 0)
+					{
+						for (int i = 0; i < Iterations; i++)
+						{
+							FirstLine += ",";
+							FirstLine += string.Format("Iteration {0}", i + 1);
+						}
+					}
+				}
+
+				Lines.Add(FirstLine);
+
+				foreach (var Task in Tasks)
+				{
+					// start with Name, StartTime
+					string Line = string.Format("{0},{1}", Task.GetTaskName(), Task.StartTime.ToString("yyyy-dd-MM HH:mm:ss"));
+
+					// now append all iteration times
+					foreach (TimeSpan TaskTime in Results[Task])
+					{
+						Line += ",";
+						if (TaskTime == TimeSpan.Zero)
+						{
+							Line += "FAILED";
+						}
+						else
+						{
+							Line += TaskTime.ToString(@"hh\:mm\:ss");
+						}
+					}
+
+					Lines.Add(Line);
+				}
+
+				File.WriteAllLines(FileName.FullName, Lines.ToArray());
+			}
+			catch (Exception Ex)
+			{
+				Log.TraceError("Failed to write CSV to {0}. {1}", FileName, Ex);
+			}
+		}
+
+		/// <summary>
+		/// Returns true/false based on whether the project supports a client configuration
+		/// </summary>
+		/// <param name="ProjectName"></param>
+		/// <returns></returns>
+		bool ProjectSupportsClientBuild(string ProjectName)
+		{
+			if (ProjectName.Equals("UE4", StringComparison.OrdinalIgnoreCase))
+			{
+				// UE4
+				return true;
+			}
+
+			FileReference ProjectFile = ProjectUtils.FindProjectFileFromName(ProjectName);
+
+			DirectoryReference SourceDir = DirectoryReference.Combine(ProjectFile.Directory, "Source");
+
+			var Files = DirectoryReference.EnumerateFiles(SourceDir, "*Client.Target.cs");
+
+			return Files.Any();
+		}
+
+
+		/// <summary>
+		/// Returns true/false based on whether the project supports a client configuration
+		/// </summary>
+		/// <param name="ProjectName"></param>
+		/// <returns></returns>
+		FileReference FindProjectSourceFile(string ProjectName)
+		{
+			FileReference SourceFile = null;
+
+			FileReference ProjectFile = ProjectUtils.FindProjectFileFromName(ProjectName);
+
+			if (ProjectFile != null)
+			{
+				DirectoryReference SourceDir = DirectoryReference.Combine(ProjectFile.Directory, "Source", ProjectName);
+
+				var Files = DirectoryReference.EnumerateFiles(SourceDir, "*.cpp", System.IO.SearchOption.AllDirectories);
+
+				SourceFile = Files.FirstOrDefault();
+			}
+
+			if (SourceFile == null)
+			{
+				// touch the write time on a file, first making it writable since it may be under P4
+				SourceFile = FileReference.Combine(CommandUtils.EngineDirectory, "Source/Runtime/Engine/Private/UnrealEngine.cpp");
+			}
+
+			Log.TraceVerbose("Will compile {0} for single-file compilation test for {1}", SourceFile, ProjectName);
+
+			return SourceFile;
+		}
+	}
+}

Engine/Source/Programs/AutomationTool/Scripts/BenchmarkBuild/BenchmarkBuildTests.cs

@@ -0,0 +1,82 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+using AutomationTool;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using UnrealBuildTool;
+
+namespace AutomationTool.Benchmark
+{
+	[Flags]
+	public enum BuildOptions
+	{
+		None = 0,
+		Clean = 1 << 0,
+		NoXGE = 1 << 1,
+	}
+
+	/// <summary>
+	/// Task that builds a target
+	/// </summary>
+	class BenchmarkBuildTask : BenchmarkTaskBase
+	{
+		protected string TaskName;
+
+		private BuildTarget Command;
+
+		public static bool SupportsXGE
+		{
+			get
+			{
+				return BuildHostPlatform.Current.Platform == UnrealTargetPlatform.Win64;
+			}
+		}
+
+		public BenchmarkBuildTask(string InProject, string InTarget, UnrealTargetPlatform InPlatform, BuildOptions InOptions)
+		{
+			bool IsVanillaUE4 = InProject == null || string.Equals(InProject, "UE4", StringComparison.OrdinalIgnoreCase);
+
+			string ModuleName = IsVanillaUE4 ? "UE4" : InProject;
+
+			TaskName = string.Format("{0} {1} {2}", ModuleName, InTarget, InPlatform);
+
+			Command = new BuildTarget();
+			Command.ProjectName = IsVanillaUE4 ? null : InProject;
+			Command.Platforms = InPlatform.ToString();
+			Command.Targets = InTarget;
+			Command.NoTools = true;
+			Command.Clean = InOptions.HasFlag(BuildOptions.Clean);
+
+			List<string> Args = new List<string>();
+
+			bool WithXGE = !InOptions.HasFlag(BuildOptions.NoXGE);
+
+			if (!WithXGE || !SupportsXGE)
+			{
+				Args.Add("NoXGE");
+				TaskName += " (noxge)";
+			}
+			else
+			{
+				TaskName += " (xge)";
+			}
+
+			Command.Params = Args.ToArray();
+		}
+
+		public override string GetTaskName()
+		{
+			return TaskName;
+		}
+
+		protected override bool PerformTask()
+		{
+			ExitCode Result = Command.Execute();
+
+			return Result == ExitCode.Success;
+		}
+	}
+}

Engine/Source/Programs/AutomationTool/Scripts/BenchmarkBuild/BenchmarkCookTask.cs

@@ -0,0 +1,163 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+using AutomationTool;
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using Tools.DotNETCommon;
+using UnrealBuildTool;
+
+namespace AutomationTool.Benchmark
+{
+	[Flags]
+	public enum CookOptions
+	{
+		None = 0,
+		Clean			= 1 << 0,
+		NoDDC			= 1 << 1,
+		NoShaderDDC		= 1 << 2,
+		Client			= 1 << 3,
+		WarmCook		= 1 << 4,
+	}
+
+	class BenchmarkCookTask : BenchmarkTaskBase
+	{
+
+		string TaskName;
+
+		string ProjectName;
+
+		FileReference ProjectFile;
+
+		string CookPlatformName;
+
+		CookOptions Options;
+
+		public BenchmarkCookTask(string InProject, UnrealTargetPlatform InPlatform, CookOptions InOptions)
+		{
+			ProjectName = InProject;
+			Options = InOptions;
+
+			ProjectFile = ProjectUtils.FindProjectFileFromName(ProjectName);
+
+			var PlatformToCookPlatform = new Dictionary<UnrealTargetPlatform, string> {
+				{ UnrealTargetPlatform.Win64, "WindowsClient" },
+				{ UnrealTargetPlatform.Mac, "MacClient" },
+				{ UnrealTargetPlatform.Linux, "LinuxClient" },
+				{ UnrealTargetPlatform.Android, "Android_ASTCClient" }
+			};
+
+			CookPlatformName = InPlatform.ToString();
+
+			if (PlatformToCookPlatform.ContainsKey(InPlatform))
+			{
+				CookPlatformName = PlatformToCookPlatform[InPlatform];
+			}
+
+			TaskName = string.Format("Cook {0} {1}", InProject, CookPlatformName);
+
+			if (Options.HasFlag(CookOptions.NoDDC))
+			{
+				TaskName += " (noddc)";
+			}
+
+			if (Options.HasFlag(CookOptions.NoShaderDDC))
+			{
+				TaskName += " (noshaderddc)";
+			}
+		}
+
+		public override string GetTaskName()
+		{
+			return TaskName;
+		}
+
+		protected override bool PerformPrequisites()
+		{
+			// build editor
+			BuildTarget Command = new BuildTarget();
+			Command.ProjectName = ProjectName;
+			Command.Platforms = BuildHostPlatform.Current.Platform.ToString();
+			Command.Targets = "Editor";
+			
+			if (Command.Execute() != ExitCode.Success)
+			{
+				return false;
+			}
+
+			// Do a cook to make sure the remote ddc is warm?
+			if (Options.HasFlag(CookOptions.WarmCook))
+			{
+				// will throw an exception if it fails
+				CommandUtils.RunCommandlet(ProjectFile, "UE4Editor-Cmd.exe", "Cook", String.Format("-TargetPlatform={0} ", CookPlatformName));
+			}
+
+			if (Options.HasFlag(CookOptions.Clean))
+			{
+				FileReference ProjectFile = ProjectUtils.FindProjectFileFromName(ProjectName);
+
+				List<DirectoryReference> DirsToClear = new List<DirectoryReference>();
+
+				DirectoryReference ProjectDir = ProjectFile.Directory;
+				
+				DirsToClear.Add(DirectoryReference.Combine(ProjectDir, "Saved"));
+				DirsToClear.Add(DirectoryReference.Combine(CommandUtils.EngineDirectory, "DerivedDataCache"));
+				DirsToClear.Add(DirectoryReference.Combine(ProjectDir, "DerivedDataCache"));
+
+				string LocalDDC = Environment.GetEnvironmentVariable("UE-LocalDataCachePath");
+
+				if (!string.IsNullOrEmpty(LocalDDC) && Directory.Exists(LocalDDC))
+				{
+					DirsToClear.Add(new DirectoryReference(LocalDDC));
+				}
+
+				foreach (var Dir in DirsToClear)
+				{
+					try
+					{
+						if (DirectoryReference.Exists(Dir))
+						{
+							Log.TraceInformation("Removing {0}", Dir);
+							DirectoryReference.Delete(Dir, true);
+						}
+					}
+					catch (Exception Ex)
+					{
+						Log.TraceWarning("Failed to remove path {0}. {1}", Dir.FullName, Ex.Message);
+					}
+				}
+			}
+
+			return true;
+		}
+
+		protected override bool PerformTask()
+		{
+			List<string> ExtraArgs = new List<string>();
+			
+			if (Options.HasFlag(CookOptions.Client))
+			{
+				ExtraArgs.Add("client");
+			}
+
+			if (Options.HasFlag(CookOptions.NoShaderDDC))
+			{
+				ExtraArgs.Add("noshaderddc");
+				//ExtraArgs.Add("noxgeshadercompile");
+			}
+
+			if (Options.HasFlag(CookOptions.NoDDC))
+			{
+				ExtraArgs.Add("ddc=noshared");
+			}			
+
+			// will throw an exception if it fails
+			CommandUtils.RunCommandlet(ProjectFile, "UE4Editor-Cmd.exe", "Cook", String.Format("-TargetPlatform={0} -{1}", CookPlatformName, string.Join(" -", ExtraArgs)));
+
+			return true;
+		}
+	}
+}

Engine/Source/Programs/AutomationTool/Scripts/BenchmarkBuild/BenchmarkNopCompileTask.cs

@@ -0,0 +1,38 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+using AutomationTool;
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using Tools.DotNETCommon;
+using UnrealBuildTool;
+
+namespace AutomationTool.Benchmark
+{
+	class BenchmarkNopCompileTask : BenchmarkBuildTask
+	{
+		BenchmarkBuildTask PreTask;
+
+		public BenchmarkNopCompileTask(string InProject, string InTarget, UnrealTargetPlatform InPlatform, BuildOptions InOptions)
+			: base(InProject, InTarget, InPlatform, InOptions)
+		{
+			PreTask = new BenchmarkBuildTask(InProject, InTarget, InPlatform, InOptions);
+			TaskName = TaskName + " (nopcompile)";
+		}
+
+		protected override bool PerformPrequisites()
+		{
+			if (!base.PerformPrequisites())
+			{
+				return false;
+			}
+
+			PreTask.Run();
+						
+			return !PreTask.Failed;
+		}
+	}
+}

Engine/Source/Programs/AutomationTool/Scripts/BenchmarkBuild/BenchmarkSingleCompileTask.cs

@@ -0,0 +1,57 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+using AutomationTool;
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using Tools.DotNETCommon;
+using UnrealBuildTool;
+
+namespace AutomationTool.Benchmark
+{
+	class BenchmarkSingleCompileTask : BenchmarkBuildTask
+	{
+		BenchmarkBuildTask PreTask;
+
+		FileReference SourceFile;
+
+		public BenchmarkSingleCompileTask(string InProject, string InTarget, UnrealTargetPlatform InPlatform, FileReference InSourceFile, BuildOptions InOptions)
+			: base(InProject, InTarget, InPlatform, InOptions)
+		{
+			PreTask = new BenchmarkBuildTask(InProject, InTarget, InPlatform, InOptions);
+			SourceFile = InSourceFile;
+			TaskName = TaskName + " (singlecompile)";
+		}
+
+		protected override bool PerformPrequisites()
+		{
+			if (!base.PerformPrequisites())
+			{
+				return false;
+			}
+
+			PreTask.Run();
+
+			FileInfo Fi = SourceFile.ToFileInfo();
+
+			bool ReadOnly = Fi.IsReadOnly;
+
+			if (ReadOnly)
+			{
+				Fi.IsReadOnly = false;
+			}
+
+			Fi.LastWriteTime = DateTime.Now;
+
+			if (ReadOnly)
+			{
+				Fi.IsReadOnly = true;
+			}
+
+			return !PreTask.Failed;
+		}
+	}
+}

Engine/Source/Programs/AutomationTool/Scripts/BenchmarkBuild/BenchmarkTaskBase.cs

@@ -0,0 +1,113 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using Tools.DotNETCommon;
+
+namespace AutomationTool.Benchmark
+{
+	/// <summary>
+	/// Base class for running tasks
+	/// </summary>
+	abstract class BenchmarkTaskBase
+	{
+		/// <summary>
+		/// True or false based on whether the task failed
+		/// </summary>
+		public bool Failed { get; protected set; }
+
+		/// <summary>
+		/// Failure message
+		/// </summary>
+		public string FailureString { get; protected set; }
+
+		/// <summary>
+		/// Don't report this test
+		/// </summary>
+		public bool SkipReport { get; protected set; }
+
+		/// <summary>
+		/// Time the task took (does not include prequisites)
+		/// </summary>
+		public TimeSpan TaskTime { get; protected set; }
+
+		/// <summary>
+		/// Time the task started
+		/// </summary>
+		public DateTime StartTime { get; protected set; }
+
+		/// <summary>
+		/// Perform any prerequisites the task requires
+		/// </summary>
+		virtual protected bool PerformPrequisites() { return true;  }
+
+		/// <summary>
+		/// Perform the actual task that is measured
+		/// </summary>
+		protected abstract bool PerformTask();
+
+		/// <summary>
+		/// Return a name for this task for reporting
+		/// </summary>
+		/// <returns></returns>
+		public abstract string GetTaskName();
+
+		/// <summary>
+		/// Run the task. Performs any prerequisites, then the actual task itself
+		/// </summary>
+		public void Run()
+		{
+			try
+			{
+				if (PerformPrequisites())
+				{
+					StartTime = DateTime.Now;
+
+					if (PerformTask())
+					{
+						TaskTime = DateTime.Now - StartTime;
+					}
+					else
+					{
+						FailureString = "Task Failed";
+						Failed = true;
+					}
+				}
+				else
+				{
+					FailureString = "Prequisites Failed";
+					Failed = true;
+				}
+				
+			}
+			catch (Exception Ex)
+			{
+				FailureString = string.Format("Exception: {0}", Ex.ToString());
+				Failed = true;
+			}		
+			
+			if (Failed)
+			{
+				Log.TraceError("{0} failed. {1}", GetTaskName(), FailureString);
+			}
+		}
+
+		/// <summary>
+		/// Report how long the task took
+		/// </summary>
+		public void Report()
+		{
+			if (!Failed)
+			{
+				Log.TraceInformation("Task {0}:\t\t\t\t{1}", GetTaskName(), TaskTime.ToString(@"hh\:mm\:ss"));
+			}
+			else
+			{
+				Log.TraceInformation("Task {0}::\t\t\t\tFailed. {1}", GetTaskName(), FailureString);
+			}
+		}
+	}
+}

Engine/Source/Programs/AutomationTool/Scripts/BuildTarget.Automation.cs

@@ -18,7 +18,10 @@ namespace AutomationTool
 	[Help("platform=PS4+XboxOne", "Platforms to build, join multiple platforms using +")]
 	[Help("configuration=Development+Test", "Configurations to build, join multiple configurations using +")]
 	[Help("target=Editor+Game", "Targets to build, join multiple targets using +")]
-	[Help("notools", "Don't build any tools (UHT, ShaderCompiler, CrashReporter")]
+	[Help("notools", "Don't build any tools (UnrealPak, Lightmass, ShaderCompiler, CrashReporter")]
+	[Help("clean", "Do a clean build")]
+	[Help("NoXGE", "Toggle to disable the distributed build process")]
+	[Help("DisableUnity", "Toggle to disable the unity build system")]
 	public class BuildTarget : BuildCommand
 	{
 		// exposed as a property so projects can derive and set this directly
@@ -32,13 +35,17 @@ namespace AutomationTool
 
 		public bool	  Clean { get; set; }
 
+		public bool	  NoTools { get; set; }
+
+		public string UBTArgs { get; set; }
+
 		protected Dictionary<string, string> TargetNames { get; set; }
 
 		public BuildTarget()
 		{
 			Platforms = HostPlatform.Current.HostEditorPlatform.ToString();
 			Configurations = "Development";
-
+			UBTArgs = "";
 			TargetNames = new Dictionary<string, string>();
 		}
 
@@ -53,14 +60,14 @@ namespace AutomationTool
 			Platforms = ParseParamValue("platform", Platforms);
 			Configurations = ParseParamValue("configuration", Configurations);
 			Clean = ParseParam("clean") || Clean;
+			NoTools = ParseParam("NoTools") || NoTools;
+			UBTArgs = ParseParamValue("ubtargs", UBTArgs);
 
 			if (string.IsNullOrEmpty(Targets))
 			{
 				throw new AutomationException("No target specified with -target. Use -help to see all options");
 			}
 
-			bool NoTools = ParseParam("notools");
-
 			IEnumerable<string> TargetList = Targets.Split(new char[] { '+' }, StringSplitOptions.RemoveEmptyEntries);
 			IEnumerable<UnrealTargetConfiguration> ConfigurationList = null;
 			IEnumerable<UnrealTargetPlatform> PlatformList = null;
@@ -144,6 +151,7 @@ namespace AutomationTool
 			}
 
 			UE4Build Build = new UE4Build(this);
+			Build.AlwaysBuildUHT = true;
 
 			UE4Build.BuildAgenda Agenda = new UE4Build.BuildAgenda();
 
@@ -153,16 +161,16 @@ namespace AutomationTool
 
 			UnrealTargetPlatform CurrentPlatform = HostPlatform.Current.HostEditorPlatform;
 
-			if (!NoTools)
+			//if (!NoTools)
 			{
-				Agenda.AddTarget("UnrealHeaderTool", CurrentPlatform, UnrealTargetConfiguration.Development, ProjectFile);
+				//Agenda.AddTarget("UnrealHeaderTool", CurrentPlatform, UnrealTargetConfiguration.Development, ProjectFile);
 			}
 
 			if (string.IsNullOrEmpty(EditorTarget) == false)
 			{
 				string TargetName = TargetNames[EditorTarget];
 
-				Agenda.AddTarget(TargetName, CurrentPlatform, UnrealTargetConfiguration.Development, ProjectFile);
+				Agenda.AddTarget(TargetName, CurrentPlatform, UnrealTargetConfiguration.Development, ProjectFile, UBTArgs);
 
 				if (!NoTools)
 				{
@@ -186,7 +194,7 @@ namespace AutomationTool
 				{
 					foreach (UnrealTargetConfiguration Config in ConfigurationList)
 					{
-						Agenda.AddTarget(TargetName, Platform, Config);
+						Agenda.AddTarget(TargetName, Platform, Config, ProjectFile, UBTArgs);
 					}
 				}
 			}