diff --git a/Engine/Source/Programs/UnrealBuildTool/Executors/ImmediateActionQueue.cs b/Engine/Source/Programs/UnrealBuildTool/Executors/ImmediateActionQueue.cs index d1d3e656a4f9..ca1198bdb37a 100644 --- a/Engine/Source/Programs/UnrealBuildTool/Executors/ImmediateActionQueue.cs +++ b/Engine/Source/Programs/UnrealBuildTool/Executors/ImmediateActionQueue.cs @@ -2,8 +2,10 @@ using EpicGames.Core; using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp.Syntax; using Microsoft.Extensions.Logging; using System; +using System.Collections; using System.Collections.Generic; using System.Linq; using System.Threading; @@ -56,17 +58,16 @@ namespace UnrealBuildTool Manual, } - /// /// Actions are assigned a runner when they need executing /// /// Type of the runner /// The action phase that this step supports - /// Action to be run queue running a task + /// Function to be run queue running a task /// If true, use the action weight as a secondary limit /// Maximum number of action actions /// Maximum weight of actions - record ImmediateActionQueueRunner(ImmediateActionQueueRunnerType Type, ActionPhase ActionPhase, Func RunAction, bool UseActionWeights = false, int MaxActions = int.MaxValue, double MaxActionWeight = int.MaxValue) + record ImmediateActionQueueRunner(ImmediateActionQueueRunnerType Type, ActionPhase ActionPhase, Func?> RunAction, bool UseActionWeights = false, int MaxActions = int.MaxValue, double MaxActionWeight = int.MaxValue) { /// /// Current number of active actions @@ -327,7 +328,7 @@ namespace UnrealBuildTool { var runAction = (LinkedAction action) => { - return new System.Action(async () => + return new Func(async () => { bool success = await _actionArtifactCache!.CompleteActionFromCacheAsync(action, CancellationToken); if (success) @@ -353,7 +354,7 @@ namespace UnrealBuildTool /// Maximum number of action actions /// Maximum weight of actions /// Created runner - public ImmediateActionQueueRunner CreateAutomaticRunner(Func runAction, bool useActionWeights, int maxActions, double maxActionWeight) + public ImmediateActionQueueRunner CreateAutomaticRunner(Func?> runAction, bool useActionWeights, int maxActions, double maxActionWeight) { ImmediateActionQueueRunner runner = new(ImmediateActionQueueRunnerType.Automatic, ActionPhase.Compile, runAction, useActionWeights, maxActions, maxActionWeight); _runners.Add(runner); @@ -368,7 +369,7 @@ namespace UnrealBuildTool /// Maximum number of action actions /// Maximum weight of actions /// Created runner - public ImmediateActionQueueRunner CreateManualRunner(Func runAction, bool useActionWeights = false, int maxActions = int.MaxValue, double maxActionWeight = double.MaxValue) + public ImmediateActionQueueRunner CreateManualRunner(Func?> runAction, bool useActionWeights = false, int maxActions = int.MaxValue, double maxActionWeight = double.MaxValue) { ImmediateActionQueueRunner runner = new(ImmediateActionQueueRunnerType.Manual, ActionPhase.Compile, runAction, useActionWeights, maxActions, maxActionWeight); _runners.Add(runner); @@ -433,33 +434,103 @@ namespace UnrealBuildTool /// True if an action was run, false if not public bool TryStartOneAction(ImmediateActionQueueRunner? runner = null) { - (System.Action? action, int completedActions) = TryStartOneActionInternal(runner); - if (action == null) + // Loop until we have an action or no completed actions (i.e. no propagated errors) + for(; ; ) { - // We found nothing, check to see if we have no active running tasks and no manual runners. - // If we don't, then it is assumed we can't queue any more work. - bool prematureDone = true; - foreach (ImmediateActionQueueRunner tryRunner in _runners) + bool hasCanceled = false; + + // Try to get the next action + Func? runAction = null; + LinkedAction? action = null; + int completedActions = 0; + lock (Actions) { - if (tryRunner.Type == ImmediateActionQueueRunnerType.Manual || tryRunner.ActiveActions != 0) + hasCanceled = CancellationTokenSource.IsCancellationRequested; + if (!hasCanceled) { - prematureDone = false; - break; + (runAction, action, completedActions) = TryStartOneActionInternal(runner); } } - if (prematureDone) + + // If we have an action, run it and account for any completed actions + if (runAction != null && action != null) { - completedActions = int.MaxValue; + Task.Factory.StartNew(() => + { + try + { + runAction().Wait(); + } + catch (Exception ex) + { + HandleException(action, ex); + } + }, CancellationToken, TaskCreationOptions.LongRunning | TaskCreationOptions.PreferFairness, TaskScheduler.Default); + AddCompletedActions(completedActions); + return true; } + + // If we have no completed actions, then we need to check for a stall + if (completedActions == 0) + { + // We found nothing, check to see if we have no active running tasks and no manual runners. + // If we don't, then it is assumed we can't queue any more work. + bool prematureDone = true; + foreach (ImmediateActionQueueRunner tryRunner in _runners) + { + if ((tryRunner.Type == ImmediateActionQueueRunnerType.Manual && !hasCanceled) || tryRunner.ActiveActions != 0) + { + prematureDone = false; + break; + } + } + if (prematureDone) + { + AddCompletedActions(int.MaxValue); + } + return false; + } + + // Otherwise add the completed actions and test again for the possibility that errors still need propagating + AddCompletedActions(completedActions); + } + } + + /// + /// Handle an exception running a task + /// + /// Action that has been run + /// Thrown exception + void HandleException(LinkedAction action, Exception ex) + { + if (ex is AggregateException aggregateEx) + { + if (aggregateEx.InnerException != null) + { + HandleException(action, aggregateEx.InnerException); + } + else + { + ExecuteResults results = new(new List(), int.MaxValue, TimeSpan.Zero, TimeSpan.Zero); + OnActionCompleted(action, false, results); + } + } + else if (ex is OperationCanceledException) + { + ExecuteResults results = new(new List(), int.MaxValue, TimeSpan.Zero, TimeSpan.Zero); + OnActionCompleted(action, false, results); } else { - Task.Factory.StartNew(() => action(), CancellationToken, TaskCreationOptions.LongRunning | TaskCreationOptions.PreferFairness, TaskScheduler.Default); + List text = new() + { + ExceptionUtils.FormatException(ex), + ExceptionUtils.FormatExceptionDetails(ex), + }; + ExecuteResults results = new(text, int.MaxValue, TimeSpan.Zero, TimeSpan.Zero); + OnActionCompleted(action, false, results); } - - AddCompletedActions(completedActions); - return action != null; } /// @@ -467,89 +538,87 @@ namespace UnrealBuildTool /// /// If specified, tasks will only be queued to this runner. Otherwise all manual runners will be used. /// Action to be run if something was queued and the number of completed actions - public (System.Action?, int) TryStartOneActionInternal(ImmediateActionQueueRunner? runner = null) + public (Func?, LinkedAction?, int) TryStartOneActionInternal(ImmediateActionQueueRunner? runner = null) { int completedActions = 0; - lock (Actions) + + // If we are starting deeper in the action collection, then never set the first pending action location + bool foundFirstPending = false; + + // Loop through the items + for (int actionIndex = _firstPendingAction; actionIndex != Actions.Length; ++actionIndex) { - // If we are starting deeper in the action collection, then never set the first pending action location - bool foundFirstPending = false; - // Loop through the items - for (int actionIndex = _firstPendingAction; actionIndex != Actions.Length; ++actionIndex) + // If the action has already reached the compilation state, then just ignore + if (Actions[actionIndex].Status != ActionStatus.Queued) { + continue; + } - // If the action has already reached the compilation state, then just ignore - if (Actions[actionIndex].Status != ActionStatus.Queued) - { - continue; - } + // If needed, update the first valid slot for searching for actions to run + if (!foundFirstPending) + { + _firstPendingAction = actionIndex; + foundFirstPending = true; + } - // If needed, update the first valid slot for searching for actions to run - if (!foundFirstPending) - { - _firstPendingAction = actionIndex; - foundFirstPending = true; - } + // Based on the ready state, use this action or mark as an error + switch (GetActionReadyState(Actions[actionIndex].Action)) + { + case ActionReadyState.NotReady: + break; - // Based on the ready state, use this action or mark as an error - switch (GetActionReadyState(Actions[actionIndex].Action)) - { - case ActionReadyState.NotReady: - break; + case ActionReadyState.Error: + Actions[actionIndex].Status = ActionStatus.Error; + completedActions++; + break; - case ActionReadyState.Error: - Actions[actionIndex].Status = ActionStatus.Error; - completedActions++; - break; - - case ActionReadyState.Ready: - ImmediateActionQueueRunner? selectedRunner = null; - System.Action? action = null; - if (runner != null) + case ActionReadyState.Ready: + ImmediateActionQueueRunner? selectedRunner = null; + Func? runAction = null; + if (runner != null) + { + if (runner.IsUnderLimits && runner.ActionPhase == Actions[actionIndex].Phase) { - if (runner.IsUnderLimits && runner.ActionPhase == Actions[actionIndex].Phase) + runAction = runner.RunAction(Actions[actionIndex].Action); + if (runAction != null) { - action = runner.RunAction(Actions[actionIndex].Action); - if (action != null) + selectedRunner = runner; + } + } + } + else + { + foreach (ImmediateActionQueueRunner tryRunner in _runners) + { + if (tryRunner.Type == ImmediateActionQueueRunnerType.Automatic && + tryRunner.IsUnderLimits && tryRunner.ActionPhase == Actions[actionIndex].Phase) + { + runAction = tryRunner.RunAction(Actions[actionIndex].Action); + if (runAction != null) { - selectedRunner = runner; - } - } - } - else - { - foreach (ImmediateActionQueueRunner tryRunner in _runners) - { - if (tryRunner.Type == ImmediateActionQueueRunnerType.Automatic && - tryRunner.IsUnderLimits && tryRunner.ActionPhase == Actions[actionIndex].Phase) - { - action = tryRunner.RunAction(Actions[actionIndex].Action); - if (action != null) - { - selectedRunner = tryRunner; - break; - } + selectedRunner = tryRunner; + break; } } } + } - if (action != null && selectedRunner != null) - { - Actions[actionIndex].Status = ActionStatus.Running; - Actions[actionIndex].Runner = selectedRunner; - selectedRunner.ActiveActions++; - selectedRunner.ActiveActionWeight += Actions[actionIndex].Action.Weight; - return (action, completedActions); - } - break; + if (runAction != null && selectedRunner != null) + { + Actions[actionIndex].Status = ActionStatus.Running; + Actions[actionIndex].Runner = selectedRunner; + selectedRunner.ActiveActions++; + selectedRunner.ActiveActionWeight += Actions[actionIndex].Action.Weight; + return (runAction, Actions[actionIndex].Action, completedActions); + } + break; - default: - throw new ApplicationException("Unexpected action ready state"); - } + default: + throw new ApplicationException("Unexpected action ready state"); } } - return (null, completedActions); + return (null, null, completedActions); } /// @@ -861,16 +930,12 @@ namespace UnrealBuildTool // prevent overwriting of error text s_previousLineLength = -1; - } - } - if (exitCode != 0 && exitCode != int.MaxValue) - { - // Cancel all other pending tasks - if (StopCompilationAfterErrors) - { - CancellationTokenSource.Cancel(); - AddCompletedActions(int.MaxValue); + // Cancel all other pending tasks + if (StopCompilationAfterErrors) + { + CancellationTokenSource.Cancel(); + } } } } diff --git a/Engine/Source/Programs/UnrealBuildTool/Executors/ParallelExecutor.cs b/Engine/Source/Programs/UnrealBuildTool/Executors/ParallelExecutor.cs index f0e75b70c50b..ef5ed83ca286 100644 --- a/Engine/Source/Programs/UnrealBuildTool/Executors/ParallelExecutor.cs +++ b/Engine/Source/Programs/UnrealBuildTool/Executors/ParallelExecutor.cs @@ -204,16 +204,12 @@ namespace UnrealBuildTool } } - private static System.Action? RunAction(ImmediateActionQueue queue, LinkedAction action) + private static Func? RunAction(ImmediateActionQueue queue, LinkedAction action) { return async () => { - Task task = RunAction(action, queue.ProcessGroup, queue.CancellationToken); - await task; - - ExecuteResults? results = task.Status == TaskStatus.RanToCompletion ? task.Result : null; - bool success = results?.ExitCode == 0; - queue.OnActionCompleted(action, success, results); + ExecuteResults results = await RunAction(action, queue.ProcessGroup, queue.CancellationToken); + queue.OnActionCompleted(action, results.ExitCode == 0, results); }; }