// Copyright Epic Games, Inc. All Rights Reserved. using System; using System.Collections.Generic; using System.IO; using System.Text.RegularExpressions; using System.Linq; using AutomationTool; using UnrealBuildTool; namespace Gauntlet { /// /// Describe the end result of a test. Until a test is complete /// TestResult.Invalid should be returned /// public enum TestResult { Invalid, Passed, Failed, WantRetry, Cancelled, TimedOut, }; /// /// Describes the current state of a test. /// public enum TestStatus { NotStarted, Pending, // Not currently used InProgress, Complete, }; /// /// Describes the priority of test. /// public enum TestPriority { Critical, High, Normal, Low, Idle }; /// /// The interface that all Gauntlet rests are required to implement. How these are /// implemented and whether responsibilities are handed off to other systems (e.g. Launch) /// is left to the implementor. It's expected that tests for major systems (e.g. Unreal) /// will likely implement their own base node. /// public interface ITestNode { /// /// Name of the test - used for logging / reporting /// string Name { get; } /// /// Maximum duration that the test is expected to run for. Tests longer than this will be halted. /// float MaxDuration { get; } /// /// Return true if the warnings and errors needs to log after summary /// bool LogWarningsAndErrorsAfterSummary { get; } /// /// What the test result should be treated as if we reach max duration. /// EMaxDurationReachedResult MaxDurationReachedResult { get; } /// /// Priority of this test in relation to any others that are running /// TestPriority Priority { get; } /// /// Sets the context that this test will run under. TODO - make this more of a contract that happens before CheckRequirements / LaunchTest /// /// /// void SetContext(ITestContext InContext); /// /// Checks if the test is ready to be started. If the test is not ready to run (e.g. resources not available) then it should return false. /// If it determines that it will never be ready it should throw an exception. /// /// bool IsReadyToStart(); /// /// Begin executing the provided test. At this point .Status should return InProgress and the test will eventually receive /// OnComplete and ShutdownTest calls /// /// /// true/false based on whether the test successfully launched bool StartTest(int Pass, int NumPasses); /// /// Called regularly to allow the test to check and report status /// void TickTest(); /// /// Sets Cancellation Reason. /// /// /// void SetCancellationReason(string Reason); /// /// Called to request that the test stop, either because it's reported that its complete or due to external factors. /// Tests should consider whether they passed or succeeded (even a terminated test may have gotten all the data it needs) /// and set their result appropriately. /// void StopTest(bool WasCancelled); /// /// Allows the node to restart with the same assigned devices. Only called if the expresses /// a .Result of TestResult.WantRetry while running. /// /// /// bool RestartTest(); /// /// Return an enum, that describes the state of the test /// TestStatus GetTestStatus(); /// /// The result of the test. Only called once GetTestStatus() returns complete, but may be called multiple /// times. /// TestResult GetTestResult(); /// /// Summary of the test. Only called once GetTestStatus() returns complete, but may be called multiple /// times. /// string GetTestSummary(); /// /// Return a list of all warnings that should be reported. May be different than Summary contents /// IEnumerable GetWarnings(); /// /// Return a list of all errors that should be reported. May be different than Summary contents /// IEnumerable GetErrors(); /// /// Called to request any that any necessary cleanup be performed. After CleanupTest is called no further calls will be /// made to this test and thus all resources should be released. /// /// /// void CleanupTest(); /// /// Output all defined commandline information for this test to the gauntlet window and exit test early. /// void DisplayCommandlineHelp(); } }