izzy/UnrealEngineUWP
0
0
Fork 0
mirror of https://github.com/izzy2lost/UnrealEngineUWP.git synced 2026-03-26 18:15:20 -07:00
Code Issues Packages Projects Releases Wiki Activity
Files
680c959a65e409b7337702bc109291ab3c2d55cf
UnrealEngineUWP/Engine/Source/Developer/AutomationController/Private/AutomationControllerManager.cpp
zahra nikbakht 680c959a65 Allow source link detection in MessageLog text when the source address is located at the right of the message (previously it would only work when the source address was on the left of the message) 
#jira UE-172487
#preflight 6425a86bc463fac9c8ffa4f2
#rb Jerome.Delattre

[CL 24859391 by zahra nikbakht in ue5-main branch]
2023-03-30 16:07:32 -04:00

1582 lines
54 KiB
C++
Raw Blame History

// Copyright Epic Games, Inc. All Rights Reserved.
#include "AutomationControllerManager.h"
#include "CoreMinimal.h"
#include "HAL/FileManager.h"
#include "Misc/CommandLine.h"
#include "Misc/Paths.h"
#include "Misc/ConfigCacheIni.h"
#include "Misc/AutomationTest.h"
#include "Misc/App.h"
#include "IAutomationReport.h"
#include "AutomationWorkerMessages.h"
#include "IMessageContext.h"
#include "MessageEndpoint.h"
#include "MessageEndpointBuilder.h"
#include "Modules/ModuleManager.h"
#include "AssetEditorMessages.h"
#include "ImageComparer.h"
#include "AutomationControllerSettings.h"
#include "Interfaces/IScreenShotToolsModule.h"
#include "Serialization/JsonSerializer.h"
#include "JsonObjectConverter.h"
#include "Misc/EngineVersion.h"
#include "Misc/FileHelper.h"
#include "PlatformHttp.h"
#include "AutomationTelemetry.h"
#include "AssetRegistry/AssetRegistryModule.h"
#if WITH_EDITOR
#include "UnrealEdGlobals.h"
#include "Editor/UnrealEdEngine.h"
#include "Logging/MessageLog.h"
#include "Tests/AutomationCommon.h"
#endif
#include "Async/ParallelFor.h"
#include UE_INLINE_GENERATED_CPP_BY_NAME(AutomationControllerManager)
// these strings are parsed by Gauntlet (AutomationLogParser) so make sure changes are replicated there!
#define AutomationTestStarting TEXT("Test Started. Name={%s} Path={%s}")
#define AutomationStateFormat TEXT("Test Completed. Result={%s} Name={%s} Path={%s}")
#define BeginEventsFormat TEXT("BeginEvents: %s")
#define EndEventsFormat TEXT("EndEvents: %s")
DEFINE_LOG_CATEGORY_STATIC(LogAutomationController, Log, All)
#define LOCTEXT_NAMESPACE "AutomationTesting"
void FAutomatedTestPassResults::AddTestResult(const IAutomationReportPtr& TestReport)
{
LLM_SCOPE_BYNAME(TEXT("AutomationTest/Report"));
const FString& FullTestPath = TestReport->GetFullTestPath();
if (!TestsMapIndex.Contains(FullTestPath))
{
FAutomatedTestResult TestResult;
TestResult.Test = TestReport;
TestResult.TestDisplayName = TestReport->GetDisplayName();
TestResult.FullTestPath = FullTestPath;
TestsMapIndex.Add(FullTestPath, Tests.Num());
Tests.Add(TestResult);
NotRun++;
}
}
FAutomatedTestResult& FAutomatedTestPassResults::GetTestResult(const IAutomationReportPtr& TestReport)
{
const FString& FullTestPath = TestReport->GetFullTestPath();
check(TestsMapIndex.Contains(FullTestPath));
return Tests[TestsMapIndex[FullTestPath]];
}
void FAutomatedTestPassResults::ReBuildTestsMapIndex()
{
LLM_SCOPE_BYNAME(TEXT("AutomationTest/Report"));
TestsMapIndex.Empty();
uint32 Index = 0;
for (const FAutomatedTestResult& TestResult : Tests)
{
TestsMapIndex.Add(TestResult.FullTestPath, Index++);
}
}
bool FAutomatedTestPassResults::ReflectResultStateToReport(IAutomationReportPtr& TestReport)
{
const FString& FullTestPath = TestReport->GetFullTestPath();
if (TestsMapIndex.Contains(FullTestPath))
{
FAutomatedTestResult& TestResult = Tests[TestsMapIndex[FullTestPath]];
if (TestResult.State == EAutomationState::InProcess)
{
// Marking incomplete test from previous pass as failure.
TestResult.AddEvent(EAutomationEventType::Error, TEXT("Test did not run until completion. The test exited prematurely."));
TestResult.State = EAutomationState::Fail;
InProcess--;
Failed++;
}
TestReport->SetState(TestResult.State);
TestResult.Test = TestReport;
return true;
}
return false;
}
void FAutomatedTestPassResults::UpdateTestResultStatus(const IAutomationReportPtr& TestReport, EAutomationState State, bool bHasWarning)
{
FAutomatedTestResult& TestResult = GetTestResult(TestReport);
if (TestResult.State == State)
return;
// Book keeping
// from transient states
switch (TestResult.State)
{
case EAutomationState::InProcess:
InProcess--;
break;
case EAutomationState::NotRun:
NotRun--;
break;
}
// to new state
switch (State)
{
case EAutomationState::Success:
if (bHasWarning)
{
SucceededWithWarnings++;
}
else
{
Succeeded++;
}
break;
case EAutomationState::Fail:
Failed++;
break;
case EAutomationState::InProcess:
TestResult.DateTime = FDateTime::Now();
InProcess++;
break;
case EAutomationState::NotRun:
NotRun++;
break;
case EAutomationState::Skipped:
// Those are not counted
break;
}
// commit the change
TestResult.State = State;
}
FAutomationControllerManager::FAutomationControllerManager()
{
UAutomationControllerSettings* Settings = UAutomationControllerSettings::StaticClass()->GetDefaultObject<UAutomationControllerSettings>();
if (Settings->CheckTestIntervalSeconds > 0.0f)
{
CheckTestIntervalSeconds = Settings->CheckTestIntervalSeconds;
}
if (Settings->GameInstanceLostTimerSeconds > 0.0f)
{
GameInstanceLostTimerSeconds = Settings->GameInstanceLostTimerSeconds;
}
bKeepPIEOpen = Settings->bKeepPIEOpen;
FString DeveloperPath;
FParse::Value(FCommandLine::Get(), TEXT("ReportOutputPath="), ReportExportPath, false);
FParse::Value(FCommandLine::Get(), TEXT("DisplayReportOutputPath="), ReportURLPath, false);
FParse::Value(FCommandLine::Get(), TEXT("DeveloperReportOutputPath="), DeveloperPath, false);
if (ReportExportPath.Len())
{
UE_LOG(LogAutomationController, Warning, TEXT("Argument -ReportOutputPath= is now -ReportExportPath=. Please update your command line!"));
}
if (ReportURLPath.Len())
{
UE_LOG(LogAutomationController, Warning, TEXT("Argument -DisplayReportOutputPath= is now -ReportURL=. Please update your command line!"));
}
if (DeveloperPath.Len())
{
UE_LOG(LogAutomationController, Warning, TEXT("Argument -DeveloperReportOutputPath= is now -DeveloperReport (no value). Please update your command line!"));
}
// read values with new names
FParse::Value(FCommandLine::Get(), TEXT("ReportExportPath="), ReportExportPath, false);
FParse::Value(FCommandLine::Get(), TEXT("ReportURL="), ReportURLPath, false);
bool bUseDeveloperPath = DeveloperPath.Len() > 0 || FParse::Param(FCommandLine::Get(), TEXT("DeveloperReport"));
if (ReportExportPath.Len() && bUseDeveloperPath)
{
ReportExportPath = ReportExportPath / TEXT("dev") / FString(FPlatformProcess::UserName()).ToLower();
}
if (ReportURLPath.Len() && bUseDeveloperPath)
{
DeveloperReportUrl = ReportURLPath / TEXT("dev") / FString(FPlatformProcess::UserName()).ToLower() / TEXT("index.html");
}
bResumeRunTest = FParse::Param(FCommandLine::Get(), TEXT("ResumeRunTest"));
}
bool FAutomationControllerManager::IsReadyForTests()
{
FAssetRegistryModule& AssetRegistryModule = FModuleManager::LoadModuleChecked<FAssetRegistryModule>("AssetRegistry");
if (AssetRegistryModule.Get().IsLoadingAssets())
{
return false;
}
#if WITH_EDITOR && !UE_BUILD_SHIPPING && WITH_AUTOMATION_TESTS
if (InteractiveFrameRateCheck == nullptr)
{
InteractiveFrameRateCheck = MakeShared<FWaitForInteractiveFrameRate>();
}
if (!InteractiveFrameRateCheck->Update())
{
return false;
}
InteractiveFrameRateCheck = nullptr;
#endif // WITH_EDITOR
return true;
}
void FAutomationControllerManager::RequestAvailableWorkers(const FGuid& SessionId)
{
//invalidate previous tests
++ExecutionCount;
DeviceClusterManager.Reset();
ControllerResetDelegate.Broadcast();
// Don't allow reports to be exported
bTestResultsAvailable = false;
//store off active session ID to reject messages that come in from different sessions
ActiveSessionId = SessionId;
//EAutomationTestFlags::FilterMask
//TODO AUTOMATION - include change list, game, etc, or remove when launcher is integrated
int32 ChangelistNumber = 10000;
FString ProcessName = TEXT("instance_name");
MessageEndpoint->Publish(FMessageEndpoint::MakeMessage<FAutomationWorkerFindWorkers>(ChangelistNumber, FApp::GetProjectName(), ProcessName, SessionId), EMessageScope::Network);
// Reset the check test timers
LastTimeUpdateTicked = FPlatformTime::Seconds();
CheckTestTimer = 0.f;
IScreenShotToolsModule& ScreenShotModule = FModuleManager::LoadModuleChecked<IScreenShotToolsModule>("ScreenShotComparisonTools");
ScreenshotManager = ScreenShotModule.GetScreenShotManager();
}
void FAutomationControllerManager::RequestTests()
{
//invalidate incoming results
ExecutionCount++;
//reset the number of responses we have received
RefreshTestResponses = 0;
ReportManager.Empty();
for ( int32 ClusterIndex = 0; ClusterIndex < DeviceClusterManager.GetNumClusters(); ++ClusterIndex )
{
int32 DevicesInCluster = DeviceClusterManager.GetNumDevicesInCluster(ClusterIndex);
if ( DevicesInCluster > 0 )
{
FMessageAddress MessageAddress = DeviceClusterManager.GetDeviceMessageAddress(ClusterIndex, 0);
UE_LOG(LogAutomationController, Log, TEXT("Requesting test list from %s"), *MessageAddress.ToString());
//issue tests on appropriate platforms
MessageEndpoint->Send(FMessageEndpoint::MakeMessage<FAutomationWorkerRequestTests>(bDeveloperDirectoryIncluded, RequestedTestFlags), MessageAddress);
}
}
}
void FAutomationControllerManager::RunTests(const bool bInIsLocalSession)
{
ExecutionCount++;
CurrentTestPass = 0;
ReportManager.SetCurrentTestPass(CurrentTestPass);
ClusterDistributionMask = 0;
bTestResultsAvailable = false;
TestRunningArray.Empty();
bIsLocalSession = bInIsLocalSession;
// Reset the check test timers
LastTimeUpdateTicked = FPlatformTime::Seconds();
CheckTestTimer = 0.f;
#if WITH_EDITOR
FMessageLog AutomationEditorLog("AutomationTestingLog");
FString NewPageName = FString::Printf(TEXT("-----Test Run %d----"), ExecutionCount);
FText NewPageNameText = FText::FromString(*NewPageName);
AutomationEditorLog.Open();
AutomationEditorLog.NewPage(NewPageNameText);
AutomationEditorLog.Info(NewPageNameText);
#endif
//reset all tests
ReportManager.ResetForExecution(NumTestPasses);
// Register All tests that we'll need to be exported as json report
if (!ReportExportPath.IsEmpty())
{
JsonTestPassResults.IsRequired = true;
TArray<IAutomationReportPtr> TestsToRun = ReportManager.GetEnabledTestReports();
// Load previous test results
if (bResumeRunTest)
{
FString ReportFileName = FString::Printf(TEXT("%s/index.json"), *ReportExportPath);
if (FPaths::FileExists(ReportFileName))
{
LoadJsonTestPassSummary(ReportFileName, TestsToRun);
}
}
for (IAutomationReportPtr& TestReport : TestsToRun)
{
JsonTestPassResults.AddTestResult(TestReport);
}
// Get the html index written down as soon as possible so once results are updated, they can be reviewed.
GenerateTestPassHtmlIndex();
}
for ( int32 ClusterIndex = 0; ClusterIndex < DeviceClusterManager.GetNumClusters(); ++ClusterIndex )
{
//enable each device cluster
ClusterDistributionMask |= ( 1 << ClusterIndex );
//for each device in this cluster
for ( int32 DeviceIndex = 0; DeviceIndex < DeviceClusterManager.GetNumDevicesInCluster(ClusterIndex); ++DeviceIndex )
{
//mark the device as idle
DeviceClusterManager.SetTest(ClusterIndex, DeviceIndex, NULL);
// Send command to reset tests (delete local files, etc)
FMessageAddress MessageAddress = DeviceClusterManager.GetDeviceMessageAddress(ClusterIndex, DeviceIndex);
UE_LOG(LogAutomationController, Log, TEXT("Sending Reset Tests to %s"), *MessageAddress.ToString());
MessageEndpoint->Send(FMessageEndpoint::MakeMessage<FAutomationWorkerResetTests>(), MessageAddress);
// Store devices info into the json report.
if (JsonTestPassResults.IsRequired)
{
const FAutomationDeviceInfo& DeviceInfo = DeviceClusterManager.GetDeviceInfo(ClusterIndex, DeviceIndex);
JsonTestPassResults.Devices.Add(DeviceInfo);
}
}
}
// Clear the logs and transient folders. Reports can be manually cleared by the user in the UI if they so desire
UE_LOG(LogAutomationController, Display, TEXT("Clearing %s and %s"), *FPaths::AutomationLogDir(), *FPaths::AutomationTransientDir());
IFileManager::Get().DeleteDirectory(*FPaths::AutomationLogDir(), false, true);
IFileManager::Get().DeleteDirectory(*FPaths::AutomationTransientDir(), false, true);
// Inform the UI we are running tests
if ( ClusterDistributionMask != 0 )
{
SetControllerStatus(EAutomationControllerModuleState::Running);
}
}
void FAutomationControllerManager::StopTests()
{
bTestResultsAvailable = false;
ClusterDistributionMask = 0;
ReportManager.StopRunningTests();
// Inform the UI we have stopped running tests
if ( DeviceClusterManager.HasActiveDevice() )
{
for (int32 ClusterIndex = 0; ClusterIndex < DeviceClusterManager.GetNumClusters(); ++ClusterIndex)
{
//for each device in this cluster
for (int32 DeviceIndex = 0; DeviceIndex < DeviceClusterManager.GetNumDevicesInCluster(ClusterIndex); ++DeviceIndex)
{
//mark the device as idle
DeviceClusterManager.SetTest(ClusterIndex, DeviceIndex, NULL);
// Send command to reset tests (delete local files, etc)
FMessageAddress MessageAddress = DeviceClusterManager.GetDeviceMessageAddress(ClusterIndex, DeviceIndex);
UE_LOG(LogAutomationController, Log, TEXT("Sending StopTests to %s"), *MessageAddress.ToString());
MessageEndpoint->Send(FMessageEndpoint::MakeMessage<FAutomationWorkerStopTests>(), MessageAddress);
}
}
SetControllerStatus(EAutomationControllerModuleState::Ready);
}
else
{
SetControllerStatus(EAutomationControllerModuleState::Disabled);
}
TestRunningArray.Empty();
// Close play window
#if WITH_EDITOR
if (GUnrealEd && !bKeepPIEOpen)
{
GUnrealEd->RequestEndPlayMap();
}
#endif
}
void FAutomationControllerManager::Init()
{
extern void EmptyLinkFunctionForStaticInitializationAutomationExecCmd();
EmptyLinkFunctionForStaticInitializationAutomationExecCmd();
AutomationTestState = EAutomationControllerModuleState::Disabled;
bTestResultsAvailable = false;
bSendAnalytics = FParse::Param(FCommandLine::Get(), TEXT("SendAutomationAnalytics"));
RequestedTestFlags = EAutomationTestFlags::SmokeFilter | EAutomationTestFlags::EngineFilter | EAutomationTestFlags::ProductFilter | EAutomationTestFlags::PerfFilter;
}
void FAutomationControllerManager::RequestLoadAsset(const FString& InAssetName)
{
MessageEndpoint->Publish(FMessageEndpoint::MakeMessage<FAssetEditorRequestOpenAsset>(InAssetName), EMessageScope::Process);
}
void FAutomationControllerManager::Tick()
{
ProcessAvailableTasks();
ProcessComparisonQueue();
}
void FAutomationControllerManager::ProcessComparisonQueue()
{
TSharedPtr<FComparisonEntry> Entry;
if ( ComparisonQueue.Peek(Entry) )
{
if ( Entry->PendingComparison.IsReady() )
{
const bool Dequeued = ComparisonQueue.Dequeue(Entry);
check(Dequeued);
FImageComparisonResult Result = Entry->PendingComparison.Get();
const FGuid UniqueId = FGuid::NewGuid();
// Send the message back to the automation worker letting it know the results of the comparison test.
{
FAutomationWorkerImageComparisonResults* Message = FMessageEndpoint::MakeMessage<FAutomationWorkerImageComparisonResults>(
UniqueId,
Result.IsNew(),
Result.AreSimilar(),
Result.MaxLocalDifference,
Result.GlobalDifference,
Result.ErrorMessage.ToString()
);
UE_LOG(LogAutomationController, Log, TEXT("Sending ImageComparisonResult to %s (IsNew=%d, AreSimilar=%d)"),
*Entry->Sender.ToString()
, Result.IsNew()
, Result.AreSimilar()
);
MessageEndpoint->Send(Message, Entry->Sender);
}
// Find the game session instance info
int32 ClusterIndex;
int32 DeviceIndex;
verify(DeviceClusterManager.FindDevice(Entry->Sender, ClusterIndex, DeviceIndex));
// Get the current test.
TSharedPtr<IAutomationReport> Report = DeviceClusterManager.GetTest(ClusterIndex, DeviceIndex);
if (Report.IsValid())
{
// Record the artifacts for the test.
TMap<EComparisonFileTypes, FString> LocalFiles;
FString ScreenshotResultsFolder = FPaths::AutomationReportsDir();
// Paths in the result are relative to the automation report directory.
LocalFiles.Add(EComparisonFileTypes::Unapproved, FPaths::Combine(ScreenshotResultsFolder, Result.ReportIncomingFilePath));
// Don't copy reference and delta if the images are similar.
if (!Result.AreSimilar())
{
// unapproved should always be valid. but approved/difference may be empty if this is a new screenshot
if (Result.ReportIncomingFilePath.Len())
{
LocalFiles.Add(EComparisonFileTypes::Approved, FPaths::Combine(ScreenshotResultsFolder, Result.ReportApprovedFilePath));
}
if (Result.ReportComparisonFilePath.Len())
{
LocalFiles.Add(EComparisonFileTypes::Difference, FPaths::Combine(ScreenshotResultsFolder, Result.ReportComparisonFilePath));
}
}
Report->AddArtifact(ClusterIndex, CurrentTestPass, FAutomationArtifact(UniqueId, Entry->ScreenshotPath, EAutomationArtifactType::Comparison, LocalFiles));
}
else
{
UE_LOG(LogAutomationController, Error, TEXT("Cannot generate screenshot report for screenshot %s as report is missing"), *Result.IncomingFilePath);
}
}
}
}
void FAutomationControllerManager::ProcessAvailableTasks()
{
// Distribute tasks
if ( ClusterDistributionMask != 0 )
{
// For each device cluster
for ( int32 ClusterIndex = 0; ClusterIndex < DeviceClusterManager.GetNumClusters(); ++ClusterIndex )
{
bool bAllTestsComplete = true;
// If any of the devices were valid
if ( ( ClusterDistributionMask & ( 1 << ClusterIndex ) ) && DeviceClusterManager.GetNumDevicesInCluster(ClusterIndex) > 0 )
{
ExecuteNextTask(ClusterIndex, bAllTestsComplete);
}
//if we're all done running our tests
if ( bAllTestsComplete )
{
//we don't need to test this cluster anymore
ClusterDistributionMask &= ~( 1 << ClusterIndex );
if ( ClusterDistributionMask == 0 )
{
ProcessResults();
// Close play window
#if WITH_EDITOR
if (GUnrealEd && !bKeepPIEOpen)
{
GUnrealEd->RequestEndPlayMap();
}
#endif
//Notify the graphical layout we are done processing results.
TestsCompleteDelegate.Broadcast();
}
}
}
}
if ( bIsLocalSession == false )
{
// Update the test status for timeouts if this is not a local session
UpdateTests();
}
}
void FAutomationControllerManager::ReportTestResults()
{
UE_LOG(LogAutomationController, Log, TEXT("Test Pass Results:"));
for ( int32 i = 0; i < JsonTestPassResults.Tests.Num(); i++ )
{
UE_LOG(LogAutomationController, Log, TEXT("%s: %s"), *JsonTestPassResults.Tests[i].TestDisplayName, *AutomationStateToString(JsonTestPassResults.Tests[i].State));
}
}
void FAutomationControllerManager::CollectTestResults(TSharedPtr<IAutomationReport> Report, const FAutomationTestResults& Results)
{
if (JsonTestPassResults.IsRequired)
{
JsonTestPassResults.UpdateTestResultStatus(Report, Results.State, Results.GetWarningTotal() > 0);
FAutomatedTestResult& TestResult = JsonTestPassResults.GetTestResult(Report);
TestResult.SetEvents(Results.GetEntries(), Results.GetWarningTotal(), Results.GetErrorTotal());
TestResult.SetArtifacts(Results.Artifacts);
TestResult.Duration = Results.Duration;
JsonTestPassResults.TotalDuration += Results.Duration;
FString ArtifactDirName = TEXT("imageCompare");
if (FEngineVersion::Current().GetChangelist() != 0)
{
ArtifactDirName = ArtifactDirName / FString::FromInt(FEngineVersion::Current().GetChangelist());
}
FString ArtifactExportPath = ReportExportPath / ArtifactDirName;
// Copy new artifacts to Report export path
FCriticalSection CS;
for (FAutomationArtifact& Artifact : TestResult.GetArtifacts())
{
TArray<EComparisonFileTypes> Keys;
Artifact.LocalFiles.GetKeys(Keys);
bool bOnlyUnapproved = Keys.Num() == 1 && Keys[0] == EComparisonFileTypes::Unapproved;
ParallelFor(Keys.Num(), [&](int32 Index)
{
const EComparisonFileTypes& Key = Keys[Index];
FString Path = Artifact.LocalFiles[Key];
FPaths::MakePathRelativeTo(Path, *FPaths::AutomationReportsDir());
Path = ArtifactDirName / Path;
{
FScopeLock Lock(&CS);
Artifact.Files.Add(Key, MoveTemp(Path));
}
if (Key == EComparisonFileTypes::Unapproved)
{
// Copy screenshot report
FScreenshotExportResult ExportResult = ScreenshotManager->ExportScreenshotComparisonResult(Artifact.Name, ArtifactExportPath, bOnlyUnapproved);
FScopeLock Lock(&CS);
if (!JsonTestPassResults.ComparisonExported && ExportResult.Success)
{
JsonTestPassResults.ComparisonExported = ExportResult.Success;
JsonTestPassResults.ComparisonExportDirectory = ExportResult.ExportPath;
}
}
});
}
}
}
bool FAutomationControllerManager::GenerateJsonTestPassSummary(FAutomatedTestPassResults& SerializedPassResults)
{
UE_LOG(LogAutomationController, Display, TEXT("Converting results to json object..."));
SerializedPassResults.ReportCreatedOn = FDateTime::Now();
FString Json;
if (FJsonObjectConverter::UStructToJsonObjectString(SerializedPassResults, Json))
{
FString ReportFileName = FString::Printf(TEXT("%s/index.json"), *ReportExportPath);
const int32 WriteAttempts = 3;
const float SleepBetweenAttempts = 0.05f;
for (int32 Attempt = 1; Attempt <= WriteAttempts; ++Attempt)
{
if (FFileHelper::SaveStringToFile(Json, *ReportFileName, FFileHelper::EEncodingOptions::ForceUTF8))
{
UE_LOG(LogAutomationController, Display, TEXT("Successfully wrote json results file!"));
return true;
}
FPlatformProcess::Sleep(SleepBetweenAttempts);
}
UE_LOG(LogAutomationController, Warning, TEXT("Failed to write test report json to '%s' after 3 attempts - No report will be generated."), *ReportFileName);
}
else
{
UE_LOG(LogAutomationController, Error, TEXT("Failed to convert test results to json object - No report will be generated."));
}
return false;
}
bool FAutomationControllerManager::GenerateTestPassHtmlIndex()
{
UE_LOG(LogAutomationController, Display, TEXT("Loading results html template..."));
FString ReportTemplate;
if (FFileHelper::LoadFileToString(ReportTemplate, *(FPaths::EngineContentDir() / TEXT("Automation/Report-Template.html"))))
{
FString ReportFileName = FString::Printf(TEXT("%s/index.html"), *ReportExportPath);
const int32 WriteAttempts = 3;
const float SleepBetweenAttempts = 0.05f;
for (int32 Attempt = 1; Attempt <= WriteAttempts; ++Attempt)
{
if (FFileHelper::SaveStringToFile(ReportTemplate, *ReportFileName, FFileHelper::EEncodingOptions::ForceUTF8))
{
UE_LOG(LogAutomationController, Display, TEXT("Successfully wrote html results file!"));
return true;
}
FPlatformProcess::Sleep(SleepBetweenAttempts);
}
UE_LOG(LogAutomationController, Warning, TEXT("Failed to write test report html to '%s' after 3 attempts - No report will be generated."), *ReportFileName);
}
else
{
UE_LOG(LogAutomationController, Error, TEXT("Failed to load test report html template - No report will be generated."));
}
return false;
}
bool FAutomationControllerManager::LoadJsonTestPassSummary(FString& ReportFilePath, TArray<IAutomationReportPtr> TestReports)
{
LLM_SCOPE_BYNAME(TEXT("AutomationTest/Report"));
FString Json;
if (!FFileHelper::LoadFileToString(Json, *ReportFilePath))
{
UE_LOG(LogAutomationController, Error, TEXT("Failed to read json results file '%s'."), *ReportFilePath);
return false;
}
if (!FJsonObjectConverter::JsonObjectStringToUStruct(Json, &JsonTestPassResults))
{
UE_LOG(LogAutomationController, Error, TEXT("Failed to load json results file '%s'."), *ReportFilePath);
return false;
}
UE_LOG(LogAutomationController, Display, TEXT("Successfully loaded json results file."));
JsonTestPassResults.ReBuildTestsMapIndex();
// Reflect Test Result Status on Report Status
for (IAutomationReportPtr& TestReport : TestReports)
{
if (!JsonTestPassResults.ReflectResultStateToReport(TestReport))
{
UE_LOG(LogAutomationController, Warning, TEXT("No match to test '%s' in json results."), *TestReport->GetFullTestPath());
}
}
return true;
}
FString FAutomationControllerManager::SlugString(const FString& DisplayString) const
{
FString GeneratedName = DisplayString;
// Convert the display label, which may consist of just about any possible character, into a
// suitable name for a UObject (remove whitespace, certain symbols, etc.)
{
for ( int32 BadCharacterIndex = 0; BadCharacterIndex < UE_ARRAY_COUNT(INVALID_OBJECTNAME_CHARACTERS) - 1; ++BadCharacterIndex )
{
const TCHAR TestChar[2] = { INVALID_OBJECTNAME_CHARACTERS[BadCharacterIndex], 0 };
const int32 NumReplacedChars = GeneratedName.ReplaceInline(TestChar, TEXT(""));
}
}
return GeneratedName;
}
FString FAutomationControllerManager::GetReportOutputPath() const
{
return ReportExportPath;
}
void FAutomationControllerManager::ExecuteNextTask( int32 ClusterIndex, OUT bool& bAllTestsCompleted )
{
bool bTestThatRequiresMultiplePraticipantsHadEnoughParticipants = false;
TArray< IAutomationReportPtr > TestsRunThisPass;
// For each device in this cluster
int32 NumDevicesInCluster = DeviceClusterManager.GetNumDevicesInCluster( ClusterIndex );
for ( int32 DeviceIndex = 0; DeviceIndex < NumDevicesInCluster; ++DeviceIndex )
{
// If this device is idle
if ( !DeviceClusterManager.GetTest(ClusterIndex, DeviceIndex).IsValid() && DeviceClusterManager.DeviceEnabled(ClusterIndex, DeviceIndex) )
{
// Get the next test that should be worked on
TSharedPtr< IAutomationReport > NextTest = ReportManager.GetNextReportToExecute(bAllTestsCompleted, ClusterIndex, CurrentTestPass, NumDevicesInCluster);
if ( NextTest.IsValid() )
{
// Get the status of the test
EAutomationState TestState = NextTest->GetState(ClusterIndex, CurrentTestPass);
if (TestState == EAutomationState::NotRun)
{
// Reserve this device for the test
DeviceClusterManager.SetTest(ClusterIndex, DeviceIndex, NextTest);
TestsRunThisPass.Add(NextTest);
// If we now have enough devices reserved for the test, run it!
TArray<FMessageAddress> DeviceAddresses = DeviceClusterManager.GetDevicesReservedForTest(ClusterIndex, NextTest);
if (DeviceAddresses.Num() == NextTest->GetNumParticipantsRequired())
{
TArray<FString> GameInstances;
// Send it to each device
for (int32 AddressIndex = 0; AddressIndex < DeviceAddresses.Num(); ++AddressIndex)
{
FAutomationTestResults TestResults;
TestResults.GameInstance = DeviceClusterManager.GetClusterGameInstance(ClusterIndex, DeviceIndex);
TestResults.State = EAutomationState::InProcess;
GameInstances.Add(TestResults.GameInstance);
NextTest->SetResults(ClusterIndex, CurrentTestPass, TestResults);
// Mark the device as busy
FMessageAddress DeviceAddress = DeviceAddresses[AddressIndex];
// Send the test to the device for execution!
UE_LOG(LogAutomationController, Log, TEXT("Sending RunTest %s to %s"), *NextTest->GetDisplayName(), *DeviceAddress.ToString());
MessageEndpoint->Send(FMessageEndpoint::MakeMessage<FAutomationWorkerRunTests>(ExecutionCount, AddressIndex, NextTest->GetCommand(), NextTest->GetDisplayName(), NextTest->GetFullTestPath(), bSendAnalytics), DeviceAddress);
// Add a test so we can check later if the device is still active
TestRunningArray.Add(FTestRunningInfo(DeviceAddress));
}
UE_LOG(LogAutomationController, Display, AutomationTestStarting, *NextTest->GetDisplayName(), *NextTest->GetFullTestPath());
if (JsonTestPassResults.IsRequired && bResumeRunTest)
{
JsonTestPassResults.UpdateTestResultStatus(NextTest, EAutomationState::InProcess);
FAutomatedTestResult& TestResult = JsonTestPassResults.GetTestResult(NextTest);
TestResult.DeviceInstance = GameInstances;
// Save the whole pass report so that if the next test triggers a critical failure we are not left with no pass report and we can resume.
GenerateJsonTestPassSummary(JsonTestPassResults);
}
NextTest->ResetNetworkCommandResponses();
}
}
}
}
else
{
// At least one device is still working
bAllTestsCompleted = false;
}
}
// Ensure any tests we have attempted to run on this pass had enough participants to successfully run.
for ( int32 TestIndex = 0; TestIndex < TestsRunThisPass.Num(); TestIndex++ )
{
IAutomationReportPtr CurrentTest = TestsRunThisPass[TestIndex];
if ( CurrentTest->GetNumDevicesRunningTest() != CurrentTest->GetNumParticipantsRequired() )
{
if ( GetNumDevicesInCluster(ClusterIndex) < CurrentTest->GetNumParticipantsRequired() )
{
FAutomationTestResults TestResults;
TestResults.State = EAutomationState::Skipped;
TestResults.AddEvent(FAutomationEvent(EAutomationEventType::Warning, FString::Printf(TEXT("Skipped because the test needed %d devices to participate, Only had %d available."), CurrentTest->GetNumParticipantsRequired(), DeviceClusterManager.GetNumDevicesInCluster(ClusterIndex))));
CurrentTest->SetResults(ClusterIndex, CurrentTestPass, TestResults);
CollectTestResults(CurrentTest, TestResults);
DeviceClusterManager.ResetAllDevicesRunningTest(ClusterIndex, CurrentTest);
}
}
}
//Check to see if we finished a pass
if ( bAllTestsCompleted && CurrentTestPass < NumTestPasses - 1 )
{
CurrentTestPass++;
ReportManager.SetCurrentTestPass(CurrentTestPass);
bAllTestsCompleted = false;
}
}
void FAutomationControllerManager::Startup()
{
MessageEndpoint = FMessageEndpoint::Builder("FAutomationControllerModule")
.Handling<FAutomationWorkerFindWorkersResponse>(this, &FAutomationControllerManager::HandleFindWorkersResponseMessage)
.Handling<FAutomationWorkerPong>(this, &FAutomationControllerManager::HandlePongMessage)
.Handling<FAutomationWorkerRequestNextNetworkCommand>(this, &FAutomationControllerManager::HandleRequestNextNetworkCommandMessage)
.Handling<FAutomationWorkerRequestTestsReplyComplete>(this, &FAutomationControllerManager::HandleRequestTestsReplyCompleteMessage)
.Handling<FAutomationWorkerRunTestsReply>(this, &FAutomationControllerManager::HandleRunTestsReplyMessage)
.Handling<FAutomationWorkerScreenImage>(this, &FAutomationControllerManager::HandleReceivedScreenShot)
.Handling<FAutomationWorkerTestDataRequest>(this, &FAutomationControllerManager::HandleTestDataRequest)
.Handling<FAutomationWorkerWorkerOffline>(this, &FAutomationControllerManager::HandleWorkerOfflineMessage)
.Handling<FAutomationWorkerTelemetryData>(this, &FAutomationControllerManager::HandleReceivedTelemetryData);
if ( MessageEndpoint.IsValid() )
{
MessageEndpoint->Subscribe<FAutomationWorkerWorkerOffline>();
}
ClusterDistributionMask = 0;
ExecutionCount = 0;
bDeveloperDirectoryIncluded = false;
RequestedTestFlags = EAutomationTestFlags::SmokeFilter | EAutomationTestFlags::EngineFilter | EAutomationTestFlags::ProductFilter | EAutomationTestFlags::PerfFilter;
NumTestPasses = 1;
//Default to machine name
DeviceGroupFlags = 0;
ToggleDeviceGroupFlag(EAutomationDeviceGroupTypes::MachineName);
}
void FAutomationControllerManager::Shutdown()
{
MessageEndpoint.Reset();
ShutdownDelegate.Broadcast();
RemoveCallbacks();
}
void FAutomationControllerManager::RemoveCallbacks()
{
ShutdownDelegate.Clear();
TestsAvailableDelegate.Clear();
TestsRefreshedDelegate.Clear();
TestsCompleteDelegate.Clear();
}
void FAutomationControllerManager::SetTestNames(const FMessageAddress& AutomationWorkerAddress, TArray<FAutomationTestInfo>& TestInfo)
{
int32 DeviceClusterIndex = INDEX_NONE;
int32 DeviceIndex = INDEX_NONE;
// Find the device that requested these tests
if ( DeviceClusterManager.FindDevice(AutomationWorkerAddress, DeviceClusterIndex, DeviceIndex) )
{
// Sort tests by display name
struct FCompareAutomationTestInfo
{
FORCEINLINE bool operator()(const FAutomationTestInfo& A, const FAutomationTestInfo& B) const
{
return A.GetDisplayName() < B.GetDisplayName();
}
};
TestInfo.Sort(FCompareAutomationTestInfo());
// Add each test to the collection
for ( int32 TestIndex = 0; TestIndex < TestInfo.Num(); ++TestIndex )
{
// Ensure our test exists. If not, add it
ReportManager.EnsureReportExists(TestInfo[TestIndex], DeviceClusterIndex, NumTestPasses);
}
}
else
{
//todo automation - make sure to report error if the device was not discovered correctly
}
// Note the response
RefreshTestResponses++;
// If we have received all the responses we expect to
if ( RefreshTestResponses == DeviceClusterManager.GetNumClusters() )
{
TestsRefreshedDelegate.Broadcast();
}
}
void FAutomationControllerManager::ProcessResults()
{
bHasErrors = false;
bHasWarning = false;
bHasLogs = false;
TArray< TSharedPtr< IAutomationReport > >& TestReports = GetReports();
if ( TestReports.Num() )
{
bTestResultsAvailable = true;
for ( int32 Index = 0; Index < TestReports.Num(); Index++ )
{
CheckChildResult(TestReports[Index]);
}
}
if ( JsonTestPassResults.IsRequired )
{
UE_LOG(LogAutomationController, Display, TEXT("Exporting Automation Report to %s."), *ReportExportPath);
FAutomatedTestPassResults SerializedPassResults = JsonTestPassResults;
{
// Sort result by failure to improve readability
SerializedPassResults.Tests.StableSort([](const FAutomatedTestResult& A, const FAutomatedTestResult& B) {
if (A.GetErrorTotal() > 0)
{
if (B.GetErrorTotal() > 0)
return (A.FullTestPath < B.FullTestPath);
else
return true;
}
else if (B.GetErrorTotal() > 0)
{
return false;
}
if (A.State == EAutomationState::Skipped)
{
if (B.State == EAutomationState::Skipped)
return (A.FullTestPath < B.FullTestPath);
else
return true;
}
else if (B.State == EAutomationState::Skipped)
{
return false;
}
if (A.GetWarningTotal() > 0)
{
if (B.GetWarningTotal() > 0)
return (A.FullTestPath < B.FullTestPath);
else
return true;
}
else if (B.GetWarningTotal() > 0)
{
return false;
}
return A.FullTestPath < B.FullTestPath;
});
}
{
FDateTime StepTime = FDateTime::Now();
UE_LOG(LogAutomationController, Log, TEXT("Writing reports to %s..."), *ReportExportPath);
// Generate Json
GenerateJsonTestPassSummary(SerializedPassResults);
UE_LOG(LogAutomationController, Log, TEXT("Exported report to '%s' in %.02f Seconds"), *ReportExportPath, (FDateTime::Now() - StepTime).GetTotalSeconds());
}
if (!ReportURLPath.IsEmpty())
{
UE_LOG(LogAutomationController, Display, TEXT("Report can be viewed in a browser at '%s'"), DeveloperReportUrl.IsEmpty() ? *ReportURLPath : *DeveloperReportUrl);
if (!DeveloperReportUrl.IsEmpty())
{
UE_LOG(LogAutomationController, Display, TEXT("Launching Report URL %s."), *DeveloperReportUrl);
FPlatformProcess::LaunchURL(*DeveloperReportUrl, nullptr, nullptr);
}
}
UE_LOG(LogAutomationController, Display, TEXT("Report can be opened in the editor at '%s'"), ReportExportPath.IsEmpty() ? *FPaths::ConvertRelativePathToFull(FPaths::AutomationReportsDir()) : *ReportExportPath);
}
// Then clean our array for the next pass.
JsonTestPassResults.ClearAllEntries();
SetControllerStatus(EAutomationControllerModuleState::Ready);
}
void FAutomationControllerManager::CheckChildResult(TSharedPtr<IAutomationReport> InReport)
{
TArray<TSharedPtr<IAutomationReport> >& ChildReports = InReport->GetChildReports();
if ( ChildReports.Num() > 0 )
{
for ( int32 Index = 0; Index < ChildReports.Num(); Index++ )
{
CheckChildResult(ChildReports[Index]);
}
}
else if ( ( bHasErrors && bHasWarning && bHasLogs ) == false && InReport->IsEnabled() )
{
for ( int32 ClusterIndex = 0; ClusterIndex < GetNumDeviceClusters(); ++ClusterIndex )
{
FAutomationTestResults TestResults = InReport->GetResults(ClusterIndex, CurrentTestPass);
if ( TestResults.GetErrorTotal() > 0 )
{
bHasErrors = true;
}
if ( TestResults.GetWarningTotal() )
{
bHasWarning = true;
}
if ( TestResults.GetLogTotal() )
{
bHasLogs = true;
}
}
}
}
void FAutomationControllerManager::SetControllerStatus(EAutomationControllerModuleState::Type InAutomationTestState)
{
if ( InAutomationTestState != AutomationTestState )
{
// Inform the UI if the test state has changed
AutomationTestState = InAutomationTestState;
TestsAvailableDelegate.Broadcast(AutomationTestState);
}
}
void FAutomationControllerManager::RemoveTestRunning(const FMessageAddress& TestAddressToRemove)
{
for ( int32 Index = 0; Index < TestRunningArray.Num(); Index++ )
{
if ( TestRunningArray[Index].OwnerMessageAddress == TestAddressToRemove )
{
TestRunningArray.RemoveAt(Index);
break;
}
}
}
void FAutomationControllerManager::AddPingResult(const FMessageAddress& ResponderAddress)
{
for ( int32 Index = 0; Index < TestRunningArray.Num(); Index++ )
{
if ( TestRunningArray[Index].OwnerMessageAddress == ResponderAddress )
{
TestRunningArray[Index].LastPingTime = 0;
break;
}
}
}
void FAutomationControllerManager::UpdateTests()
{
const double kIgnoreAsFrameHitchDelta = 2.0f;
const double TickDelta = FPlatformTime::Seconds() - LastTimeUpdateTicked;
LastTimeUpdateTicked = FPlatformTime::Seconds();
// If this tick was very long then don't check the health of tests. If we've been blocked for a while on a load and ping responses haven't yet been processed
// then otherwise might add a delta to the ping time that causes the test to look like it's timed out.
if (TickDelta < kIgnoreAsFrameHitchDelta)
{
CheckTestTimer += TickDelta;
if (CheckTestTimer > CheckTestIntervalSeconds)
{
for ( int32 Index = 0; Index < TestRunningArray.Num(); Index++ )
{
TestRunningArray[Index].LastPingTime += CheckTestTimer;
if (TestRunningArray[Index].LastPingTime > GameInstanceLostTimerSeconds)
{
// Find the game session instance info
int32 ClusterIndex;
int32 DeviceIndex;
verify(DeviceClusterManager.FindDevice(TestRunningArray[Index].OwnerMessageAddress, ClusterIndex, DeviceIndex));
//verify this device thought it was busy
TSharedPtr <IAutomationReport> Report = DeviceClusterManager.GetTest(ClusterIndex, DeviceIndex);
check(Report.IsValid());
bHasErrors = true;
FAutomationTestResults TestResults;
TestResults.State = EAutomationState::Fail;
TestResults.GameInstance = DeviceClusterManager.GetClusterGameInstance(ClusterIndex, DeviceIndex);
FString DeviceName = DeviceClusterManager.GetClusterDeviceName(ClusterIndex, DeviceIndex);
TestResults.AddEvent(FAutomationEvent(EAutomationEventType::Error, FString::Printf(TEXT("Timeout waiting for device %s"), *DeviceName)));
UE_LOG(LogAutomationController, Error, TEXT("Removing test and marking failure after timeout waiting for device %s LastPingTime was greater than %.1f"), *DeviceName, GameInstanceLostTimerSeconds);
// Set the results
Report->SetResults(ClusterIndex, CurrentTestPass, TestResults);
bTestResultsAvailable = true;
const FAutomationTestResults& FinalResults = Report->GetResults(ClusterIndex, CurrentTestPass);
// Gather all of the data relevant to this test for our json reporting.
CollectTestResults(Report, FinalResults);
// Disable the device in the cluster so it is not used again
DeviceClusterManager.DisableDevice(ClusterIndex, DeviceIndex);
// Remove the running test
TestRunningArray.RemoveAt(Index--);
// If there are no more devices, set the module state to disabled
if ( DeviceClusterManager.HasActiveDevice() == false )
{
// Process results first to write out the report
ProcessResults();
UE_LOG(LogAutomationController, Display, TEXT("Module disabled"));
SetControllerStatus(EAutomationControllerModuleState::Disabled);
ClusterDistributionMask = 0;
}
else
{
UE_LOG(LogAutomationController, Display, TEXT("Module not disabled. Keep looking."));
// Remove the cluster from the mask if there are no active devices left
if ( DeviceClusterManager.GetNumActiveDevicesInCluster(ClusterIndex) == 0 )
{
ClusterDistributionMask &= ~( 1 << ClusterIndex );
}
if ( TestRunningArray.Num() == 0 )
{
SetControllerStatus(EAutomationControllerModuleState::Ready);
}
}
}
else
{
MessageEndpoint->Send(FMessageEndpoint::MakeMessage<FAutomationWorkerPing>(), TestRunningArray[Index].OwnerMessageAddress);
}
}
CheckTestTimer = 0.f;
}
}
else
{
UE_LOG(LogAutomationController, Log, TEXT("Ignoring very large delta of %.02f seconds in calls to FAutomationControllerManager::Tick() and not penalizing unresponsive tests"), TickDelta);
}
}
const bool FAutomationControllerManager::ExportReport(uint32 FileExportTypeMask)
{
return ReportManager.ExportReport(FileExportTypeMask, GetNumDeviceClusters());
}
bool FAutomationControllerManager::IsTestRunnable(IAutomationReportPtr InReport) const
{
bool bIsRunnable = false;
for ( int32 ClusterIndex = 0; ClusterIndex < GetNumDeviceClusters(); ++ClusterIndex )
{
if ( InReport->IsSupported(ClusterIndex) )
{
if ( GetNumDevicesInCluster(ClusterIndex) >= InReport->GetNumParticipantsRequired() )
{
bIsRunnable = true;
break;
}
}
}
return bIsRunnable;
}
/* FAutomationControllerModule callbacks
*****************************************************************************/
void FAutomationControllerManager::HandleFindWorkersResponseMessage(const FAutomationWorkerFindWorkersResponse& Message, const TSharedRef<IMessageContext, ESPMode::ThreadSafe>& Context)
{
UE_LOG(LogAutomationController, Log, TEXT("Received FindWorkersResponseMessage from %s"), *Context->GetSender().ToString());
if ( Message.SessionId == ActiveSessionId )
{
DeviceClusterManager.AddDeviceFromMessage(Context->GetSender(), Message, DeviceGroupFlags);
}
RequestTests();
SetControllerStatus(EAutomationControllerModuleState::Ready);
}
void FAutomationControllerManager::HandlePongMessage( const FAutomationWorkerPong& Message, const TSharedRef<IMessageContext, ESPMode::ThreadSafe>& Context )
{
AddPingResult(Context->GetSender());
}
void FAutomationControllerManager::HandleReceivedScreenShot(const FAutomationWorkerScreenImage& Message, const TSharedRef<IMessageContext, ESPMode::ThreadSafe>& Context)
{
UE_LOG(LogAutomationController, Log, TEXT("ReceivedScreenShot for %s (%dx%d) from %s"), *Message.Metadata.TestName, Message.Metadata.Width, Message.Metadata.Height, *Context->GetSender().ToString());
bool bTree = true;
FString OutputSubFolder = FPaths::Combine(Message.Metadata.Context, Message.Metadata.ScreenShotName);
FString OutputFolder = FPaths::Combine(FPaths::AutomationTransientDir(), FPaths::GetPath(Message.ScreenShotName));
FString IncomingFileName = FPaths::Combine(OutputFolder,FPaths::GetCleanFilename(Message.ScreenShotName));
FString TraceFileName = FPaths::ChangeExtension(IncomingFileName, TEXT(".rdc"));
FString DirectoryPath = FPaths::GetPath(IncomingFileName);
if (!IFileManager::Get().MakeDirectory(*DirectoryPath, bTree))
{
UE_LOG(LogAutomationController, Error, TEXT("Failed to create directory %s for incoming screenshot"), *DirectoryPath);
return;
}
if (!FFileHelper::SaveArrayToFile(Message.ScreenImage, *IncomingFileName))
{
uint32 WriteErrorCode = FPlatformMisc::GetLastError();
TCHAR WriteErrorBuffer[2048];
FPlatformMisc::GetSystemErrorMessage(WriteErrorBuffer, 2048, WriteErrorCode);
UE_LOG(LogAutomationController, Warning, TEXT("Fail to save screenshot to %s. WriteError: %u (%s)"), *IncomingFileName, WriteErrorCode, WriteErrorBuffer);
return;
}
if (Message.FrameTrace.Num() > 0)
{
if (!FFileHelper::SaveArrayToFile(Message.FrameTrace, *TraceFileName))
{
uint32 WriteErrorCode = FPlatformMisc::GetLastError();
TCHAR WriteErrorBuffer[2048];
FPlatformMisc::GetSystemErrorMessage(WriteErrorBuffer, 2048, WriteErrorCode);
UE_LOG(LogAutomationController, Warning, TEXT("Faild to save frame trace to %s. WriteError: %u (%s)"), *TraceFileName, WriteErrorCode, WriteErrorBuffer);
}
}
// TODO Automation There is identical code in, Engine\Source\Runtime\AutomationWorker\Private\AutomationWorkerModule.cpp,
// need to move this code into common area.
FString Json;
if ( FJsonObjectConverter::UStructToJsonObjectString(Message.Metadata, Json) )
{
FString MetadataPath = FPaths::ChangeExtension(IncomingFileName, TEXT("json"));
FFileHelper::SaveStringToFile(Json, *MetadataPath, FFileHelper::EEncodingOptions::ForceUTF8WithoutBOM);
}
// compare the incoming image and throw it away afterwards (note - there will be a copy in the report)
TSharedRef<FComparisonEntry> Comparison = MakeShareable(new FComparisonEntry());
Comparison->Sender = Context->GetSender();
Comparison->ScreenshotPath = Message.Metadata.Context / Message.Metadata.ScreenShotName;
Comparison->PendingComparison = ScreenshotManager->CompareScreenshotAsync(IncomingFileName, Message.Metadata, EScreenShotCompareOptions::DiscardImage);
ComparisonQueue.Enqueue(Comparison);
}
void FAutomationControllerManager::HandleTestDataRequest(const FAutomationWorkerTestDataRequest& Message, const TSharedRef<IMessageContext, ESPMode::ThreadSafe>& Context)
{
UE_LOG(LogAutomationController, Log, TEXT("Received TestDataRequest from %s"), *Context->GetSender().ToString());
const FString TestDataRoot = FPaths::ConvertRelativePathToFull(FPaths::ProjectDir() / TEXT("Test"));
const FString DataFile = Message.DataType / Message.DataPlatform / Message.DataTestName / Message.DataName + TEXT(".json");
const FString DataFullPath = TestDataRoot / DataFile;
// Generate the folder for the data if it doesn't exist.
const bool bTree = true;
IFileManager::Get().MakeDirectory(*FPaths::GetPath(DataFile), bTree);
bool bIsNew = true;
FString ResponseJsonData = Message.JsonData;
if ( FPaths::FileExists(DataFullPath) )
{
if ( FFileHelper::LoadFileToString(ResponseJsonData, *DataFullPath) )
{
bIsNew = false;
}
else
{
// TODO Error
}
}
if ( bIsNew )
{
FString IncomingTestData = FPaths::Combine(FPaths::AutomationTransientDir(), TEXT("Data/"), DataFile);
if ( FFileHelper::SaveStringToFile(Message.JsonData, *IncomingTestData) )
{
//TODO Anything extra to do here?
}
else
{
//TODO What do we do if this fails?
}
}
FAutomationWorkerTestDataResponse* ResponseMessage = FMessageEndpoint::MakeMessage<FAutomationWorkerTestDataResponse>();
ResponseMessage->bIsNew = bIsNew;
ResponseMessage->JsonData = ResponseJsonData;
MessageEndpoint->Send(ResponseMessage, Context->GetSender());
}
void FAutomationControllerManager::HandlePerformanceDataRequest(const FAutomationWorkerPerformanceDataRequest& Message, const TSharedRef<IMessageContext, ESPMode::ThreadSafe>& Context)
{
//TODO Read/Performance data.
UE_LOG(LogAutomationController, Log, TEXT("Received PerformanceDataRequest from %s"), *Context->GetSender().ToString());
FAutomationWorkerPerformanceDataResponse* ResponseMessage = FMessageEndpoint::MakeMessage<FAutomationWorkerPerformanceDataResponse>();
ResponseMessage->bSuccess = true;
ResponseMessage->ErrorMessage = TEXT("");
MessageEndpoint->Send(ResponseMessage, Context->GetSender());
}
void FAutomationControllerManager::HandleRequestNextNetworkCommandMessage(const FAutomationWorkerRequestNextNetworkCommand& Message, const TSharedRef<IMessageContext, ESPMode::ThreadSafe>& Context)
{
UE_LOG(LogAutomationController, Log, TEXT("Received RequestNextNetworkCommandMessage from %s"), *Context->GetSender().ToString());
// Harvest iteration of running the tests this result came from (stops stale results from being committed to subsequent runs)
if ( Message.ExecutionCount == ExecutionCount )
{
// Find the device id for the address
int32 ClusterIndex;
int32 DeviceIndex;
verify(DeviceClusterManager.FindDevice(Context->GetSender(), ClusterIndex, DeviceIndex));
// Verify this device thought it was busy
TSharedPtr<IAutomationReport> Report = DeviceClusterManager.GetTest(ClusterIndex, DeviceIndex);
check(Report.IsValid());
// Increment network command responses
bool bAllResponsesReceived = Report->IncrementNetworkCommandResponses();
// Test if we've accumulated all responses AND this was the result for the round of test running AND we're still running tests
if ( bAllResponsesReceived && ( ClusterDistributionMask & ( 1 << ClusterIndex ) ) )
{
// Reset the counter
Report->ResetNetworkCommandResponses();
// For every device in this networked test
TArray<FMessageAddress> DeviceAddresses = DeviceClusterManager.GetDevicesReservedForTest(ClusterIndex, Report);
check(DeviceAddresses.Num() == Report->GetNumParticipantsRequired());
// Send it to each device
for ( int32 AddressIndex = 0; AddressIndex < DeviceAddresses.Num(); ++AddressIndex )
{
//send "next command message" to worker
MessageEndpoint->Send(FMessageEndpoint::MakeMessage<FAutomationWorkerNextNetworkCommandReply>(), DeviceAddresses[AddressIndex]);
}
}
}
}
void FAutomationControllerManager::HandleRequestTestsReplyCompleteMessage(const FAutomationWorkerRequestTestsReplyComplete& Message, const TSharedRef<IMessageContext, ESPMode::ThreadSafe>& Context)
{
UE_LOG(LogAutomationController, Log, TEXT("Received RequestTestsReplyCompleteMessage from %s"), *Context->GetSender().ToString());
LLM_SCOPE_BYNAME(TEXT("AutomationTest/Controller"));
TArray<FAutomationTestInfo> TestInfo;
TestInfo.Reset(Message.Tests.Num());
for (const FAutomationWorkerSingleTestReply& SingleTestReply : Message.Tests)
{
FAutomationTestInfo NewTest = SingleTestReply.GetTestInfo();
TestInfo.Add(NewTest);
}
UE_LOG(LogAutomationController, Log, TEXT("%d tests available on %s"), TestInfo.Num(), *Context->GetSender().ToString());
SetTestNames(Context->GetSender(), TestInfo);
}
void FAutomationControllerManager::HandleReceivedTelemetryData(const FAutomationWorkerTelemetryData& Message, const TSharedRef<IMessageContext, ESPMode::ThreadSafe>& Context)
{
FAutomationTelemetry::HandleAddTelemetry(Message);
}
void FAutomationControllerManager::ReportAutomationResult(const TSharedPtr<IAutomationReport> InReport, int32 ClusterIndex, int32 PassIndex)
{
FName CategoryName = "LogAutomationController";
#if WITH_EDITOR
FMessageLog AutomationEditorLog("AutomationTestingLog");
// we log these messages ourselves for non-editor platforms so suppress this.
AutomationEditorLog.SuppressLoggingToOutputLog(true);
AutomationEditorLog.Open();
#endif
const FAutomationTestResults& Results = InReport->GetResults(ClusterIndex, PassIndex);
const FString StateString = AutomationStateToString(Results.State);
// write results to editor panel
#if WITH_EDITOR
FFormatNamedArguments Arguments;
Arguments.Add(TEXT("TestName"), FText::FromString(InReport->GetFullTestPath()));
Arguments.Add(TEXT("Result"), FText::FromString(StateString));
TSharedRef<FTextToken> Token = FTextToken::Create(FText::Format(LOCTEXT("TestSuccessOrFailure", "Test '{TestName}' completed with result '{Result}'"), Arguments));
if (Results.State == EAutomationState::Success)
{
AutomationEditorLog.Info()->AddToken(Token);
}
else if (Results.State == EAutomationState::Skipped)
{
AutomationEditorLog.Warning()->AddToken(Token);
}
else
{
AutomationEditorLog.Error()->AddToken(Token);
}
#endif
// Now log
FString ResultString = FString::Printf(AutomationStateFormat, *StateString, *InReport->GetDisplayName(), *InReport->GetFullTestPath());
if (Results.State == EAutomationState::Fail)
{
UE_LOG(LogAutomationController, Error, TEXT("%s"), *ResultString);
}
else
{
UE_LOG(LogAutomationController, Display, TEXT("%s"), *ResultString);
}
// bracket these for easy parsing
UE_LOG(LogAutomationController, Log, BeginEventsFormat, *InReport->GetFullTestPath());
for (const FAutomationExecutionEntry& Entry : Results.GetEntries())
{
#if WITH_EDITOR
TSharedRef<FTextToken> TextToken = FTextToken::Create(FText::FromString(Entry.ToStringFormattedEditorLog()), false);
#endif
switch (Entry.Event.Type)
{
case EAutomationEventType::Info:
UE_LOG(LogAutomationController, Log, TEXT("%s"), *Entry.ToString());
#if WITH_EDITOR
AutomationEditorLog.Info()->AddToken(TextToken);
#endif
break;
case EAutomationEventType::Warning:
UE_LOG(LogAutomationController, Warning, TEXT("%s"), *Entry.ToString());
#if WITH_EDITOR
AutomationEditorLog.Warning()->AddToken(TextToken);
#endif
break;
case EAutomationEventType::Error:
UE_LOG(LogAutomationController, Error, TEXT("%s"), *Entry.ToString());
#if WITH_EDITOR
AutomationEditorLog.Error()->AddToken(TextToken);
#endif
break;
}
}
UE_LOG(LogAutomationController, Log, EndEventsFormat, *InReport->GetFullTestPath());
#undef AutomationSuccessFormat
#undef AutomationFailureFormat
#undef BeginEventsFormat
#undef EndEventsFormat
}
void FAutomationControllerManager::HandleRunTestsReplyMessage(const FAutomationWorkerRunTestsReply& Message, const TSharedRef<IMessageContext, ESPMode::ThreadSafe>& Context)
{
// If we should commit these results
if ( Message.ExecutionCount == ExecutionCount )
{
FAutomationTestResults TestResults;
TestResults.State = Message.State;
TestResults.Duration = Message.Duration;
// Mark device as back on the market
int32 ClusterIndex;
int32 DeviceIndex;
verify(DeviceClusterManager.FindDevice(Context->GetSender(), ClusterIndex, DeviceIndex));
TestResults.GameInstance = DeviceClusterManager.GetClusterGameInstance(ClusterIndex, DeviceIndex);
TestResults.SetEvents(Message.Entries, Message.WarningTotal, Message.ErrorTotal);
// Verify this device thought it was busy
TSharedPtr<IAutomationReport> Report = DeviceClusterManager.GetTest(ClusterIndex, DeviceIndex);
if (Report.IsValid())
{
Report->SetResults(ClusterIndex, CurrentTestPass, TestResults);
const FAutomationTestResults& FinalResults = Report->GetResults(ClusterIndex, CurrentTestPass);
ReportAutomationResult(Report, ClusterIndex, CurrentTestPass);
// Gather all of the data relevant to this test for our json reporting.
CollectTestResults(Report, FinalResults);
}
// Device is now good to go
DeviceClusterManager.SetTest(ClusterIndex, DeviceIndex, NULL);
}
// Remove the running test
RemoveTestRunning(Context->GetSender());
}
void FAutomationControllerManager::HandleWorkerOfflineMessage( const FAutomationWorkerWorkerOffline& Message, const TSharedRef<IMessageContext, ESPMode::ThreadSafe>& Context )
{
FMessageAddress DeviceMessageAddress = Context->GetSender();
DeviceClusterManager.Remove(DeviceMessageAddress);
}
bool FAutomationControllerManager::IsDeviceGroupFlagSet( EAutomationDeviceGroupTypes::Type InDeviceGroup ) const
{
const uint32 FlagMask = 1 << InDeviceGroup;
return (DeviceGroupFlags & FlagMask) > 0;
}
void FAutomationControllerManager::ToggleDeviceGroupFlag( EAutomationDeviceGroupTypes::Type InDeviceGroup )
{
const uint32 FlagMask = 1 << InDeviceGroup;
DeviceGroupFlags = DeviceGroupFlags ^ FlagMask;
}
void FAutomationControllerManager::UpdateDeviceGroups( )
{
DeviceClusterManager.ReGroupDevices( DeviceGroupFlags );
// Update the reports in case the number of clusters changed
int32 NumOfClusters = DeviceClusterManager.GetNumClusters();
ReportManager.ClustersUpdated(NumOfClusters);
}
void FAutomationControllerManager::ResetAutomationTestTimeout(const TCHAR* Reason)
{
//GLog->Logf(ELogVerbosity::Display, TEXT("Resetting automation test timeout: %s"), Reason);
LastTimeUpdateTicked = FPlatformTime::Seconds();
}
#undef LOCTEXT_NAMESPACE
Reference in New Issue View Git Blame Copy Permalink