You've already forked UnrealEngineUWP
mirror of
https://github.com/izzy2lost/UnrealEngineUWP.git
synced 2026-03-26 18:15:20 -07:00
60839c1816
* Rename blacklist to excludelist (requested by high management) * Support section exclusion rule to be able to exclude entire section of tests (ie PathTracing is only supported on Win64 for now) * Mark excluded test as skipped in the report instead of entirely removed for test list. Check for exclusion just before running the test. * Remove NotEnoughParticipant state in favor of Skipped (same conditions lead to Skipped state with appropriate messaging) * Add support for exclusion management from the Test Automation window. (added a column at the end of each row) * Expose device information to UE test report * Add support for metadata in Gauntlet test report for Horde Limitations: * Management through the UI is limited to which test is available through in the active worker node. That's mean Runtime only tests are not listed from a worker that is Editor(the default) and platform specific are not clearly identified. * For platforms, the mechanic to access their config and save it will remain to be done. In the meantime, it needs to be done manually through the target platform config file. #jira UE-125960 #jira UE-125974 #rb Chris.Constantinescu, Eric.Knapik, Louise.Rasmussen [CL 17607554 by Jerome Delattre in ue5-main branch]
253 lines
11 KiB
C++
253 lines
11 KiB
C++
// Copyright Epic Games, Inc. All Rights Reserved.
|
|
|
|
#include "AutomationAnalytics.h"
|
|
#include "AutomationWorkerPrivate.h"
|
|
|
|
#include "AnalyticsET.h"
|
|
#include "AnalyticsEventAttribute.h"
|
|
#include "AutomationWorkerMessages.h"
|
|
#include "Interfaces/IAnalyticsProvider.h"
|
|
#include "IAnalyticsProviderET.h"
|
|
#include "Misc/App.h"
|
|
#include "Misc/CommandLine.h"
|
|
#include "Misc/EngineVersion.h"
|
|
|
|
#include "IAutomationWorkerModule.h"
|
|
|
|
|
|
DEFINE_LOG_CATEGORY(LogAutomationAnalytics);
|
|
|
|
|
|
/* Static initialization
|
|
*****************************************************************************/
|
|
|
|
bool FAutomationAnalytics::bIsInitialized;
|
|
TSharedPtr<IAnalyticsProviderET> FAutomationAnalytics::Analytics;
|
|
TArray<FString> FAutomationAnalytics::AutomationEventNames;
|
|
TArray<FString> FAutomationAnalytics::AutomationParamNames;
|
|
FString FAutomationAnalytics::MachineSpec;
|
|
|
|
|
|
/* FAutomationAnalytics static functions
|
|
*****************************************************************************/
|
|
|
|
IAnalyticsProvider& FAutomationAnalytics::GetProvider()
|
|
{
|
|
checkf(bIsInitialized && IsAvailable(), TEXT("FAutomationAnalytics::GetProvider called outside of Initialize/Shutdown."));
|
|
|
|
return *Analytics.Get();
|
|
}
|
|
|
|
|
|
void FAutomationAnalytics::Initialize()
|
|
{
|
|
checkf(!bIsInitialized, TEXT("FAutomationAnalytics::Initialize called more than once."));
|
|
|
|
//Set the config for Analytics
|
|
Analytics = FAnalyticsET::Get().CreateAnalyticsProvider(FAnalyticsET::Config(FString::Printf(TEXT("AutomationAnalytics.%s"), GInternalProjectName), TEXT("https://datarouter.ol.epicgames.com/")));
|
|
if (Analytics.IsValid())
|
|
{
|
|
Analytics->SetUserID(FString::Printf(TEXT("%s|%s|%s"), *FPlatformMisc::GetLoginId(), *FPlatformMisc::GetEpicAccountId(), *FPlatformMisc::GetOperatingSystemId()));
|
|
Analytics->StartSession();
|
|
|
|
MachineSpec = FParse::Param(FCommandLine::Get(), TEXT("60hzmin"))
|
|
? TEXT("60hzmin")
|
|
: FParse::Param(FCommandLine::Get(), TEXT("30hzmin"))
|
|
? TEXT("30hzmin")
|
|
: TEXT("");
|
|
|
|
bIsInitialized = true;
|
|
InititalizeAnalyticParameterNames();
|
|
}
|
|
}
|
|
|
|
|
|
bool FAutomationAnalytics::IsAvailable()
|
|
{
|
|
return Analytics.IsValid();
|
|
}
|
|
|
|
|
|
void FAutomationAnalytics::Shutdown()
|
|
{
|
|
Analytics.Reset();
|
|
bIsInitialized = false;
|
|
}
|
|
|
|
|
|
FString FAutomationAnalytics::GetAutomationEventName(EAutomationEventName::Type InEventName)
|
|
{
|
|
check(InEventName < AutomationEventNames.Num());
|
|
return AutomationEventNames[InEventName];
|
|
}
|
|
|
|
|
|
FString FAutomationAnalytics::GetAutomationParamName(EAutomationAnalyticParam::Type InEngineParam)
|
|
{
|
|
check(InEngineParam < AutomationParamNames.Num());
|
|
return AutomationParamNames[InEngineParam];
|
|
}
|
|
|
|
|
|
void FAutomationAnalytics::InititalizeAnalyticParameterNames()
|
|
{
|
|
// Engine Event Names
|
|
AutomationEventNames.Reserve(EAutomationEventName::NUM_ENGINE_EVENT_NAMES);
|
|
AutomationEventNames.SetNum(EAutomationEventName::NUM_ENGINE_EVENT_NAMES);
|
|
|
|
AutomationEventNames[EAutomationEventName::FPSCapture] = TEXT("FPSCapture");
|
|
AutomationEventNames[EAutomationEventName::AutomationTestResults] = TEXT("AutomationTest");
|
|
|
|
for (int32 EventIndex = 0; EventIndex < EAutomationEventName::NUM_ENGINE_EVENT_NAMES; EventIndex++)
|
|
{
|
|
if (AutomationEventNames[EventIndex].IsEmpty())
|
|
{
|
|
UE_LOG(LogAutomationAnalytics, Error, TEXT("Engine Events missing a event: %d!"), EventIndex);
|
|
}
|
|
}
|
|
|
|
// Engine Event Params
|
|
AutomationParamNames.Reserve(EAutomationAnalyticParam::NUM_ENGINE_PARAMS);
|
|
AutomationParamNames.SetNum(EAutomationAnalyticParam::NUM_ENGINE_PARAMS);
|
|
|
|
AutomationParamNames[EAutomationAnalyticParam::MapName] = TEXT("MapName");
|
|
AutomationParamNames[EAutomationAnalyticParam::MatineeName] = TEXT("MatineeName");
|
|
AutomationParamNames[EAutomationAnalyticParam::TimeStamp] = TEXT("TimeStamp");
|
|
AutomationParamNames[EAutomationAnalyticParam::Platform] = TEXT("Platform");
|
|
AutomationParamNames[EAutomationAnalyticParam::Spec] = TEXT("Spec");
|
|
AutomationParamNames[EAutomationAnalyticParam::CL] = TEXT("CL");
|
|
AutomationParamNames[EAutomationAnalyticParam::FPS] = TEXT("FPS");
|
|
AutomationParamNames[EAutomationAnalyticParam::BuildConfiguration] = TEXT("BuildConfiguration");
|
|
AutomationParamNames[EAutomationAnalyticParam::AverageFrameTime] = TEXT("AverageFrameTime");
|
|
AutomationParamNames[EAutomationAnalyticParam::AverageGameThreadTime] = TEXT("AverageGameThreadTime");
|
|
AutomationParamNames[EAutomationAnalyticParam::AverageRenderThreadTime] = TEXT("AverageRenderThreadTime");
|
|
AutomationParamNames[EAutomationAnalyticParam::AverageGPUTime] = TEXT("AverageGPUTime");
|
|
AutomationParamNames[EAutomationAnalyticParam::PercentOfFramesAtLeast30FPS] = TEXT("PercentOfFramesAtLeast30FPS");
|
|
AutomationParamNames[EAutomationAnalyticParam::PercentOfFramesAtLeast60FPS] = TEXT("PercentOfFramesAtLeast60FPS");
|
|
|
|
AutomationParamNames[EAutomationAnalyticParam::TestName] = TEXT("TestName");
|
|
AutomationParamNames[EAutomationAnalyticParam::BeautifiedName] = TEXT("BeautifiedName");
|
|
AutomationParamNames[EAutomationAnalyticParam::ExecutionCount] = TEXT("ExecutionCount");
|
|
AutomationParamNames[EAutomationAnalyticParam::IsSuccess] = TEXT("IsSuccess");
|
|
AutomationParamNames[EAutomationAnalyticParam::Duration] = TEXT("Duration");
|
|
AutomationParamNames[EAutomationAnalyticParam::ErrorCount] = TEXT("ErrorCount");
|
|
AutomationParamNames[EAutomationAnalyticParam::WarningCount] = TEXT("WarningCount");
|
|
|
|
for (int32 ParamIndex = 0; ParamIndex < EAutomationAnalyticParam::NUM_ENGINE_PARAMS; ParamIndex++)
|
|
{
|
|
if (AutomationParamNames[ParamIndex].IsEmpty())
|
|
{
|
|
UE_LOG(LogAutomationAnalytics, Error, TEXT("Engine Events missing a param: %d!"), ParamIndex);
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
void FAutomationAnalytics::SetInitialParameters(TArray<FAnalyticsEventAttribute>& ParamArray)
|
|
{
|
|
int32 TimeStamp;
|
|
FString PlatformName;
|
|
|
|
TimeStamp = FApp::GetCurrentTime();
|
|
PlatformName = FPlatformProperties::PlatformName();
|
|
|
|
ParamArray.Add(FAnalyticsEventAttribute(GetAutomationParamName(EAutomationAnalyticParam::TimeStamp), TimeStamp));
|
|
ParamArray.Add(FAnalyticsEventAttribute(GetAutomationParamName(EAutomationAnalyticParam::Platform), PlatformName));
|
|
}
|
|
|
|
|
|
void FAutomationAnalytics::FireEvent_FPSCapture(const FAutomationPerformanceSnapshot& PerfSnapshot)
|
|
{
|
|
// @EventName FPSCapture
|
|
//
|
|
// @Trigger
|
|
//
|
|
// @Type Static
|
|
//
|
|
// @EventParam TimeStamp int32 The time in seconds when the test when the capture ended
|
|
// @EventParam Platform string The platform in which the test is run on
|
|
// @EventParam CL string The Changelist for the build
|
|
// @EventParam Spec string The spec of the machine running the test
|
|
// @EventParam MapName string The map that the test was run on
|
|
// @EventParam MatineeName string The name of the matinee that fired the event
|
|
// @EventParam FPS string
|
|
// @EventParam BuildConfiguration string Debug/Development/Test/Shipping
|
|
// @EventParam AverageFrameTime string Time for a frame in ms
|
|
// @EventParam AverageGameThreadTime string Time for the game thread in ms
|
|
// @EventParam AverageRenderThreadTime string Time for the rendering thread in ms
|
|
// @EventParam AverageGPUTime string Time for the GPU to flush in ms
|
|
// @EventParam PercentOfFramesAtLeast30FPS string
|
|
// @EventParam PercentOfFramesAtLeast60FPS string
|
|
//
|
|
// @Comments
|
|
|
|
if (Analytics.IsValid())
|
|
{
|
|
TArray<FAnalyticsEventAttribute> ParamArray;
|
|
SetInitialParameters(ParamArray);
|
|
ParamArray.Add(FAnalyticsEventAttribute(GetAutomationParamName(EAutomationAnalyticParam::CL), PerfSnapshot.Changelist));
|
|
ParamArray.Add(FAnalyticsEventAttribute(GetAutomationParamName(EAutomationAnalyticParam::Spec), MachineSpec));
|
|
ParamArray.Add(FAnalyticsEventAttribute(GetAutomationParamName(EAutomationAnalyticParam::MapName), PerfSnapshot.MapName));
|
|
ParamArray.Add(FAnalyticsEventAttribute(GetAutomationParamName(EAutomationAnalyticParam::MatineeName), PerfSnapshot.MatineeName));
|
|
ParamArray.Add(FAnalyticsEventAttribute(GetAutomationParamName(EAutomationAnalyticParam::FPS), PerfSnapshot.AverageFPS));
|
|
|
|
ParamArray.Add(FAnalyticsEventAttribute(GetAutomationParamName(EAutomationAnalyticParam::BuildConfiguration), PerfSnapshot.BuildConfiguration));
|
|
ParamArray.Add(FAnalyticsEventAttribute(GetAutomationParamName(EAutomationAnalyticParam::AverageFrameTime), PerfSnapshot.AverageFrameTime));
|
|
ParamArray.Add(FAnalyticsEventAttribute(GetAutomationParamName(EAutomationAnalyticParam::AverageGameThreadTime), PerfSnapshot.AverageGameThreadTime));
|
|
ParamArray.Add(FAnalyticsEventAttribute(GetAutomationParamName(EAutomationAnalyticParam::AverageRenderThreadTime), PerfSnapshot.AverageRenderThreadTime));
|
|
ParamArray.Add(FAnalyticsEventAttribute(GetAutomationParamName(EAutomationAnalyticParam::AverageGPUTime), PerfSnapshot.AverageGPUTime));
|
|
ParamArray.Add(FAnalyticsEventAttribute(GetAutomationParamName(EAutomationAnalyticParam::PercentOfFramesAtLeast30FPS), PerfSnapshot.PercentOfFramesAtLeast30FPS));
|
|
ParamArray.Add(FAnalyticsEventAttribute(GetAutomationParamName(EAutomationAnalyticParam::PercentOfFramesAtLeast60FPS), PerfSnapshot.PercentOfFramesAtLeast60FPS));
|
|
|
|
Analytics->RecordEvent(GetAutomationEventName(EAutomationEventName::FPSCapture), ParamArray);
|
|
}
|
|
}
|
|
|
|
|
|
void FAutomationAnalytics::FireEvent_AutomationTestResults(const FAutomationWorkerRunTestsReply* TestResults, const FString& BeautifiedTestName)
|
|
{
|
|
// @EventName AutomationTestResults
|
|
//
|
|
// @Trigger Fires when an automation test has completed
|
|
//
|
|
// @Type Static
|
|
//
|
|
// @EventParam TimeStamp int32 The time in seconds when the test ended
|
|
// @EventParam Platform string The platform in which the test is run on
|
|
// @EventParam CL string The Changelist for the build
|
|
// @EventParam TestName string The execution string of the test being run
|
|
// @EventParam BeautifiedTestName string The name displayed in the UI
|
|
// @EventParam ExecutionCount string
|
|
// @EventParam IsSuccess bool Where the test was a success or not
|
|
// @EventParam Duration float The duration the test took to complete
|
|
// @EventParam ErrorCount int32 The amount of errors during the test
|
|
// @EventParam WarningCount int32 The amount of warnings during the test
|
|
// @EventParam CL string The Changelist for the build
|
|
//
|
|
// @Comments
|
|
|
|
if (Analytics.IsValid())
|
|
{
|
|
TArray<FAnalyticsEventAttribute> ParamArray;
|
|
SetInitialParameters(ParamArray);
|
|
ParamArray.Add(FAnalyticsEventAttribute(GetAutomationParamName(EAutomationAnalyticParam::TestName), TestResults->TestName));
|
|
ParamArray.Add(FAnalyticsEventAttribute(GetAutomationParamName(EAutomationAnalyticParam::BeautifiedName), BeautifiedTestName));
|
|
ParamArray.Add(FAnalyticsEventAttribute(GetAutomationParamName(EAutomationAnalyticParam::ExecutionCount), TestResults->ExecutionCount));
|
|
ParamArray.Add(FAnalyticsEventAttribute(GetAutomationParamName(EAutomationAnalyticParam::IsSuccess), TestResults->State == EAutomationState::Success));
|
|
ParamArray.Add(FAnalyticsEventAttribute(GetAutomationParamName(EAutomationAnalyticParam::Duration), TestResults->Duration));
|
|
ParamArray.Add(FAnalyticsEventAttribute(GetAutomationParamName(EAutomationAnalyticParam::ErrorCount), TestResults->ErrorTotal));
|
|
ParamArray.Add(FAnalyticsEventAttribute(GetAutomationParamName(EAutomationAnalyticParam::WarningCount), TestResults->WarningTotal));
|
|
|
|
const int32 ChangeList = FEngineVersion::Current().GetChangelist();
|
|
FString ChangeListString = FString::FromInt(ChangeList);
|
|
ParamArray.Add(FAnalyticsEventAttribute(GetAutomationParamName(EAutomationAnalyticParam::CL), ChangeListString));
|
|
|
|
Analytics->RecordEvent(GetAutomationEventName(EAutomationEventName::AutomationTestResults), ParamArray);
|
|
}
|
|
}
|
|
|
|
bool FAutomationAnalytics::IsInitialized()
|
|
{
|
|
return bIsInitialized;
|
|
}
|