// Copyright 1998-2017 Epic Games, Inc. All Rights Reserved. #include "AutomationAnalytics.h" #include "Misc/CommandLine.h" #include "Misc/App.h" #include "Misc/EngineVersion.h" #include "Interfaces/IAutomationWorkerModule.h" #include "AutomationWorkerMessages.h" #include "AutomationWorkerPrivate.h" #include "AnalyticsEventAttribute.h" #include "IAnalyticsProviderET.h" #include "AnalyticsET.h" DEFINE_LOG_CATEGORY(LogAutomationAnalytics); bool FAutomationAnalytics::bIsInitialized; TSharedPtr FAutomationAnalytics::Analytics; TArray FAutomationAnalytics::AutomationEventNames; TArray FAutomationAnalytics::AutomationParamNames; FString FAutomationAnalytics::MachineSpec; /** * Get analytics pointer */ IAnalyticsProvider& FAutomationAnalytics::GetProvider() { checkf(bIsInitialized && IsAvailable(), TEXT("FAutomationAnalytics::GetProvider called outside of Initialize/Shutdown.")); return *Analytics.Get(); } void FAutomationAnalytics::Initialize() { checkf(!bIsInitialized, TEXT("FAutomationAnalytics::Initialize called more than once.")); //Set the config for Analytics Analytics = FAnalyticsET::Get().CreateAnalyticsProvider(FAnalyticsET::Config(FString::Printf(TEXT("AutomationAnalytics.%s"), GInternalGameName), TEXT("https://datarouter.ol.epicgames.com/"))); if (Analytics.IsValid()) { Analytics->SetUserID(FString::Printf(TEXT("%s|%s|%s"), *FPlatformMisc::GetLoginId(), *FPlatformMisc::GetEpicAccountId(), *FPlatformMisc::GetOperatingSystemId())); Analytics->StartSession(); MachineSpec = FParse::Param(FCommandLine::Get(), TEXT("60hzmin")) ? TEXT("60hzmin") : FParse::Param(FCommandLine::Get(), TEXT("30hzmin")) ? TEXT("30hzmin") : TEXT(""); bIsInitialized = true; InititalizeAnalyticParameterNames(); } } void FAutomationAnalytics::Shutdown() { Analytics.Reset(); bIsInitialized = false; } FString FAutomationAnalytics::GetAutomationEventName(EAutomationEventName::Type InEventName) { check(InEventName < AutomationEventNames.Num()); return AutomationEventNames[InEventName]; } FString FAutomationAnalytics::GetAutomationParamName(EAutomationAnalyticParam::Type InEngineParam) { check(InEngineParam < AutomationParamNames.Num()); return AutomationParamNames[InEngineParam]; } void FAutomationAnalytics::InititalizeAnalyticParameterNames() { // Engine Event Names AutomationEventNames.Reserve(EAutomationEventName::NUM_ENGINE_EVENT_NAMES); AutomationEventNames.SetNum(EAutomationEventName::NUM_ENGINE_EVENT_NAMES); AutomationEventNames[EAutomationEventName::FPSCapture] = TEXT("FPSCapture"); AutomationEventNames[EAutomationEventName::AutomationTestResults] = TEXT("AutomationTest"); for (int32 EventIndex = 0; EventIndex < EAutomationEventName::NUM_ENGINE_EVENT_NAMES; EventIndex++) { if (AutomationEventNames[EventIndex].IsEmpty()) { UE_LOG(LogAutomationAnalytics, Error, TEXT("Engine Events missing a event: %d!"), EventIndex); } } // Engine Event Params AutomationParamNames.Reserve(EAutomationAnalyticParam::NUM_ENGINE_PARAMS); AutomationParamNames.SetNum(EAutomationAnalyticParam::NUM_ENGINE_PARAMS); AutomationParamNames[EAutomationAnalyticParam::MapName] = TEXT("MapName"); AutomationParamNames[EAutomationAnalyticParam::MatineeName] = TEXT("MatineeName"); AutomationParamNames[EAutomationAnalyticParam::TimeStamp] = TEXT("TimeStamp"); AutomationParamNames[EAutomationAnalyticParam::Platform] = TEXT("Platform"); AutomationParamNames[EAutomationAnalyticParam::Spec] = TEXT("Spec"); AutomationParamNames[EAutomationAnalyticParam::CL] = TEXT("CL"); AutomationParamNames[EAutomationAnalyticParam::FPS] = TEXT("FPS"); AutomationParamNames[EAutomationAnalyticParam::BuildConfiguration] = TEXT("BuildConfiguration"); AutomationParamNames[EAutomationAnalyticParam::AverageFrameTime] = TEXT("AverageFrameTime"); AutomationParamNames[EAutomationAnalyticParam::AverageGameThreadTime] = TEXT("AverageGameThreadTime"); AutomationParamNames[EAutomationAnalyticParam::AverageRenderThreadTime] = TEXT("AverageRenderThreadTime"); AutomationParamNames[EAutomationAnalyticParam::AverageGPUTime] = TEXT("AverageGPUTime"); AutomationParamNames[EAutomationAnalyticParam::PercentOfFramesAtLeast30FPS] = TEXT("PercentOfFramesAtLeast30FPS"); AutomationParamNames[EAutomationAnalyticParam::PercentOfFramesAtLeast60FPS] = TEXT("PercentOfFramesAtLeast60FPS"); AutomationParamNames[EAutomationAnalyticParam::TestName] = TEXT("TestName"); AutomationParamNames[EAutomationAnalyticParam::BeautifiedName] = TEXT("BeautifiedName"); AutomationParamNames[EAutomationAnalyticParam::ExecutionCount] = TEXT("ExecutionCount"); AutomationParamNames[EAutomationAnalyticParam::IsSuccess] = TEXT("IsSuccess"); AutomationParamNames[EAutomationAnalyticParam::Duration] = TEXT("Duration"); AutomationParamNames[EAutomationAnalyticParam::ErrorCount] = TEXT("ErrorCount"); AutomationParamNames[EAutomationAnalyticParam::WarningCount] = TEXT("WarningCount"); for (int32 ParamIndex = 0; ParamIndex < EAutomationAnalyticParam::NUM_ENGINE_PARAMS; ParamIndex++) { if (AutomationParamNames[ParamIndex].IsEmpty()) { UE_LOG(LogAutomationAnalytics, Error, TEXT("Engine Events missing a param: %d!"), ParamIndex); } } } //Timestamp,Platform void FAutomationAnalytics::SetInitialParameters(TArray& ParamArray) { int32 TimeStamp; FString PlatformName; TimeStamp = FApp::GetCurrentTime(); PlatformName = FPlatformProperties::PlatformName(); ParamArray.Add(FAnalyticsEventAttribute(GetAutomationParamName(EAutomationAnalyticParam::TimeStamp), TimeStamp)); ParamArray.Add(FAnalyticsEventAttribute(GetAutomationParamName(EAutomationAnalyticParam::Platform), PlatformName)); } /* * @EventName FPSCapture * * @Trigger * * @Type Static * * @EventParam TimeStamp int32 The time in seconds when the test when the capture ended * @EventParam Platform string The platform in which the test is run on * @EventParam CL string The Changelist for the build * @EventParam Spec string The spec of the machine running the test * @EventParam MapName string The map that the test was run on * @EventParam MatineeName string The name of the matinee that fired the event * @EventParam FPS string * @EventParam BuildConfiguration string Debug/Development/Test/Shipping * @EventParam AverageFrameTime string Time for a frame in ms * @EventParam AverageGameThreadTime string Time for the game thread in ms * @EventParam AverageRenderThreadTime string Time for the rendering thread in ms * @EventParam AverageGPUTime string Time for the GPU to flush in ms * @EventParam PercentOfFramesAtLeast30FPS string * @EventParam PercentOfFramesAtLeast60FPS string * * @Comments * */ void FAutomationAnalytics::FireEvent_FPSCapture(const FAutomationPerformanceSnapshot& PerfSnapshot) { if (Analytics.IsValid()) { TArray ParamArray; SetInitialParameters(ParamArray); ParamArray.Add(FAnalyticsEventAttribute(GetAutomationParamName(EAutomationAnalyticParam::CL), PerfSnapshot.Changelist)); ParamArray.Add(FAnalyticsEventAttribute(GetAutomationParamName(EAutomationAnalyticParam::Spec), MachineSpec)); ParamArray.Add(FAnalyticsEventAttribute(GetAutomationParamName(EAutomationAnalyticParam::MapName), PerfSnapshot.MapName)); ParamArray.Add(FAnalyticsEventAttribute(GetAutomationParamName(EAutomationAnalyticParam::MatineeName), PerfSnapshot.MatineeName)); ParamArray.Add(FAnalyticsEventAttribute(GetAutomationParamName(EAutomationAnalyticParam::FPS), PerfSnapshot.AverageFPS)); ParamArray.Add(FAnalyticsEventAttribute(GetAutomationParamName(EAutomationAnalyticParam::BuildConfiguration), PerfSnapshot.BuildConfiguration)); ParamArray.Add(FAnalyticsEventAttribute(GetAutomationParamName(EAutomationAnalyticParam::AverageFrameTime), PerfSnapshot.AverageFrameTime)); ParamArray.Add(FAnalyticsEventAttribute(GetAutomationParamName(EAutomationAnalyticParam::AverageGameThreadTime), PerfSnapshot.AverageGameThreadTime)); ParamArray.Add(FAnalyticsEventAttribute(GetAutomationParamName(EAutomationAnalyticParam::AverageRenderThreadTime), PerfSnapshot.AverageRenderThreadTime)); ParamArray.Add(FAnalyticsEventAttribute(GetAutomationParamName(EAutomationAnalyticParam::AverageGPUTime), PerfSnapshot.AverageGPUTime)); ParamArray.Add(FAnalyticsEventAttribute(GetAutomationParamName(EAutomationAnalyticParam::PercentOfFramesAtLeast30FPS), PerfSnapshot.PercentOfFramesAtLeast30FPS)); ParamArray.Add(FAnalyticsEventAttribute(GetAutomationParamName(EAutomationAnalyticParam::PercentOfFramesAtLeast60FPS), PerfSnapshot.PercentOfFramesAtLeast60FPS)); Analytics->RecordEvent(GetAutomationEventName(EAutomationEventName::FPSCapture), ParamArray); } } /* * @EventName AutomationTestResults * * @Trigger Fires when an automation test has completed * * @Type Static * * @EventParam TimeStamp int32 The time in seconds when the test ended * @EventParam Platform string The platform in which the test is run on * @EventParam CL string The Changelist for the build * @EventParam TestName string The execution string of the test being run * @EventParam BeautifiedTestName string The name displayed in the UI * @EventParam ExecutionCount string * @EventParam IsSuccess bool Where the test was a success or not * @EventParam Duration float The duration the test took to complete * @EventParam ErrorCount int32 The amount of errors during the test * @EventParam WarningCount int32 The amount of warnings during the test * @EventParam CL string The Changelist for the build * * @Comments * */ void FAutomationAnalytics::FireEvent_AutomationTestResults(const FAutomationWorkerRunTestsReply* TestResults, const FString& BeautifiedTestName) { if (Analytics.IsValid()) { TArray ParamArray; SetInitialParameters(ParamArray); ParamArray.Add(FAnalyticsEventAttribute(GetAutomationParamName(EAutomationAnalyticParam::TestName), TestResults->TestName)); ParamArray.Add(FAnalyticsEventAttribute(GetAutomationParamName(EAutomationAnalyticParam::BeautifiedName), BeautifiedTestName)); ParamArray.Add(FAnalyticsEventAttribute(GetAutomationParamName(EAutomationAnalyticParam::ExecutionCount), TestResults->ExecutionCount)); ParamArray.Add(FAnalyticsEventAttribute(GetAutomationParamName(EAutomationAnalyticParam::IsSuccess), TestResults->Success)); ParamArray.Add(FAnalyticsEventAttribute(GetAutomationParamName(EAutomationAnalyticParam::Duration), TestResults->Duration)); ParamArray.Add(FAnalyticsEventAttribute(GetAutomationParamName(EAutomationAnalyticParam::ErrorCount), TestResults->ErrorTotal)); ParamArray.Add(FAnalyticsEventAttribute(GetAutomationParamName(EAutomationAnalyticParam::WarningCount), TestResults->WarningTotal)); const int32 ChangeList = FEngineVersion::Current().GetChangelist(); FString ChangeListString = FString::FromInt(ChangeList); ParamArray.Add(FAnalyticsEventAttribute(GetAutomationParamName(EAutomationAnalyticParam::CL), ChangeListString)); Analytics->RecordEvent(GetAutomationEventName(EAutomationEventName::AutomationTestResults), ParamArray); } } bool FAutomationAnalytics::IsInitialized() { return bIsInitialized; }