UnrealEngineUWP/Engine/Source/Editor/UnrealEd/Private/Commandlets/SummarizeTraceCommandlet.cpp

// Copyright Epic Games, Inc. All Rights Reserved.

/*=============================================================================
	SummarizeTraceCommandlet.cpp: Commandlet for summarizing a utrace
=============================================================================*/

#include "Commandlets/SummarizeTraceCommandlet.h"

#include "Containers/StringConv.h"
#include "GenericPlatform/GenericPlatformFile.h"
#include "HAL/PlatformFileManager.h"
#include "Misc/Crc.h"
#include "Misc/FileHelper.h"
#include "Misc/Paths.h"
#include "String/ParseTokens.h"
#include "Trace/Analysis.h"
#include "Trace/Analyzer.h"
#include "Trace/DataStream.h"
#include "TraceServices/Model/Log.h"
#include "TraceServices/Utils.h"
#include "ProfilingDebugging/CountersTrace.h"

/*
 * The following could be in TraceServices. This way if the format of CPU scope
 * events change this interface acts as a compatibility contract for external
 * tools. In the future it may be in a separate library so that Trace and
 * Insights' instrumentation are more broadly available.
 */

static uint64 Decode7bit(const uint8*& Cursor)
{
	uint64 Value = 0;
	uint64 ByteIndex = 0;
	bool bHasMoreBytes;
	do
	{
		uint8 ByteValue = *Cursor++;
		bHasMoreBytes = ByteValue & 0x80;
		Value |= uint64(ByteValue & 0x7f) << (ByteIndex * 7);
		++ByteIndex;
	} 	while (bHasMoreBytes);
	return Value;
}

/**
 * Base class to extend for CPU scope analysis. Derived class instances are meant to be registered as
 * delegates to the FCpuScopeStreamProcessor to handle scope events and perform meaningful analysis.
 */
class FCpuScopeAnalyzer
{
public:
	enum class EScopeEventType : uint32
	{
		Enter,
		Exit
	};

	struct FScopeEvent
	{
		EScopeEventType ScopeEventType;
		uint32 ScopeId;
		uint32 ThreadId;
		double Timestamp; // As Seconds
	};

	struct FScope
	{
		uint32 ScopeId;
		uint32 ThreadId;
		double EnterTimestamp; // As Seconds
		double ExitTimestamp;  // As Seconds
	};

public:
	virtual ~FCpuScopeAnalyzer() = default;

	/** Invoked when a CPU scope is discovered. This function is always invoked first when a CPU scope is encountered for the first time.*/
	virtual void OnCpuScopeDiscovered(uint32 ScopeId) {}

	/** Invoked when CPU scope specification is encountered in the trace stream. */
	virtual void OnCpuScopeName(uint32 ScopeId, const FString& ScopeName) {};

	/** Invoked when a scope is entered. The scope name might not be known yet. */
	virtual void OnCpuScopeEnter(const FScopeEvent& ScopeEnter, const FString* ScopeName) {};

	/** Invoked when a scope is exited. The scope name might not be known yet. */
	virtual void OnCpuScopeExit(const FScope& Scope, const FString* ScopeName) {};

	/** Invoked when a root event on the specified thread along with all child events down to the leaves are know. */
	virtual void OnCpuScopeTree(uint32 ThreadId, const TArray<FCpuScopeAnalyzer::FScopeEvent>& ScopeEvents, const TFunction<const FString*(uint32)>& ScopeLookupNameFn) {};

	/** Invoked when the trace stream has been fully consumed/processed. */
	virtual void OnCpuScopeAnalysisEnd() {};

	static constexpr uint32 CoroutineSpecId        = (1u << 31u) - 1u;
	static constexpr uint32 CoroutineUnknownSpecId = (1u << 31u) - 2u;
};

/**
 * Decodes, format and routes CPU scope events embedded into a trace stream to specialized CPU scope analyzers. This processor
 * decodes the low level events, keeps a small state and publishes higher level events to registered analyzers. The purpose
 * is to decode the stream only once and let several registered analyzers reuse the small state built up by this processor. This
 * matters when processing very large traces with possibly billions of scope events.
 */
class FCpuScopeStreamProcessor
	: public UE::Trace::IAnalyzer
{
public:
	FCpuScopeStreamProcessor();

	/** Register an analyzer with this processor. The processor decodes the trace stream and invokes the registered analyzers when a CPU scope event occurs.*/
	void AddCpuScopeAnalyzer(TSharedPtr<FCpuScopeAnalyzer> Analyzer);

private:
	struct FScopeEnter
	{
		uint32 ScopeId;
		double Timestamp; // As Seconds
	};

	// Contains scope events for a root scope and its children along with extra info to analyze that tree at once.
	struct FScopeTreeInfo
	{
		// Records the current root scope and its children to run analysis that needs to know the parent/child relationship.
		TArray<FCpuScopeAnalyzer::FScopeEvent> ScopeEvents;

		// Indicates if one of the scope in the current hierarchy is nameless. (Its names specs hasn't been received yet).
		bool bHasNamelessScopes = false;

		void Reset()
		{
			ScopeEvents.Reset();
			bHasNamelessScopes = false;
		}
	};

	// For each thread we track what the stack of scopes are, for matching end-to-start
	struct FThread
	{
		TArray<FScopeEnter> ScopeStack;

		// The events recorded for the current root scope and its children to run analysis that needs to know the parent/child relationship, for example to compute time including childreen and time
		// excluding childreen.
		FScopeTreeInfo ScopeTreeInfo;

		// Scope trees for which as least one scope name was unknown. Some analysis need scope names, but scope names/event can be emitted out of order by the engine depending on thread scheduling.
		// Some scope tree cannot be analyzed right away and need to be delayed until all scope names are discovered.
		TArray<FScopeTreeInfo> DelayedScopeTreeInfo;
	};

private:
	// UE::Trace::IAnalyzer interface.
	virtual void OnAnalysisBegin(const FOnAnalysisContext& Context) override;
	virtual void OnAnalysisEnd() override;
	virtual bool OnEvent(uint16 RouteId, EStyle Style, const FOnEventContext& Context) override;

	// Internal implementation.
	void OnEventSpec(const FOnEventContext& Context);
	void OnBatch(const FOnEventContext& Context);
	void OnBatchV2(const FOnEventContext& Context);
	void OnCpuScopeSpec(uint32 ScopeId, const FString* ScopeName);
	void OnCpuScopeEnter(uint32 ScopeId, uint32 ThreadId, double Timestamp);
	void OnCpuScopeExit(uint32 ThreadId, double Timestamp);
	void OnCpuScopeTree(uint32 ThreadId, const FScopeTreeInfo& ScopeTreeInfo);
	const FString* LookupScopeName(uint32 ScopeId) { return ScopeId < static_cast<uint32>(ScopeNames.Num()) && ScopeNames[ScopeId] ? &ScopeNames[ScopeId].GetValue() : nullptr; }

private:
	// The state at any moment of the threads, indexes are doled out on the process-side
	TArray<FThread> Threads;

	// The scope names, the array index correspond to the scope Id. If the optional is not set, the scope hasn't been encountered yet.
	TArray<TOptional<FString>> ScopeNames;

	// List of analyzers to invoke when a scope event is decoded.
	TArray<TSharedPtr<FCpuScopeAnalyzer>> ScopeAnalyzers;

	// Scope name lookup function, cached for efficiency.
	TFunction<const FString*(uint32 ScopeId)> LookupScopeNameFn;


};

enum
{
	// CpuProfilerTrace.cpp
	RouteId_CpuProfiler_EventSpec,
	RouteId_CpuProfiler_EventBatch,
	RouteId_CpuProfiler_EndCapture,
	RouteId_CpuProfiler_EventBatchV2,
	RouteId_CpuProfiler_EndCaptureV2
};

FCpuScopeStreamProcessor::FCpuScopeStreamProcessor()
 : LookupScopeNameFn([this](uint32 ScopeId) { return LookupScopeName(ScopeId); })
{
}

void FCpuScopeStreamProcessor::AddCpuScopeAnalyzer(TSharedPtr<FCpuScopeAnalyzer> Analyzer)
{
	ScopeAnalyzers.Add(MoveTemp(Analyzer));
}

void FCpuScopeStreamProcessor::OnAnalysisBegin(const FOnAnalysisContext& Context)
{
	Context.InterfaceBuilder.RouteEvent(RouteId_CpuProfiler_EventSpec, "CpuProfiler", "EventSpec");
	Context.InterfaceBuilder.RouteEvent(RouteId_CpuProfiler_EventBatch, "CpuProfiler", "EventBatch");
	Context.InterfaceBuilder.RouteEvent(RouteId_CpuProfiler_EndCapture, "CpuProfiler", "EndCapture");
	Context.InterfaceBuilder.RouteEvent(RouteId_CpuProfiler_EventBatchV2, "CpuProfiler", "EventBatchV2");
	Context.InterfaceBuilder.RouteEvent(RouteId_CpuProfiler_EndCaptureV2, "CpuProfiler", "EndCaptureV2");
}

void FCpuScopeStreamProcessor::OnAnalysisEnd()
{
	// Analyze scope trees that contained 'nameless' context when they were captured. Unless the trace was truncated,
	// all scope names should be known now.
	uint32 ThreadId = 0;
	for (FThread& Thread : Threads)
	{
		for (FScopeTreeInfo& DelayedScopeTree : Threads[ThreadId].DelayedScopeTreeInfo)
		{
			// Run summary analysis for this delayed hierarchy.
			OnCpuScopeTree(ThreadId, DelayedScopeTree);
		}

		++ThreadId;
	}

	for (TSharedPtr<FCpuScopeAnalyzer>& Analyzer : ScopeAnalyzers)
	{
		Analyzer->OnCpuScopeAnalysisEnd();
	}
}

bool FCpuScopeStreamProcessor::OnEvent(uint16 RouteId, EStyle Style, const FOnEventContext& Context)
{
	switch (RouteId)
	{
	case RouteId_CpuProfiler_EventSpec:
		OnEventSpec(Context);
		break;

	case RouteId_CpuProfiler_EventBatch:
	case RouteId_CpuProfiler_EndCapture:
		OnBatch(Context);
		break;
	case RouteId_CpuProfiler_EventBatchV2:
	case RouteId_CpuProfiler_EndCaptureV2:
		OnBatchV2(Context);
		break;
	};

	return true;
}

void FCpuScopeStreamProcessor::OnEventSpec(const FOnEventContext& Context)
{
	const FEventData& EventData = Context.EventData;
	FString Name;
	uint32 Id = EventData.GetValue<uint32>("Id");
	EventData.GetString("Name", Name);
	OnCpuScopeSpec(Id - 1, &Name);
}

void FCpuScopeStreamProcessor::OnBatch(const FOnEventContext& Context)
{
	const FEventData& EventData = Context.EventData;
	const FEventTime& EventTime = Context.EventTime;

	uint32 ThreadId = Context.ThreadInfo.GetId();

	TArrayView<const uint8> DataView = TraceServices::FTraceAnalyzerUtils::LegacyAttachmentArray("Data", Context);
	const uint8* Cursor = DataView.GetData();
	const uint8* End = Cursor + DataView.Num();
	uint64 LastCycle = 0;
	while (Cursor < End)
	{
		uint64 Value = Decode7bit(Cursor);
		uint64 Cycle = LastCycle + (Value >> 1);
		LastCycle = Cycle;

		double TimeStamp = EventTime.AsSeconds(Cycle);
		if (Value & 1)
		{
			uint64 ScopeId = Decode7bit(Cursor);
			OnCpuScopeEnter(static_cast<uint32>(ScopeId - 1), ThreadId, TimeStamp);
		}
		else
		{
			OnCpuScopeExit(ThreadId, TimeStamp);
		}
	}
}

void FCpuScopeStreamProcessor::OnBatchV2(const FOnEventContext& Context)
{
	const FEventData& EventData = Context.EventData;
	const FEventTime& EventTime = Context.EventTime;

	uint32 ThreadId = Context.ThreadInfo.GetId();

	TArrayView<const uint8> DataView = TraceServices::FTraceAnalyzerUtils::LegacyAttachmentArray("Data", Context);
	const uint8* Cursor = DataView.GetData();
	const uint8* End = Cursor + DataView.Num();
	uint64 LastCycle = 0;
	while (Cursor < End)
	{
		uint64 Value = Decode7bit(Cursor);
		uint64 Cycle = LastCycle + (Value >> 2);
		LastCycle = Cycle;
		double TimeStamp = EventTime.AsSeconds(Cycle);

		if (Value & 2ull)
		{		
			if (Value & 1ull)			
			{
				uint64 CoroutineId = Decode7bit(Cursor);
				uint32 TimerScopeDepth = Decode7bit(Cursor);
				OnCpuScopeEnter(FCpuScopeAnalyzer::CoroutineSpecId, ThreadId, TimeStamp);
				for (uint32 i = 0; i < TimerScopeDepth; ++i)
				{
					OnCpuScopeEnter(FCpuScopeAnalyzer::CoroutineUnknownSpecId, ThreadId, TimeStamp);
				}
			}
			else
			{			
				uint32 TimerScopeDepth = Decode7bit(Cursor);
				for (uint32 i = 0; i < TimerScopeDepth; ++i)
				{
					OnCpuScopeExit(ThreadId, TimeStamp);
				}
				OnCpuScopeExit(ThreadId, TimeStamp);
			}
		}
		else
		{
			if (Value & 1)
			{
				uint64 ScopeId = Decode7bit(Cursor);
				OnCpuScopeEnter(static_cast<uint32>(ScopeId - 1), ThreadId, TimeStamp);
			}
			else
			{
				OnCpuScopeExit(ThreadId, TimeStamp);
			}
		}
	}
}

void FCpuScopeStreamProcessor::OnCpuScopeSpec(uint32 ScopeId, const FString* ScopeName)
{
	if (ScopeId >= uint32(ScopeNames.Num()))
	{
		uint32 Num = (ScopeId + 128) & ~127;
		ScopeNames.SetNum(Num);
	}

	// If the optional is not set, the scope hasn't been encountered yet (Set to a blank names means encountered, but names hasn't be encountered yet)
	bool bDiscovered = !ScopeNames[ScopeId].IsSet();

	// Store the discovery state (setting the optional) and the actual name if known.
	ScopeNames[ScopeId].Emplace(ScopeName ? *ScopeName : FString());

	// Notify the analyzers.
	for (TSharedPtr<FCpuScopeAnalyzer>& Analyzer : ScopeAnalyzers)
	{
		if (bDiscovered)
		{
			Analyzer->OnCpuScopeDiscovered(ScopeId);
		}
		if (ScopeName)
		{
			Analyzer->OnCpuScopeName(ScopeId, *ScopeName);
		}
	}
}

void FCpuScopeStreamProcessor::OnCpuScopeEnter(uint32 ScopeId, uint32 ThreadId, double Timestamp)
{
	if (ThreadId >= uint32(Threads.Num()))
	{
		Threads.SetNum(ThreadId + 1);
	}

	FCpuScopeAnalyzer::FScopeEvent ScopeEvent{FCpuScopeAnalyzer::EScopeEventType::Enter, ScopeId, ThreadId, Timestamp};
	Threads[ThreadId].ScopeStack.Add(FScopeEnter{ScopeId, Timestamp});
	Threads[ThreadId].ScopeTreeInfo.ScopeEvents.Add(ScopeEvent);

	// Scope specs/events can be received out of order depending on engine thread scheduling.
	if (ScopeId >= static_cast<uint32>(ScopeNames.Num()) || !ScopeNames[ScopeId].IsSet())
	{
		// This is a newly discovered scope. Create an entry for it and notify the discovery.
		OnCpuScopeSpec(ScopeId, nullptr);
	}

	// Notify the registered scope analyzers.
	for (TSharedPtr<FCpuScopeAnalyzer>& Analyzer : ScopeAnalyzers)
	{
		Analyzer->OnCpuScopeEnter(ScopeEvent, LookupScopeName(ScopeId));
	}
}

void FCpuScopeStreamProcessor::OnCpuScopeExit(uint32 ThreadId, double Timestamp)
{
	if (ThreadId >= uint32(Threads.Num()) || Threads[ThreadId].ScopeStack.Num() <= 0)
	{
		return;
	}

	FScopeEnter ScopeEnter = Threads[ThreadId].ScopeStack.Pop();
	
	Threads[ThreadId].ScopeTreeInfo.ScopeEvents.Add(FCpuScopeAnalyzer::FScopeEvent{FCpuScopeAnalyzer::EScopeEventType::Exit, ScopeEnter.ScopeId, ThreadId, Timestamp});
	Threads[ThreadId].ScopeTreeInfo.bHasNamelessScopes |= ScopeNames[ScopeEnter.ScopeId].GetValue().IsEmpty();

	// Notify the registered scope analyzers.
	FCpuScopeAnalyzer::FScope Scope{ScopeEnter.ScopeId, ThreadId, ScopeEnter.Timestamp, Timestamp};
	for (TSharedPtr<FCpuScopeAnalyzer>& Analyzer : ScopeAnalyzers)
	{
		Analyzer->OnCpuScopeExit(Scope, LookupScopeName(ScopeEnter.ScopeId));
	}

	// The root scope on this thread just poped out.
	if (Threads[ThreadId].ScopeStack.IsEmpty())
	{
		if (Threads[ThreadId].ScopeTreeInfo.bHasNamelessScopes)
		{
			// Delay the analysis until all the scope names are known.
			Threads[ThreadId].DelayedScopeTreeInfo.Add(MoveTemp(Threads[ThreadId].ScopeTreeInfo));
		}
		else
		{
			// Run analysis for this scope tree.
			OnCpuScopeTree(ThreadId, Threads[ThreadId].ScopeTreeInfo);
		}

		Threads[ThreadId].ScopeTreeInfo.Reset();
	}
}

void FCpuScopeStreamProcessor::OnCpuScopeTree(uint32 ThreadId, const FScopeTreeInfo& ScopeTreeInfo)
{
	for (TSharedPtr<FCpuScopeAnalyzer>& Analyzer : ScopeAnalyzers)
	{
		Analyzer->OnCpuScopeTree(ThreadId, ScopeTreeInfo.ScopeEvents, LookupScopeNameFn);
	}
}


class FCountersAnalyzer
	: public UE::Trace::IAnalyzer
{
public:
	struct FCounterName
	{
		const TCHAR*		Name;
		ETraceCounterType	Type;
		uint16				Id;
	};

	struct FCounterIntValue
	{
		uint16				Id;
		int64				Value;
	};

	struct FCounterFloatValue
	{
		uint16				Id;
		double				Value;
	};

	virtual void		OnCounterName(const FCounterName& CounterName) = 0;
	virtual void		OnCounterIntValue(const FCounterIntValue& NewValue) = 0;
	virtual void		OnCounterFloatValue(const FCounterFloatValue& NewValue) = 0;

private:
	virtual void		OnAnalysisBegin(const FOnAnalysisContext& Context) override;
	virtual bool		OnEvent(uint16 RouteId, EStyle Style, const FOnEventContext& Context) override;
	void				OnCountersSpec(const FOnEventContext& Context);
	void				OnCountersSetValueInt(const FOnEventContext& Context);
	void				OnCountersSetValueFloat(const FOnEventContext& Context);
};

enum
{
	// CountersTrace.cpp
	RouteId_Counters_Spec,
	RouteId_Counters_SetValueInt,
	RouteId_Counters_SetValueFloat,
};

void FCountersAnalyzer::OnAnalysisBegin(const FOnAnalysisContext& Context)
{
	Context.InterfaceBuilder.RouteEvent(RouteId_Counters_Spec, "Counters", "Spec");
	Context.InterfaceBuilder.RouteEvent(RouteId_Counters_SetValueInt, "Counters", "SetValueInt");
	Context.InterfaceBuilder.RouteEvent(RouteId_Counters_SetValueFloat, "Counters", "SetValueFloat");
}

bool FCountersAnalyzer::OnEvent(uint16 RouteId, EStyle Style, const FOnEventContext& Context)
{
	switch (RouteId)
	{
	case RouteId_Counters_Spec:
		OnCountersSpec(Context);
		break;

	case RouteId_Counters_SetValueInt:
		OnCountersSetValueInt(Context);
		break;

	case RouteId_Counters_SetValueFloat:
		OnCountersSetValueFloat(Context);
		break;
	}

	return true;
}

void FCountersAnalyzer::OnCountersSpec(const FOnEventContext& Context)
{
	const FEventData& EventData = Context.EventData;
	FString Name;
	uint16 Id = EventData.GetValue<uint16>("Id");
	ETraceCounterType Type = static_cast<ETraceCounterType>(EventData.GetValue<uint8>("Type"));
	EventData.GetString("Name", Name);
	OnCounterName({ *Name, Type, uint16(Id - 1) });
}

void FCountersAnalyzer::OnCountersSetValueInt(const FOnEventContext& Context)
{
	const FEventData& EventData = Context.EventData;
	uint16 CounterId = EventData.GetValue<uint16>("CounterId");
	int64 Value = EventData.GetValue<int64>("Value");
	OnCounterIntValue({ uint16(CounterId - 1), Value });
}

void FCountersAnalyzer::OnCountersSetValueFloat(const FOnEventContext& Context)
{
	const FEventData& EventData = Context.EventData;
	uint16 CounterId = EventData.GetValue<uint16>("CounterId");
	double Value = EventData.GetValue<double>("Value");
	OnCounterFloatValue({ uint16(CounterId - 1), Value });
}

class FBookmarksAnalyzer
	: public UE::Trace::IAnalyzer
{
public:
	struct FBookmarkSpecEvent
	{
		uint64			Id;
		const TCHAR*	FileName;
		int32			Line;
		const TCHAR*	FormatString;
	};

	struct FBookmarkEvent
	{
		uint64					Id;
		double					Timestamp;
		TArrayView<const uint8>	FormatArgs;
	};

	virtual void		OnBookmarkSpecEvent(const FBookmarkSpecEvent& BookmarkSpecEvent) = 0;
	virtual void		OnBookmarkEvent(const FBookmarkEvent& BookmarkEvent) = 0;

private:
	virtual void		OnAnalysisBegin(const FOnAnalysisContext& Context) override;
	virtual bool		OnEvent(uint16 RouteId, EStyle Style, const FOnEventContext& Context) override;
	void				OnBookmarksSpec(const FOnEventContext& Context);
	void				OnBookmarksBookmark(const FOnEventContext& Context);
};

enum
{
	// MiscTrace.cpp
	RouteId_Bookmarks_BookmarkSpec,
	RouteId_Bookmarks_Bookmark,
};

void FBookmarksAnalyzer::OnAnalysisBegin(const FOnAnalysisContext& Context)
{
	Context.InterfaceBuilder.RouteEvent(RouteId_Bookmarks_BookmarkSpec, "Misc", "BookmarkSpec");
	Context.InterfaceBuilder.RouteEvent(RouteId_Bookmarks_Bookmark, "Misc", "Bookmark");
}

bool FBookmarksAnalyzer::OnEvent(uint16 RouteId, EStyle Style, const FOnEventContext& Context)
{
	switch (RouteId)
	{
	case RouteId_Bookmarks_BookmarkSpec:
		OnBookmarksSpec(Context);
		break;

	case RouteId_Bookmarks_Bookmark:
		OnBookmarksBookmark(Context);
		break;
	}

	return true;
}

void FBookmarksAnalyzer::OnBookmarksSpec(const FOnEventContext& Context)
{
	const FEventData& EventData = Context.EventData;
	uint64 Id = EventData.GetValue<uint64>("BookmarkPoint");
	int32 Line = EventData.GetValue<int32>("Line");
	FString FileName;
	EventData.GetString("FileName", FileName);
	FString FormatString;
	EventData.GetString("FormatString", FormatString);
	OnBookmarkSpecEvent({ Id, *FileName, Line, *FormatString });
}

void FBookmarksAnalyzer::OnBookmarksBookmark(const FOnEventContext& Context)
{
	const FEventData& EventData = Context.EventData;
	uint64 Id = EventData.GetValue<uint64>("BookmarkPoint");
	uint64 Cycle = EventData.GetValue<uint64>("Cycle");
	double Timestamp = Context.EventTime.AsSeconds(Cycle);
	TArrayView<const uint8> FormatArgsView = TraceServices::FTraceAnalyzerUtils::LegacyAttachmentArray("FormatArgs", Context);
	OnBookmarkEvent({ Id, Timestamp, FormatArgsView });
}

/*
 * This too could be housed elsewhere, along with an API to make it easier to
 * run analysis on trace files. The current model is influenced a little too much
 * on the store model that Insights' browser mode hosts.
 */

class FFileDataStream : public UE::Trace::IInDataStream
{
public:
	FFileDataStream()
		: Handle(nullptr)
	{
	}

	~FFileDataStream()
	{
		delete Handle;
	}

	bool Open(const TCHAR* Path)
	{
		Handle = FPlatformFileManager::Get().GetPlatformFile().OpenRead(Path);
		if (Handle == nullptr)
		{
			return false;
		}
		Remaining = Handle->Size();
		return true;
	}

	virtual int32 Read(void* Data, uint32 Size) override
	{
		if (Remaining <= 0 || Handle == nullptr)
		{
			return 0;
		}

		Size = (Size < Remaining) ? Size : Remaining;
		Remaining -= Size;
		return Handle->Read((uint8*)Data, Size) ? Size : 0;
	}

	IFileHandle* Handle;
	uint64 Remaining;
};

/*
 * Helper classes for the SummarizeTrace commandlet. Aggregates statistics about a trace.
 */

struct FSummarizeScope
{
	FString Name;
	uint64 Count = 0;
	double TotalDurationSeconds = 0.0;

	double FirstStartSeconds = 0.0;
	double FirstFinishSeconds = 0.0;
	double FirstDurationSeconds = 0.0;

	double LastStartSeconds = 0.0;
	double LastFinishSeconds = 0.0;
	double LastDurationSeconds = 0.0;

	double MinDurationSeconds = 1e10;
	double MaxDurationSeconds = -1e10;
	double MeanDurationSeconds = 0.0;
	double VarianceAcc = 0.0; // Accumulator for Welford's

	void AddDuration(double StartSeconds, double FinishSeconds)
	{
		Count += 1;

		// compute the duration
		double DurationSeconds = FinishSeconds - StartSeconds;

		// only set first for the first sample, compare exact zero
		if (FirstStartSeconds == 0.0)
		{
			FirstStartSeconds = StartSeconds;
			FirstFinishSeconds = FinishSeconds;
			FirstDurationSeconds = DurationSeconds;
		}

		LastStartSeconds = StartSeconds;
		LastFinishSeconds = FinishSeconds;
		LastDurationSeconds = DurationSeconds;

		// set duration statistics
		TotalDurationSeconds += DurationSeconds;
		MinDurationSeconds = FMath::Min(MinDurationSeconds, DurationSeconds);
		MaxDurationSeconds = FMath::Max(MaxDurationSeconds, DurationSeconds);
		UpdateVariance(DurationSeconds);
	}

	void UpdateVariance(double DurationSeconds)
	{
		ensure(Count);

		// Welford's increment
		double OldMeanDurationSeconds = MeanDurationSeconds;
		MeanDurationSeconds = MeanDurationSeconds + ((DurationSeconds - MeanDurationSeconds) / double(Count));
		VarianceAcc = VarianceAcc + ((DurationSeconds - MeanDurationSeconds) * (DurationSeconds - OldMeanDurationSeconds));
	}

	double GetDeviationDurationSeconds() const
	{
		if (Count > 1)
		{
			// Welford's final step, dependent on sample count
			double VarianceSecondsSquared = VarianceAcc / double(Count - 1);

			// stddev is sqrt of variance, to restore to units of seconds (vs. seconds squared)
			return sqrt(VarianceSecondsSquared);
		}
		else
		{
			return 0.0;
		}
	}

	void Merge(const FSummarizeScope& Scope)
	{
		check(Name == Scope.Name);
		TotalDurationSeconds += Scope.TotalDurationSeconds;
		MinDurationSeconds = FMath::Min(MinDurationSeconds, Scope.MinDurationSeconds);
		MaxDurationSeconds = FMath::Max(MaxDurationSeconds, Scope.MaxDurationSeconds);
		Count += Scope.Count;
	}

	FString GetValue(const FStringView& Statistic) const
	{
		if (Statistic == TEXT("Name"))
		{
			return Name;
		}
		else if (Statistic == TEXT("Count"))
		{
			return FString::Printf(TEXT("%llu"), Count);
		}
		else if (Statistic == TEXT("TotalDurationSeconds"))
		{
			return FString::Printf(TEXT("%f"), TotalDurationSeconds);
		}
		else if (Statistic == TEXT("FirstStartSeconds"))
		{
			return FString::Printf(TEXT("%f"), FirstStartSeconds);
		}
		else if (Statistic == TEXT("FirstFinishSeconds"))
		{
			return FString::Printf(TEXT("%f"), FirstFinishSeconds);
		}
		else if (Statistic == TEXT("FirstDurationSeconds"))
		{
			return FString::Printf(TEXT("%f"), FirstDurationSeconds);
		}
		else if (Statistic == TEXT("LastStartSeconds"))
		{
			return FString::Printf(TEXT("%f"), LastStartSeconds);
		}
		else if (Statistic == TEXT("LastFinishSeconds"))
		{
			return FString::Printf(TEXT("%f"), LastFinishSeconds);
		}
		else if (Statistic == TEXT("LastDurationSeconds"))
		{
			return FString::Printf(TEXT("%f"), LastDurationSeconds);
		}
		else if (Statistic == TEXT("MinDurationSeconds"))
		{
			return FString::Printf(TEXT("%f"), MinDurationSeconds);
		}
		else if (Statistic == TEXT("MaxDurationSeconds"))
		{
			return FString::Printf(TEXT("%f"), MaxDurationSeconds);
		}
		else if (Statistic == TEXT("MeanDurationSeconds"))
		{
			return FString::Printf(TEXT("%f"), MeanDurationSeconds);
		}
		else if (Statistic == TEXT("DeviationDurationSeconds"))
		{
			return FString::Printf(TEXT("%f"), GetDeviationDurationSeconds());
		}
		return FString();
	}

	// for deduplication
	bool operator==(const FSummarizeScope& Scope) const
	{
		return Name == Scope.Name;
	}

	// for sorting descending
	bool operator<(const FSummarizeScope& Scope) const
	{
		return TotalDurationSeconds > Scope.TotalDurationSeconds;
	}
};

static uint32 GetTypeHash(const FSummarizeScope& Scope)
{
	return FCrc::StrCrc32(*Scope.Name);
}

struct FSummarizeBookmark
{
	FString Name;
	uint64 Count = 0;

	double FirstSeconds = 0.0;
	double LastSeconds = 0.0;

	void AddTimestamp(double Seconds)
	{
		Count += 1;

		// only set first for the first sample, compare exact zero
		if (FirstSeconds == 0.0)
		{
			FirstSeconds = Seconds;
		}

		LastSeconds = Seconds;
	}

	FString GetValue(const FStringView& Statistic) const
	{
		if (Statistic == TEXT("Name"))
		{
			return Name;
		}
		else if (Statistic == TEXT("Count"))
		{
			return FString::Printf(TEXT("%llu"), Count);
		}
		else if (Statistic == TEXT("FirstSeconds"))
		{
			return FString::Printf(TEXT("%f"), FirstSeconds);
		}
		else if (Statistic == TEXT("LastSeconds"))
		{
			return FString::Printf(TEXT("%f"), LastSeconds);
		}
		return FString();
	}

	// for deduplication
	bool operator==(const FSummarizeBookmark& Bookmark) const
	{
		return Name == Bookmark.Name;
	}
};

static uint32 GetTypeHash(const FSummarizeBookmark& Bookmark)
{
	return FCrc::StrCrc32(*Bookmark.Name);
}

/**
 * This analyzer aggregates CPU scopes having the same name together, counting the number of occurrences, total duration,
 * average occurrence duration, for each scope name.
 */
class FSummarizeCpuAnalyzer
	: public FCpuScopeAnalyzer
{
public:
	FSummarizeCpuAnalyzer(TFunction<void(const TArray<FSummarizeScope>&)> InPublishFn);

protected:
	virtual void OnCpuScopeDiscovered(uint32 ScopeId) override;
	virtual void OnCpuScopeName(uint32 ScopeId, const FString& ScopeName) override;
	virtual void OnCpuScopeExit(const FScope& Scope, const FString* ScopeName) override;
	virtual void OnCpuScopeAnalysisEnd() override;

private:
	TMap<uint32, FSummarizeScope> Scopes;

	// Function invoked to publish the list of scopes.
	TFunction<void(const TArray<FSummarizeScope>&)> PublishFn;
};

FSummarizeCpuAnalyzer::FSummarizeCpuAnalyzer(TFunction<void(const TArray<FSummarizeScope>&)> InPublishFn)
	: PublishFn(MoveTemp(InPublishFn))
{
	OnCpuScopeDiscovered(FCpuScopeAnalyzer::CoroutineSpecId);
	OnCpuScopeDiscovered(FCpuScopeAnalyzer::CoroutineUnknownSpecId);
	OnCpuScopeName(FCpuScopeAnalyzer::CoroutineSpecId, TEXT("Coroutine"));
	OnCpuScopeName(FCpuScopeAnalyzer::CoroutineUnknownSpecId, TEXT("<unknown>"));
}

void FSummarizeCpuAnalyzer::OnCpuScopeDiscovered(uint32 ScopeId)
{
	if (!Scopes.Find(ScopeId))
	{
		Scopes.Add(ScopeId, FSummarizeScope());		
	}
}

void FSummarizeCpuAnalyzer::OnCpuScopeName(uint32 ScopeId, const FString& ScopeName)
{
	Scopes[ScopeId].Name = ScopeName;
}

void FSummarizeCpuAnalyzer::OnCpuScopeExit(const FScope& Scope, const FString* ScopeName)
{
	Scopes[Scope.ScopeId].AddDuration(Scope.EnterTimestamp, Scope.ExitTimestamp);
}

void FSummarizeCpuAnalyzer::OnCpuScopeAnalysisEnd()
{
	TArray<FSummarizeScope> LocalScopes;
	// Eliminates scopes that don't have a name. (On scope discovery, the array is expended to creates blank scopes that may never be filled).
	for (TMap<uint32, FSummarizeScope>::TIterator Iter = Scopes.CreateIterator(); Iter; ++Iter)
	{
		if(!Iter.Value().Name.IsEmpty())
		{
			LocalScopes.Add(Iter.Value());
		}
	}
	
	// Publish the scopes.
	PublishFn(LocalScopes);
}


/**
 * Summarizes matched CPU scopes, excluding time consumed by immediate children if any. The analyzer uses pattern matching to
 * selects the scopes of interest and detects parent/child relationship. Once a parent/child relationship is established in a
 * scope tree, the analyzer can substract the time consumed by its immediate children if any.
 *
 * Such analysis is often meaningful for reentrant/recursive scope. For example, the UE modules can be loaded recursively and
 * it is useful to know how much time a module used to load itself vs how much time it use to recursively load its dependent
 * modules. In that example, we need to know which scope timers are actually the recursion vs the other intermediate scopes.
 * So, pattern-matching scope names is used to deduce a relationship between scopes in a tree of scopes.
 *
 * For the LoadModule example described above, if the analyzer gets this scope tree as input:
 *
 * |-LoadModule_Module1----------------------------------------------------------|
 *    |- StartupModule -------------------------------------------------------|
 *      |-LoadModule_Module1Dep1------------|  |-LoadModule_Module1Dep2------|
 *        |-StartupModule-----------------|      |-StartupModule----------|
 *
 * It would turn it into the one below if the REGEX to match was "LoadModule_.*"
 * 
 * |-LoadModule_Module1----------------------------------------------------------|
 *      |-LoadModule_Module1Dep1------------|  |-LoadModule_Module1Dep2------|
 *
 * And it would compute the exclusive time required to load Module1 by substracting the time consumed to
 * load Module1Dep1 and Module1Dep2.
 *
 * @note If the matching expression was to match all scopes, the analyser would summarize all scopes,
 *       accounting for their exclusive time.
 */
class FCpuScopeHierarchyAnalyzer : public FCpuScopeAnalyzer
{
public:
	/**
	 * Constructs the analyzer
	 * @param InAnalyzerName    The name of this analyzer. Some output statistics will also be derived from this name.
	 * @param InMatchFn         Invoked by the analyzer to determine if a scope should be accounted by the analyzer. If it returns true, the scope is kept, otherwise, it is ignored.
	 * @param InPublishFn       Invoked at the end of the analysis to post process the scopes summaries procuded by this analysis, possibly eliminating or renaming them then to publish them.
	 *                          The first parameter is the summary of all matched scopes together while the array contains summary of scopes that matched the expression by grouped by exact name match.
	 * 
	 * @note This analyzer publishes summarized scope names with a suffix ".excl" as it computes exclusive time to  prevent name collisions with other analyzers. The summary of all scopes
	 *       suffixed with ".excl.all" and gets its base name from the analyzer name. The scopes in the array gets their names from the scope name themselves, but suffixed with .excl.
	 */
	FCpuScopeHierarchyAnalyzer(const FString& InAnalyzerName, TFunction<bool(const FString&)> InMatchFn, TFunction<void(const FSummarizeScope&, TArray<FSummarizeScope>&&)> InPublishFn);

	/**
	 * Runs analysis on a collection of scopes events. The scopes are expected to be from the same thread and form a 'tree', meaning
	 * it has the root scope events on that thread as well as all children below the root down to the leaves.
	 * @param ThreadId The thread on which the scopes were recorded.
	 * @param ScopeEvents The scopes events containing one root event along with its hierarchy.
	 * @param InScopeNameLookup Callback function to lookup scope names from scope ID.
	 */
	virtual void OnCpuScopeTree(uint32 ThreadId, const TArray<FCpuScopeAnalyzer::FScopeEvent>& ScopeEvents, const TFunction<const FString*(uint32 /*ScopeId*/)>& InScopeNameLookup) override;

	/**
	 * Invoked to notify that the trace session ended and that the analyzer can publish the statistics gathered.
	 * The analyzer calls the publishing function passed at construction time to publish the analysis results.
	 */
	virtual void OnCpuScopeAnalysisEnd() override;

private:
	// The function invoked to filter (by name) the scopes of interest. A scopes is kept if this function returns true.
	TFunction<bool(const FString&)> MatchesFn;

	// Aggregate all scopes matching the filter function, accounting for the parent/child relationship, so the duration stats will be from
	// the 'exclusive' time (itself - duration of immediate children) of the matched scope.
	FSummarizeScope MatchedScopesSummary;

	// Among the scopes matching the filter function, grouped by scope name summaries, accounting for the parent/child relationship. So the duration stats will be
	// from the 'exclusive' time (itself - duration of immediate children).
	TMap<FString, FSummarizeScope> ExactNameMatchScopesSummaries;

	// Invoked at the end of the analysis to publich scope summaries.
	TFunction<void(const FSummarizeScope&, TArray<FSummarizeScope>&&)> PublishFn;
};

FCpuScopeHierarchyAnalyzer::FCpuScopeHierarchyAnalyzer(const FString& InAnalyzerName, TFunction<bool(const FString&)> InMatchFn, TFunction<void(const FSummarizeScope&, TArray<FSummarizeScope>&&)> InPublishFn)
 : MatchesFn(MoveTemp(InMatchFn))
 , PublishFn(MoveTemp(InPublishFn))
{
	MatchedScopesSummary.Name = InAnalyzerName;
}

void FCpuScopeHierarchyAnalyzer::OnCpuScopeTree(uint32 ThreadId, const TArray<FCpuScopeAnalyzer::FScopeEvent>& ScopeEvents, const TFunction<const FString*(uint32 /*ScopeId*/)>& InScopeNameLookup)
{
	// Scope matching the pattern.
	struct FMatchScopeEnter
	{
		FMatchScopeEnter(uint32 InScopeId, double InTimestamp) : ScopeId(InScopeId), Timestamp(InTimestamp) {}
		uint32 ScopeId;
		double Timestamp;
		FTimespan ChildrenDuration;
	};

	TArray<FMatchScopeEnter> ScopeStack;

	// Replay and filter this scope hierarchy to only keep the ones matching the condition/regex. (See class documentation for a visual example)
	for (const FCpuScopeAnalyzer::FScopeEvent& ScopeEvent : ScopeEvents)
	{
		const FString* ScopeName = InScopeNameLookup(ScopeEvent.ScopeId);
		if (!ScopeName || !MatchesFn(*ScopeName))
		{
			continue;
		}

		if (ScopeEvent.ScopeEventType == FCpuScopeAnalyzer::EScopeEventType::Enter)
		{
			ScopeStack.Emplace(ScopeEvent.ScopeId, ScopeEvent.Timestamp);
		}
		else // Scope Exit
		{
			FMatchScopeEnter EnterScope = ScopeStack.Pop();
			double EnterTimestampSecs = EnterScope.Timestamp;
			uint32 ScopeId = EnterScope.ScopeId;

			// Total time consumed by this scope.
			FTimespan InclusiveDuration = FTimespan::FromSeconds(ScopeEvent.Timestamp - EnterTimestampSecs);

			// Total time consumed by this scope, excluding the time consumed by matched 'children' scopes.
			FTimespan ExclusiveDuration = InclusiveDuration - EnterScope.ChildrenDuration;

			if (ScopeStack.Num() > 0)
			{
				// Track how much time this 'child' consumed inside its parent.
				ScopeStack.Last().ChildrenDuration += InclusiveDuration;
			}

			// Aggregate this scope with all other scopes of the exact same name, excluding children duration, so that we have the 'self only' starts.
			FSummarizeScope& ExactNameScopeSummary = ExactNameMatchScopesSummaries.FindOrAdd(*ScopeName);
			ExactNameScopeSummary.Name = *ScopeName;
			ExactNameScopeSummary.AddDuration(EnterTimestampSecs, EnterTimestampSecs + ExclusiveDuration.GetTotalSeconds());

			// Aggregate this scope with all other scopes matching the pattern, but excluding the children time, so that we have the stats of 'self only'.
			MatchedScopesSummary.AddDuration(EnterTimestampSecs, EnterTimestampSecs + ExclusiveDuration.GetTotalSeconds());
		}
	}
}

void FCpuScopeHierarchyAnalyzer::OnCpuScopeAnalysisEnd()
{
	MatchedScopesSummary.Name += TEXT(".excl.all");

	TArray<FSummarizeScope> ScopeSummaries;
	for (TPair<FString, FSummarizeScope>& Pair : ExactNameMatchScopesSummaries)
	{
		Pair.Value.Name += TEXT(".excl");
		ScopeSummaries.Add(Pair.Value);
	}

	// Publish the scopes.
	PublishFn(MatchedScopesSummary, MoveTemp(ScopeSummaries));
}


//
// FSummarizeCountersAnalyzer - Tally Counters from counter set/increment events
//

class FSummarizeCountersAnalyzer
	: public FCountersAnalyzer
{
public:
	virtual void OnCounterName(const FCounterName& CounterName) override;
	virtual void OnCounterIntValue(const FCounterIntValue& NewValue) override;
	virtual void OnCounterFloatValue(const FCounterFloatValue& NewValue) override;

	struct FCounter
	{
		FString Name;
		ETraceCounterType Type;

		union
		{
			int64 IntValue;
			double FloatValue;
		};

		FCounter(FString InName, ETraceCounterType InType)
		{
			Name = InName;
			Type = InType;
			switch (Type)
			{
			case TraceCounterType_Int:
				IntValue = 0;
				break;

			case TraceCounterType_Float:
				FloatValue = 0.0;
				break;
			}
		}

		template<typename ValueType>
		void SetValue(ValueType InValue) = delete;

		template<>
		void SetValue(int64 InValue)
		{
			ensure(Type == TraceCounterType_Int);
			if (Type == TraceCounterType_Int)
			{
				IntValue = InValue;
			}
		}

		template<>
		void SetValue(double InValue)
		{
			ensure(Type == TraceCounterType_Float);
			if (Type == TraceCounterType_Float)
			{
				FloatValue = InValue;
			}
		}

		FString GetValue() const
		{
			switch (Type)
			{
			case TraceCounterType_Int:
				return FString::Printf(TEXT("%lld"), IntValue);

			case TraceCounterType_Float:
				return FString::Printf(TEXT("%f"), FloatValue);
			}

			ensure(false);
			return TEXT("");
		}
	};

	TMap<uint16, FCounter> Counters;
};

void FSummarizeCountersAnalyzer::OnCounterName(const FCounterName& CounterName)
{
	Counters.Add(CounterName.Id, FCounter(CounterName.Name, CounterName.Type));
}

void FSummarizeCountersAnalyzer::OnCounterIntValue(const FCounterIntValue& NewValue)
{
	FCounter* FoundCounter = Counters.Find(NewValue.Id);
	ensure(FoundCounter);
	if (FoundCounter)
	{
		FoundCounter->SetValue(NewValue.Value);
	}
}

void FSummarizeCountersAnalyzer::OnCounterFloatValue(const FCounterFloatValue& NewValue)
{
	FCounter* FoundCounter = Counters.Find(NewValue.Id);
	ensure(FoundCounter);
	if (FoundCounter)
	{
		FoundCounter->SetValue(NewValue.Value);
	}
}

//
// FSummarizeBookmarksAnalyzer - Tally Bookmarks from bookmark events
//

class FSummarizeBookmarksAnalyzer
	: public FBookmarksAnalyzer
{
	virtual void OnBookmarkSpecEvent(const FBookmarkSpecEvent& BookmarkSpecEvent) override;
	virtual void OnBookmarkEvent(const FBookmarkEvent& BookmarkEvent) override;

	FSummarizeBookmark* FindStartBookmarkForEndBookmark(const FString& Name);

public:
	struct FBookmarkSpec
	{
		uint64	Id;
		FString	FileName;
		int32	Line;
		FString	FormatString;
	};

	// Keyed by a unique memory address
	TMap<uint64, FBookmarkSpec> BookmarkSpecs;

	// Keyed by name
	TMap<FString, FSummarizeBookmark> Bookmarks;

	// Bookmarks named formed to scopes, see FindStartBookmarkForEndBookmark
	TMap<FString, FSummarizeScope> Scopes;
};

void FSummarizeBookmarksAnalyzer::OnBookmarkSpecEvent(const FBookmarkSpecEvent& BookmarkSpecEvent)
{
	BookmarkSpecs.Add(BookmarkSpecEvent.Id, {
		BookmarkSpecEvent.Id,
		BookmarkSpecEvent.FileName,
		BookmarkSpecEvent.Line,
		BookmarkSpecEvent.FormatString
		});
}

void FSummarizeBookmarksAnalyzer::OnBookmarkEvent(const FBookmarkEvent& BookmarkEvent)
{
	FBookmarkSpec* Spec = BookmarkSpecs.Find(BookmarkEvent.Id);
	if (Spec)
	{
		TCHAR FormattedString[65535];
		TraceServices::FormatString(FormattedString, sizeof(FormattedString) / sizeof(FormattedString[0]), *Spec->FormatString, BookmarkEvent.FormatArgs.GetData());

		FString Name (FormattedString);

		FSummarizeBookmark* FoundBookmark = Bookmarks.Find(Name);
		if (!FoundBookmark)
		{
			FoundBookmark = &Bookmarks.Add(Name, FSummarizeBookmark());
			FoundBookmark->Name = Name;
		}

		FoundBookmark->AddTimestamp(BookmarkEvent.Timestamp);

		FSummarizeBookmark* StartBookmark = FindStartBookmarkForEndBookmark(Name);
		if (StartBookmark)
		{
			FString ScopeName = FString(TEXT("Generated Scope for ")) + StartBookmark->Name;
			FSummarizeScope* FoundScope = Scopes.Find(ScopeName);
			if (!FoundScope)
			{
				FoundScope = &Scopes.Add(ScopeName, FSummarizeScope());
				FoundScope->Name = ScopeName;
			}

			FoundScope->AddDuration(StartBookmark->LastSeconds, BookmarkEvent.Timestamp);
		}
	}
}

FSummarizeBookmark* FSummarizeBookmarksAnalyzer::FindStartBookmarkForEndBookmark(const FString& Name)
{
	int32 Index = Name.Find(TEXT("Complete"));
	if (Index != -1)
	{
		FString StartName = Name;
		StartName.RemoveAt(Index, TCString<TCHAR>::Strlen(TEXT("Complete")));
		return Bookmarks.Find(StartName);
	}

	return nullptr;
}

/*
* Begin SummarizeTrace commandlet implementation
*/

// Defined here to prevent adding logs in the code above, which will likely be moved elsewhere.
DEFINE_LOG_CATEGORY_STATIC(LogSummarizeTrace, Log, All);

/*
* Helpers for the csv files
*/

static bool IsCsvSafeString(const FString& String)
{
	static struct DisallowedCharacter
	{
		const TCHAR Character;
		bool First;
	}
	DisallowedCharacters[] =
	{
		// breaks simple csv files
		{ TEXT('\n'), true },
		{ TEXT('\r'), true },
		{ TEXT(','), true },
	};

	// sanitize strings for a bog-simple csv file
	bool bDisallowed = false;
	int32 Index = 0;
	for (struct DisallowedCharacter& DisallowedCharacter : DisallowedCharacters)
	{
		if (String.FindChar(DisallowedCharacter.Character, Index))
		{
			if (DisallowedCharacter.First)
			{
				UE_LOG(LogSummarizeTrace, Display, TEXT("A string contains disallowed character '%c'. See log for full list."), DisallowedCharacter.Character);
				DisallowedCharacter.First = false;
			}

			UE_LOG(LogSummarizeTrace, Verbose, TEXT("String '%s' contains disallowed character '%c', skipping..."), *String, DisallowedCharacter.Character);
			bDisallowed = true;
		}

		if (bDisallowed)
		{
			break;
		}
	}

	return !bDisallowed;
}

struct StatisticDefinition
{
	StatisticDefinition()
	{}

	StatisticDefinition(const FString& InName, const FString& InStatistic,
		const FString& InTelemetryContext, const FString& InTelemetryDataPoint, const FString& InTelemetryUnit,
		const FString& InBaselineWarningThreshold, const FString& InBaselineErrorThreshold)
		: Name(InName)
		, Statistic(InStatistic)
		, TelemetryContext(InTelemetryContext)
		, TelemetryDataPoint(InTelemetryDataPoint)
		, TelemetryUnit(InTelemetryUnit)
		, BaselineWarningThreshold(InBaselineWarningThreshold)
		, BaselineErrorThreshold(InBaselineErrorThreshold)
	{}

	StatisticDefinition(const StatisticDefinition& InStatistic)
		: Name(InStatistic.Name)
		, Statistic(InStatistic.Statistic)
		, TelemetryContext(InStatistic.TelemetryContext)
		, TelemetryDataPoint(InStatistic.TelemetryDataPoint)
		, TelemetryUnit(InStatistic.TelemetryUnit)
		, BaselineWarningThreshold(InStatistic.BaselineWarningThreshold)
		, BaselineErrorThreshold(InStatistic.BaselineErrorThreshold)
	{}

	bool operator==(const StatisticDefinition& InStatistic) const
	{
		return Name == InStatistic.Name
			&& Statistic == InStatistic.Statistic
			&& TelemetryContext == InStatistic.TelemetryContext
			&& TelemetryDataPoint == InStatistic.TelemetryDataPoint
			&& TelemetryUnit == InStatistic.TelemetryUnit
			&& BaselineWarningThreshold == InStatistic.BaselineWarningThreshold
			&& BaselineErrorThreshold == InStatistic.BaselineErrorThreshold;
	}

	static bool LoadFromCSV(const FString& FilePath, TMultiMap<FString, StatisticDefinition>& NameToDefinitionMap, TSet<FString>& OutWildcardNames);

	FString Name;
	FString Statistic;
	FString TelemetryContext;
	FString TelemetryDataPoint;
	FString TelemetryUnit;
	FString BaselineWarningThreshold;
	FString BaselineErrorThreshold;
};

bool StatisticDefinition::LoadFromCSV(const FString& FilePath, TMultiMap<FString, StatisticDefinition>& NameToDefinitionMap, TSet<FString>& OutWildcardNames)
{
	TArray<FString> ParsedCSVFile;
	FFileHelper::LoadFileToStringArray(ParsedCSVFile, *FilePath);

	int NameColumn = -1;
	int StatisticColumn = -1;
	int TelemetryContextColumn = -1;
	int TelemetryDataPointColumn = -1;
	int TelemetryUnitColumn = -1;
	int BaselineWarningThresholdColumn = -1;
	int BaselineErrorThresholdColumn = -1;
	struct Column
	{
		const TCHAR* Name = nullptr;
		int* Index = nullptr;
	}
	Columns[] =
	{
		{ TEXT("Name"), &NameColumn },
		{ TEXT("Statistic"), &StatisticColumn },
		{ TEXT("TelemetryContext"), &TelemetryContextColumn },
		{ TEXT("TelemetryDataPoint"), &TelemetryDataPointColumn },
		{ TEXT("TelemetryUnit"), &TelemetryUnitColumn },
		{ TEXT("BaselineWarningThreshold"), &BaselineWarningThresholdColumn },
		{ TEXT("BaselineErrorThreshold"), &BaselineErrorThresholdColumn },
	};

	bool bValidColumns = true;
	for (int CSVIndex = 0; CSVIndex < ParsedCSVFile.Num() && bValidColumns; ++CSVIndex)
	{
		const FString& CSVEntry = ParsedCSVFile[CSVIndex];
		TArray<FString> Fields;
		UE::String::ParseTokens(CSVEntry.TrimStartAndEnd(), TEXT(','),
			[&Fields](FStringView Field)
			{
				Fields.Add(FString(Field));
			});

		if (CSVIndex == 0) // is this the header row?
		{
			for (struct Column& Column : Columns)
			{
				for (int FieldIndex = 0; FieldIndex < Fields.Num(); ++FieldIndex)
				{
					if (Fields[FieldIndex] == Column.Name)
					{
						(*Column.Index) = FieldIndex;
						break;
					}
				}

				if (*Column.Index == -1)
				{
					bValidColumns = false;
				}
			}
		}
		else // else it is a data row, pull each element from appropriate column
		{
			const FString& Name(Fields[NameColumn]);
			const FString& Statistic(Fields[StatisticColumn]);
			const FString& TelemetryContext(Fields[TelemetryContextColumn]);
			const FString& TelemetryDataPoint(Fields[TelemetryDataPointColumn]);
			const FString& TelemetryUnit(Fields[TelemetryUnitColumn]);
			const FString& BaselineWarningThreshold(Fields[BaselineWarningThresholdColumn]);
			const FString& BaselineErrorThreshold(Fields[BaselineErrorThresholdColumn]);

			if (Name.Contains("*") || Name.Contains("?")) // Wildcards.
			{
				OutWildcardNames.Add(Name);
			}
			NameToDefinitionMap.AddUnique(Name, StatisticDefinition(Name, Statistic, TelemetryContext, TelemetryDataPoint, TelemetryUnit, BaselineWarningThreshold, BaselineErrorThreshold));
		}
	}

	return bValidColumns;
}

/*
* Helper class for the telemetry csv file
*/

struct TelemetryDefinition
{
	TelemetryDefinition()
	{}

	TelemetryDefinition(const FString& InTestName, const FString& InContext, const FString& InDataPoint, const FString& InUnit,
		const FString& InMeasurement, const FString* InBaseline = nullptr)
		: TestName(InTestName)
		, Context(InContext)
		, DataPoint(InDataPoint)
		, Unit(InUnit)
		, Measurement(InMeasurement)
		, Baseline(InBaseline ? *InBaseline : FString ())
	{}

	TelemetryDefinition(const TelemetryDefinition& InStatistic)
		: TestName(InStatistic.TestName)
		, Context(InStatistic.Context)
		, DataPoint(InStatistic.DataPoint)
		, Unit(InStatistic.Unit)
		, Measurement(InStatistic.Measurement)
		, Baseline(InStatistic.Baseline)
	{}

	bool operator==(const TelemetryDefinition& InStatistic) const
	{
		return TestName == InStatistic.TestName
			&& Context == InStatistic.Context
			&& DataPoint == InStatistic.DataPoint
			&& Measurement == InStatistic.Measurement
			&& Baseline == InStatistic.Baseline
			&& Unit == InStatistic.Unit;
	}

	static bool LoadFromCSV(const FString& FilePath, TMap<TPair<FString,FString>, TelemetryDefinition>& ContextAndDataPointToDefinitionMap);
	static bool MeasurementWithinThreshold(const FString& Value, const FString& BaselineValue, const FString& Threshold);
	static FString SignFlipThreshold(const FString& Threshold);

	FString TestName;
	FString Context;
	FString DataPoint;
	FString Unit;
	FString Measurement;
	FString Baseline;
};

bool TelemetryDefinition::LoadFromCSV(const FString& FilePath, TMap<TPair<FString, FString>, TelemetryDefinition>& ContextAndDataPointToDefinitionMap)
{
	TArray<FString> ParsedCSVFile;
	FFileHelper::LoadFileToStringArray(ParsedCSVFile, *FilePath);

	int TestNameColumn = -1;
	int ContextColumn = -1;
	int DataPointColumn = -1;
	int UnitColumn = -1;
	int MeasurementColumn = -1;
	int BaselineColumn = -1;
	struct Column
	{
		const TCHAR* Name = nullptr;
		int* Index = nullptr;
		bool bRequired = true;
	}
	Columns[] =
	{
		{ TEXT("TestName"), &TestNameColumn },
		{ TEXT("Context"), &ContextColumn },
		{ TEXT("DataPoint"), &DataPointColumn },
		{ TEXT("Unit"), &UnitColumn },
		{ TEXT("Measurement"), &MeasurementColumn },
		{ TEXT("Baseline"), &BaselineColumn, false },
	};

	bool bValidColumns = true;
	for (int CSVIndex = 0; CSVIndex < ParsedCSVFile.Num() && bValidColumns; ++CSVIndex)
	{
		const FString& CSVEntry = ParsedCSVFile[CSVIndex];
		TArray<FString> Fields;
		UE::String::ParseTokens(CSVEntry.TrimStartAndEnd(), TEXT(','),
			[&Fields](FStringView Field)
			{
				Fields.Add(FString(Field));
			});

		if (CSVIndex == 0) // is this the header row?
		{
			for (struct Column& Column : Columns)
			{
				for (int FieldIndex = 0; FieldIndex < Fields.Num(); ++FieldIndex)
				{
					if (Fields[FieldIndex] == Column.Name)
					{
						(*Column.Index) = FieldIndex;
						break;
					}
				}

				if (*Column.Index == -1 && Column.bRequired)
				{
					bValidColumns = false;
				}
			}
		}
		else // else it is a data row, pull each element from appropriate column
		{
			const FString& TestName(Fields[TestNameColumn]);
			const FString& Context(Fields[ContextColumn]);
			const FString& DataPoint(Fields[DataPointColumn]);
			const FString& Unit(Fields[UnitColumn]);
			const FString& Measurement(Fields[MeasurementColumn]);

			FString Baseline;
			if (BaselineColumn != -1)
			{
				Baseline = Fields[BaselineColumn];
			}

			ContextAndDataPointToDefinitionMap.Add(TPair<FString, FString>(Context, DataPoint), TelemetryDefinition(TestName, Context, DataPoint, Unit, Measurement, &Baseline));
		}
	}

	return bValidColumns;
}

bool TelemetryDefinition::MeasurementWithinThreshold(const FString& MeasurementValue, const FString& BaselineValue, const FString& Threshold)
{
	if (Threshold.IsEmpty())
	{
		return true;
	}

	// detect threshold as delta percentage
	int32 PercentIndex = INDEX_NONE;
	if (Threshold.FindChar(TEXT('%'), PercentIndex))
	{
		FString ThresholdWithoutPercentSign = Threshold;
		ThresholdWithoutPercentSign.RemoveAt(PercentIndex);

		double Factor = 1.0 + (FCString::Atod(*ThresholdWithoutPercentSign) / 100.0);
		double RationalValue = FCString::Atod(*MeasurementValue);
		double RationalBaselineValue = FCString::Atod(*BaselineValue);
		if (Factor >= 1.0)
		{
			return RationalValue < (RationalBaselineValue * Factor);
		}
		else
		{
			return RationalValue > (RationalBaselineValue * Factor);
		}
	}
	else // threshold as delta cardinal value
	{
		// rational number, use float math
		if (Threshold.Contains(TEXT(".")))
		{
			double Delta = FCString::Atod(*Threshold);
			double RationalValue = FCString::Atod(*MeasurementValue);
			double RationalBaselineValue = FCString::Atod(*BaselineValue);
			if (Delta > 0.0)
			{
				return RationalValue <= (RationalBaselineValue + Delta);
			}
			else if (Delta < 0.0)
			{
				return RationalValue >= (RationalBaselineValue + Delta);
			}
			else
			{
				return fabs(RationalBaselineValue - RationalValue) < FLT_EPSILON;
			}
		}
		else // natural number, use int math
		{
			int64 Delta = FCString::Strtoi64(*Threshold, nullptr, 10);
			int64 NaturalValue = FCString::Strtoi64(*MeasurementValue, nullptr, 10);
			int64 NaturalBaselineValue = FCString::Strtoi64(*BaselineValue, nullptr, 10);
			if (Delta > 0)
			{
				return NaturalValue <= (NaturalBaselineValue + Delta);
			}
			else if (Delta < 0)
			{
				return NaturalValue >= (NaturalBaselineValue + Delta);
			}
			else
			{
				return NaturalValue == NaturalBaselineValue;
			}
		}
	}
}

FString TelemetryDefinition::SignFlipThreshold(const FString& Threshold)
{
	FString SignFlipped;

	if (Threshold.StartsWith(TEXT("-")))
	{
		SignFlipped = Threshold.RightChop(1);
	}
	else
	{
		SignFlipped = FString(TEXT("-")) + Threshold;
	}

	return SignFlipped;
}

/*
 * SummarizeTrace commandlet ingests a utrace file and summarizes the
 * cpu scope events within it, and summarizes each event to a csv. It
 * also can generate a telemetry file given statistics csv about what
 * events and what statistics you would like to track.
 */

USummarizeTraceCommandlet::USummarizeTraceCommandlet(const FObjectInitializer& ObjectInitializer)
	: Super(ObjectInitializer)
{
}

int32 USummarizeTraceCommandlet::Main(const FString& CmdLineParams)
{
	TArray<FString> Tokens;
	TArray<FString> Switches;
	TMap<FString, FString> ParamVals;
	UCommandlet::ParseCommandLine(*CmdLineParams, Tokens, Switches, ParamVals);

	// Display help
	if (Switches.Contains("help"))
	{
		UE_LOG(LogSummarizeTrace, Log, TEXT("SummarizeTrace"));
		UE_LOG(LogSummarizeTrace, Log, TEXT("This commandlet will summarize a utrace into something more easily ingestable by a reporting tool (csv)."));
		UE_LOG(LogSummarizeTrace, Log, TEXT("Options:"));
		UE_LOG(LogSummarizeTrace, Log, TEXT(" Required: -inputfile=<utrace path>   (The utrace you wish to process)"));
		UE_LOG(LogSummarizeTrace, Log, TEXT(" Optional: -testname=<string>         (Test name to use in telemetry csv)"));
		UE_LOG(LogSummarizeTrace, Log, TEXT(" Optional: -alltelemetry              (Dump all data to telemetry csv)"));
		return 0;
	}

	FString TraceFileName;
	if (FParse::Value(*CmdLineParams, TEXT("inputfile="), TraceFileName, true))
	{
		UE_LOG(LogSummarizeTrace, Display, TEXT("Loading trace from %s"), *TraceFileName);
	}
	else
	{
		UE_LOG(LogSummarizeTrace, Error, TEXT("You must specify a utrace file using -inputfile=<path>"));
		return 1;
	}

	bool bAllTelemetry = FParse::Param(*CmdLineParams, TEXT("alltelemetry"));

	// load the stats file to know which event name and statistic name to generate in the telemetry csv
	// the telemetry csv is ingested completely, so this just highlights specific data elements we want to track
	TMultiMap<FString, StatisticDefinition> NameToDefinitionMap;
	TSet<FString> CpuScopeNamesWithWildcards;
	if (!bAllTelemetry)
	{
		FString GlobalStatisticsFileName = FPaths::RootDir() / TEXT("Engine") / TEXT("Build") / TEXT("EditorPerfStats.csv");
		if (FPaths::FileExists(GlobalStatisticsFileName))
		{
			UE_LOG(LogSummarizeTrace, Display, TEXT("Loading global statistics from %s"), *GlobalStatisticsFileName);
			bool bCSVOk = StatisticDefinition::LoadFromCSV(GlobalStatisticsFileName, NameToDefinitionMap, CpuScopeNamesWithWildcards);
			check(bCSVOk);
		}
		FString ProjectStatisticsFileName = FPaths::ProjectDir() / TEXT("Build") / TEXT("EditorPerfStats.csv");
		if (FPaths::FileExists(ProjectStatisticsFileName))
		{
			UE_LOG(LogSummarizeTrace, Display, TEXT("Loading project statistics from %s"), *ProjectStatisticsFileName);
			bool bCSVOk = StatisticDefinition::LoadFromCSV(ProjectStatisticsFileName, NameToDefinitionMap, CpuScopeNamesWithWildcards);
		}
	}

	bool bFound;
	if (FPaths::FileExists(TraceFileName))
	{
		bFound = true;
	}
	else
	{
		bFound = false;
		TArray<FString> SearchPaths;
		SearchPaths.Add(FPaths::Combine(FPaths::EngineDir(), TEXT("Programs"), TEXT("UnrealInsights"), TEXT("Saved"), TEXT("TraceSessions")));
		SearchPaths.Add(FPaths::EngineDir());
		SearchPaths.Add(FPaths::ProjectDir());
		for (const FString& SearchPath : SearchPaths)
		{
			FString PossibleTraceFileName = FPaths::Combine(SearchPath, TraceFileName);
			if (FPaths::FileExists(PossibleTraceFileName))
			{
				TraceFileName = PossibleTraceFileName;
				bFound = true;
				break;
			}
		}
	}

	if (!bFound)
	{
		UE_LOG(LogSummarizeTrace, Error, TEXT("Trace file '%s' was not found"), *TraceFileName);
		return 1;
	}

	FFileDataStream DataStream;
	if (!DataStream.Open(*TraceFileName))
	{
		UE_LOG(LogSummarizeTrace, Error, TEXT("Unable to open trace file '%s' for read"), *TraceFileName);
		return 1;
	}

	// setup analysis context with analyzers
	UE::Trace::FAnalysisContext AnalysisContext;

	// List of summarized scopes.
	TArray<FSummarizeScope> CollectedScopeSummaries;

	// Analyze CPU scope timer individually.
	TSharedPtr<FSummarizeCpuAnalyzer> IndividualScopeAnalyzer = MakeShared<FSummarizeCpuAnalyzer>(
		[&CollectedScopeSummaries](const TArray<FSummarizeScope>& ScopeSummaries)
		{
			// Collect all individual scopes summary from this analyzer.
			CollectedScopeSummaries.Append(ScopeSummaries);
		});

	// Analyze 'LoadModule*' scope timer hierarchically to account individual load time only (substracting time consumed to load dependent module(s)).
	TSharedPtr<FCpuScopeHierarchyAnalyzer> HierarchicalScopeAnalyzer = MakeShared<FCpuScopeHierarchyAnalyzer>(
		TEXT("LoadModule"), // Analyzer Name.
		[](const FString& ScopeName)
		{
			return ScopeName.StartsWith("LoadModule_"); // When analyzing a tree of scopes, only keeps scope with name starting with 'LoadModule'.
		},
		[&CollectedScopeSummaries](const FSummarizeScope& AllModulesStats, TArray<FSummarizeScope>&& ModuleStats)
		{
			// Module should be loaded only once and the check below should be true but the load function can start loading X, process some dependencies which could
			// trigger loading X again within the first scope. The engine code gracefully handle this case and don't load twice, but we end up with two 'load x' scope.
			// Both scope times be added together, providing the correct sum for that module though.
			//check(AllModulesStats.Count == ModuleStats.Num())

			// Publish the total nb. of module loaded, total time to the modules, avg time per module, etc (all module load times exclude the time to load sub-modules)
			CollectedScopeSummaries.Add(AllModulesStats);

			// Sort the summaries descending.
			ModuleStats.Sort([](const FSummarizeScope& Lhs, const FSummarizeScope& Rhs) { return Lhs.TotalDurationSeconds >= Rhs.TotalDurationSeconds; });

			// Publish top N longuest load module. The ModuleStats are pre-sorted from the longest to the shorted timer.
			CollectedScopeSummaries.Append(ModuleStats.GetData(), 10);
		});

	FCpuScopeStreamProcessor CpuScopeStreamProcessor;
	CpuScopeStreamProcessor.AddCpuScopeAnalyzer(IndividualScopeAnalyzer);
	CpuScopeStreamProcessor.AddCpuScopeAnalyzer(HierarchicalScopeAnalyzer);
	AnalysisContext.AddAnalyzer(CpuScopeStreamProcessor);

	FSummarizeCountersAnalyzer CountersAnalyzer;
	AnalysisContext.AddAnalyzer(CountersAnalyzer);
	FSummarizeBookmarksAnalyzer BookmarksAnalyzer;
	AnalysisContext.AddAnalyzer(BookmarksAnalyzer);

	// kick processing on a thread
	UE::Trace::FAnalysisProcessor AnalysisProcessor = AnalysisContext.Process(DataStream);

	// sync on completion
	AnalysisProcessor.Wait();

	TSet<FSummarizeScope> DeduplicatedScopes;
	auto IngestScope = [](TSet<FSummarizeScope>& DeduplicatedScopes, const FSummarizeScope& Scope)
	{
		if (Scope.Name.IsEmpty())
		{
			return;
		}

		if (Scope.Count == 0)
		{
			return;
		}

		FSummarizeScope* FoundScope = DeduplicatedScopes.Find(Scope);
		if (FoundScope)
		{
			FoundScope->Merge(Scope);
		}
		else
		{
			DeduplicatedScopes.Add(Scope);
		}
	};
	for (const FSummarizeScope& Scope : CollectedScopeSummaries)
	{
		IngestScope(DeduplicatedScopes, Scope);
	}
	for (const TMap<FString, FSummarizeScope>::ElementType& ScopeItem : BookmarksAnalyzer.Scopes)
	{
		IngestScope(DeduplicatedScopes, ScopeItem.Value);
	}

	UE_LOG(LogSummarizeTrace, Display, TEXT("Sorting %d events by total time accumulated..."), DeduplicatedScopes.Num());
	TArray<FSummarizeScope> SortedScopes;
	for (const FSummarizeScope& Scope : DeduplicatedScopes)
	{
		SortedScopes.Add(Scope);
	}
	SortedScopes.Sort();

	// csv is UTF-8, so encode every string we print
	auto WriteUTF8 = [](IFileHandle* Handle, const FString& String)
	{
		const auto& UTF8String = StringCast<ANSICHAR>(*String);
		Handle->Write(reinterpret_cast<const uint8*>(UTF8String.Get()), UTF8String.Length());
	};

	// some locals to help with all the derived files we are about to generate
	const FString TracePath = FPaths::GetPath(TraceFileName);
	const FString TraceFileBasename = FPaths::GetBaseFilename(TraceFileName);

	// generate a summary csv files, always
	FString CsvFileName = TraceFileBasename + TEXT("Scopes");
	CsvFileName = FPaths::Combine(TracePath, FPaths::SetExtension(CsvFileName, "csv"));
	UE_LOG(LogSummarizeTrace, Display, TEXT("Writing %s..."), *CsvFileName);
	IFileHandle* CsvHandle = FPlatformFileManager::Get().GetPlatformFile().OpenWrite(*CsvFileName);
	if (!CsvHandle)
	{
		UE_LOG(LogSummarizeTrace, Error, TEXT("Unable to open csv '%s' for write"), *CsvFileName);
		return 1;
	}
	else
	{
		// no newline, see row printfs
		WriteUTF8(CsvHandle, FString::Printf(TEXT("Name,Count,TotalDurationSeconds,FirstStartSeconds,FirstFinishSeconds,FirstDurationSeconds,LastStartSeconds,LastFinishSeconds,LastDurationSeconds,MinDurationSeconds,MaxDurationSeconds,MeanDurationSeconds,DeviationDurationSeconds,")));
		for (const FSummarizeScope& Scope : SortedScopes)
		{
			if (!IsCsvSafeString(Scope.Name))
			{
				continue;
			}

			// note newline is at the front of every data line to prevent final extraneous newline, per customary for csv
			WriteUTF8(CsvHandle, FString::Printf(TEXT("\n%s,%llu,%f,%f,%f,%f,%f,%f,%f,%f,%f,%f,%f,"), *Scope.Name, Scope.Count, Scope.TotalDurationSeconds, Scope.FirstStartSeconds, Scope.FirstFinishSeconds, Scope.FirstDurationSeconds, Scope.FirstStartSeconds, Scope.FirstFinishSeconds, Scope.FirstDurationSeconds, Scope.MinDurationSeconds, Scope.MaxDurationSeconds, Scope.MeanDurationSeconds, Scope.GetDeviationDurationSeconds()));
		}
		CsvHandle->Flush();
		delete CsvHandle;
		CsvHandle = nullptr;
	}

	CsvFileName = TraceFileBasename + TEXT("Counters");
	CsvFileName = FPaths::Combine(TracePath, FPaths::SetExtension(CsvFileName, "csv"));
	UE_LOG(LogSummarizeTrace, Display, TEXT("Writing %s..."), *CsvFileName);
	CsvHandle = FPlatformFileManager::Get().GetPlatformFile().OpenWrite(*CsvFileName);
	if (!CsvHandle)
	{
		UE_LOG(LogSummarizeTrace, Error, TEXT("Unable to open csv '%s' for write"), *CsvFileName);
		return 1;
	}
	else
	{
		// no newline, see row printfs
		WriteUTF8(CsvHandle, FString::Printf(TEXT("Name,Value,")));
		for (const TMap<uint16, FSummarizeCountersAnalyzer::FCounter>::ElementType& Counter : CountersAnalyzer.Counters)
		{
			if (!IsCsvSafeString(Counter.Value.Name))
			{
				continue;
			}

			// note newline is at the front of every data line to prevent final extraneous newline, per customary for csv
			WriteUTF8(CsvHandle, FString::Printf(TEXT("\n%s,%s,"), *Counter.Value.Name, *Counter.Value.GetValue()));
		}
		CsvHandle->Flush();
		delete CsvHandle;
		CsvHandle = nullptr;
	}

	CsvFileName = TraceFileBasename + TEXT("Bookmarks");
	CsvFileName = FPaths::Combine(TracePath, FPaths::SetExtension(CsvFileName, "csv"));
	UE_LOG(LogSummarizeTrace, Display, TEXT("Writing %s..."), *CsvFileName);
	CsvHandle = FPlatformFileManager::Get().GetPlatformFile().OpenWrite(*CsvFileName);
	if (!CsvHandle)
	{
		UE_LOG(LogSummarizeTrace, Error, TEXT("Unable to open csv '%s' for write"), *CsvFileName);
		return 1;
	}
	else
	{
		// no newline, see row printfs
		WriteUTF8(CsvHandle, FString::Printf(TEXT("Name,Count,FirstSeconds,LastSeconds,")));
		for (const TMap<FString, FSummarizeBookmark>::ElementType& Bookmark : BookmarksAnalyzer.Bookmarks)
		{
			if (!IsCsvSafeString(Bookmark.Value.Name))
			{
				continue;
			}

			// note newline is at the front of every data line to prevent final extraneous newline, per customary for csv
			WriteUTF8(CsvHandle, FString::Printf(TEXT("\n%s,%d,%f,%f,"), *Bookmark.Value.Name, Bookmark.Value.Count, Bookmark.Value.FirstSeconds, Bookmark.Value.LastSeconds));
		}
		CsvHandle->Flush();
		delete CsvHandle;
		CsvHandle = nullptr;
	}

	FString TelemetryCsvFileName = TraceFileBasename + TEXT("Telemetry");
	TelemetryCsvFileName = FPaths::Combine(TracePath, FPaths::SetExtension(TelemetryCsvFileName, "csv"));

	// override the test name
	FString TestName = TraceFileBasename;
	FParse::Value(*CmdLineParams, TEXT("testname="), TestName, true);

	TArray<TelemetryDefinition> TelemetryData;
	TSet<FString> ResolvedStatistics;
	{
		TArray<StatisticDefinition> Statistics;

		// resolve scopes to telemetry
		for (const FSummarizeScope& Scope : SortedScopes)
		{
			if (!IsCsvSafeString(Scope.Name))
			{
				continue;
			}

			if (bAllTelemetry)
			{
				TelemetryData.Add(TelemetryDefinition(TestName, Scope.Name, TEXT("Count"), Scope.GetValue(TEXT("Count")), TEXT("Count")));
				TelemetryData.Add(TelemetryDefinition(TestName, Scope.Name, TEXT("TotalDurationSeconds"), Scope.GetValue(TEXT("TotalDurationSeconds")), TEXT("Seconds")));
				TelemetryData.Add(TelemetryDefinition(TestName, Scope.Name, TEXT("MinDurationSeconds"), Scope.GetValue(TEXT("MinDurationSeconds")), TEXT("Seconds")));
				TelemetryData.Add(TelemetryDefinition(TestName, Scope.Name, TEXT("MaxDurationSeconds"), Scope.GetValue(TEXT("MaxDurationSeconds")), TEXT("Seconds")));
				TelemetryData.Add(TelemetryDefinition(TestName, Scope.Name, TEXT("MeanDurationSeconds"), Scope.GetValue(TEXT("MeanDurationSeconds")), TEXT("Seconds")));
				TelemetryData.Add(TelemetryDefinition(TestName, Scope.Name, TEXT("DeviationDurationSeconds"), Scope.GetValue(TEXT("DeviationDurationSeconds")), TEXT("Seconds")));
			}
			else
			{
				// Is that scope summary desired in the output, using an exact name match?
				NameToDefinitionMap.MultiFind(Scope.Name, Statistics, true);
				for (const StatisticDefinition& Statistic : Statistics)
				{
					TelemetryData.Add(TelemetryDefinition(TestName, Statistic.TelemetryContext, Statistic.TelemetryDataPoint, Statistic.TelemetryUnit, Scope.GetValue(Statistic.Statistic)));
					ResolvedStatistics.Add(Statistic.Name);
				}
				Statistics.Reset();

				// If the configuration file contains scope names with wildcard characters * and ?
				for (const FString& Pattern : CpuScopeNamesWithWildcards)
				{
					// Check if the current scope names matches the pattern.
					if (Scope.Name.MatchesWildcard(Pattern))
					{
						// Find the statistic definition for this pattern.
						NameToDefinitionMap.MultiFind(Pattern, Statistics, true);
						for (const StatisticDefinition& Statistic : Statistics)
						{
							// Use the scope name as data point. Normally, the data point is configured in the .csv as a 1:1 match (1 scopeName=> 1 DataPoint) but in this
							// case, it is a one to many relationship (1 pattern => * matches).
							const FString& DataPoint = Scope.Name;
							TelemetryData.Add(TelemetryDefinition(TestName, Statistic.TelemetryContext, DataPoint, Statistic.TelemetryUnit, Scope.GetValue(Statistic.Statistic)));
							ResolvedStatistics.Add(Statistic.Name);
						}
						Statistics.Reset();
					}
				}
			}
		}

		// resolve counters to telemetry
		for (const TMap<uint16, FSummarizeCountersAnalyzer::FCounter>::ElementType& Counter : CountersAnalyzer.Counters)
		{
			if (!IsCsvSafeString(Counter.Value.Name))
			{
				continue;
			}

			if (bAllTelemetry)
			{
				TelemetryData.Add(TelemetryDefinition(TestName, Counter.Value.Name, TEXT("Count"), Counter.Value.GetValue(), TEXT("Count")));
			}
			else
			{
				NameToDefinitionMap.MultiFind(Counter.Value.Name, Statistics, true);
				ensure(Statistics.Num() <= 1); // there should only be one, the counter value
				for (const StatisticDefinition& Statistic : Statistics)
				{
					TelemetryData.Add(TelemetryDefinition(TestName, Statistic.TelemetryContext, Statistic.TelemetryDataPoint, Statistic.TelemetryUnit, Counter.Value.GetValue()));
					ResolvedStatistics.Add(Statistic.Name);
				}
				Statistics.Reset();			
			}
		}

		// resolve bookmarks to telemetry
		for (const TMap<FString, FSummarizeBookmark>::ElementType& Bookmark : BookmarksAnalyzer.Bookmarks)
		{
			if (!IsCsvSafeString(Bookmark.Value.Name))
			{
				continue;
			}

			if (bAllTelemetry)
			{
				TelemetryData.Add(TelemetryDefinition(TestName, Bookmark.Value.Name, TEXT("Count"), Bookmark.Value.GetValue(TEXT("Count")), TEXT("Count")));
				TelemetryData.Add(TelemetryDefinition(TestName, Bookmark.Value.Name, TEXT("TotalDurationSeconds"), Bookmark.Value.GetValue(TEXT("TotalDurationSeconds")), TEXT("Seconds")));
				TelemetryData.Add(TelemetryDefinition(TestName, Bookmark.Value.Name, TEXT("MinDurationSeconds"), Bookmark.Value.GetValue(TEXT("MinDurationSeconds")), TEXT("Seconds")));
				TelemetryData.Add(TelemetryDefinition(TestName, Bookmark.Value.Name, TEXT("MaxDurationSeconds"), Bookmark.Value.GetValue(TEXT("MaxDurationSeconds")), TEXT("Seconds")));
				TelemetryData.Add(TelemetryDefinition(TestName, Bookmark.Value.Name, TEXT("MeanDurationSeconds"), Bookmark.Value.GetValue(TEXT("MeanDurationSeconds")), TEXT("Seconds")));
				TelemetryData.Add(TelemetryDefinition(TestName, Bookmark.Value.Name, TEXT("DeviationDurationSeconds"), Bookmark.Value.GetValue(TEXT("DeviationDurationSeconds")), TEXT("Seconds")));
			}
			else
			{
				NameToDefinitionMap.MultiFind(Bookmark.Value.Name, Statistics, true);
				ensure(Statistics.Num() <= 1); // there should only be one, the bookmark itself
				for (const StatisticDefinition& Statistic : Statistics)
				{
					TelemetryData.Add(TelemetryDefinition(TestName, Statistic.TelemetryContext, Statistic.TelemetryDataPoint, Statistic.TelemetryUnit, Bookmark.Value.GetValue(Statistic.Statistic)));
					ResolvedStatistics.Add(Statistic.Name);
				}
				Statistics.Reset();			
			}
		}
	}

	if (!bAllTelemetry)
	{
		// compare vs. baseline telemetry file, if it exists
		// note this does assume that the tracefile basename is directly comparable to a file in the baseline folder
		FString BaselineTelemetryCsvFilePath = FPaths::Combine(FPaths::EngineDir(), TEXT("Build"), TEXT("Baseline"), FPaths::SetExtension(TraceFileBasename + TEXT("Telemetry"), "csv"));
		if (FParse::Param(*CmdLineParams, TEXT("skipbaseline")))
		{
			BaselineTelemetryCsvFilePath.Empty();
		}
		if (FPaths::FileExists(BaselineTelemetryCsvFilePath))
		{
			UE_LOG(LogSummarizeTrace, Display, TEXT("Comparing telemetry to baseline telemetry %s..."), *TelemetryCsvFileName);

			// each context (scope name or coutner name) and data point (statistic name) pair form a key, an item to check
			TMap<TPair<FString, FString>, TelemetryDefinition> ContextAndDataPointToDefinitionMap;
			bool bCSVOk = TelemetryDefinition::LoadFromCSV(*BaselineTelemetryCsvFilePath, ContextAndDataPointToDefinitionMap);
			check(bCSVOk);

			// for every telemetry item we wrote for this trace...
			for (TelemetryDefinition& Telemetry : TelemetryData)
			{
				// the threshold is defined along with the original statistic map
				const StatisticDefinition* RelatedStatistic = nullptr;

				// find the statistic definition
				TArray<StatisticDefinition> Statistics;
				NameToDefinitionMap.MultiFind(Telemetry.Context, Statistics, true);
				for (const StatisticDefinition& Statistic : Statistics)
				{
					// the find will match on name, here we just need to find the right statistic for that named item
					if (Statistic.Statistic == Telemetry.DataPoint)
					{
						// we found it!
						RelatedStatistic = &Statistic;
						break;
					}
				}

				// do we still have the statistic definition in our current stats file? (if we don't that's fine, we don't care about it anymore)
				if (RelatedStatistic)
				{
					// find the corresponding keyed telemetry item in the baseline telemetry file...
					TelemetryDefinition* BaselineTelemetry = ContextAndDataPointToDefinitionMap.Find(TPair<FString, FString>(Telemetry.Context, Telemetry.DataPoint));
					if (BaselineTelemetry)
					{
						Telemetry.Baseline = BaselineTelemetry->Measurement;

						// let's only report on statistics that have an assigned threshold, to keep things concise
						if (!RelatedStatistic->BaselineWarningThreshold.IsEmpty() || !RelatedStatistic->BaselineErrorThreshold.IsEmpty())
						{
							// verify that this telemetry measurement is within the allowed threshold as defined in the current stats file
							if (TelemetryDefinition::MeasurementWithinThreshold(Telemetry.Measurement, BaselineTelemetry->Measurement, RelatedStatistic->BaselineWarningThreshold))
							{
								FString SignFlippedWarningThreshold = TelemetryDefinition::SignFlipThreshold(RelatedStatistic->BaselineWarningThreshold);

								// check if it's beyond the threshold the other way and needs lowering in the stats csv
								if (!TelemetryDefinition::MeasurementWithinThreshold(Telemetry.Measurement, BaselineTelemetry->Measurement, SignFlippedWarningThreshold))
								{
									FString BaselineRelPath = FPaths::ConvertRelativePathToFull(BaselineTelemetryCsvFilePath);
									FPaths::MakePathRelativeTo(BaselineRelPath, *FPaths::RootDir());

									UE_LOG(LogSummarizeTrace, Warning, TEXT("Telemetry %s,%s,%s,%s significantly within baseline value %s using warning threshold %s. Please submit a new baseline to %s or adjust the threshold in the statistics file."),
										*Telemetry.TestName, *Telemetry.Context, *Telemetry.DataPoint, *Telemetry.Measurement,
										*BaselineTelemetry->Measurement, *RelatedStatistic->BaselineWarningThreshold,
										*BaselineRelPath);
								}
								else // it's within tolerance, just report that it's ok
								{
									UE_LOG(LogSummarizeTrace, Verbose, TEXT("Telemetry %s,%s,%s,%s within baseline value %s using warning threshold %s"),
										*Telemetry.TestName, *Telemetry.Context, *Telemetry.DataPoint, *Telemetry.Measurement,
										*BaselineTelemetry->Measurement, *RelatedStatistic->BaselineWarningThreshold);
								}
							}
							else
							{
								// it's outside warning threshold, check if it's inside the error threshold to just issue a warning
								if (TelemetryDefinition::MeasurementWithinThreshold(Telemetry.Measurement, BaselineTelemetry->Measurement, RelatedStatistic->BaselineErrorThreshold))
								{
									UE_LOG(LogSummarizeTrace, Warning, TEXT("Telemetry %s,%s,%s,%s beyond baseline value %s using warning threshold %s. This could be a performance regression!"),
										*Telemetry.TestName, *Telemetry.Context, *Telemetry.DataPoint, *Telemetry.Measurement,
										*BaselineTelemetry->Measurement, *RelatedStatistic->BaselineWarningThreshold);
								}
								else // it's outside the error threshold, hard error
								{
									UE_LOG(LogSummarizeTrace, Error, TEXT("Telemetry %s,%s,%s,%s beyond baseline value %s using error threshold %s. This could be a performance regression!"),
										*Telemetry.TestName, *Telemetry.Context, *Telemetry.DataPoint, *Telemetry.Measurement,
										*BaselineTelemetry->Measurement, *RelatedStatistic->BaselineErrorThreshold);
								}
							}
						}
					}
					else
					{
						UE_LOG(LogSummarizeTrace, Display, TEXT("Telemetry for %s,%s has no baseline measurement, skipping..."), *Telemetry.Context, *Telemetry.DataPoint);
					}
				}
			}
		}

		// check for references to statistics desired for telemetry that are unresolved
		for (const FString& StatisticName : ResolvedStatistics)
		{
			NameToDefinitionMap.Remove(StatisticName);
		}

		for (const TPair<FString, StatisticDefinition>& Statistic : NameToDefinitionMap)
		{
			UE_LOG(LogSummarizeTrace, Error, TEXT("Failed to find resolve telemety data for statistic \"%s\""), *Statistic.Value.Name);
		}

		if (!NameToDefinitionMap.IsEmpty())
		{
			return 1;
		}
	}

	UE_LOG(LogSummarizeTrace, Display, TEXT("Writing telemetry to %s..."), *TelemetryCsvFileName);
	IFileHandle* TelemetryCsvHandle = FPlatformFileManager::Get().GetPlatformFile().OpenWrite(*TelemetryCsvFileName);
	if (TelemetryCsvHandle)
	{
		// no newline, see row printfs
		WriteUTF8(TelemetryCsvHandle, FString::Printf(TEXT("TestName,Context,DataPoint,Unit,Measurement,Baseline,")));
		for (const TelemetryDefinition& Telemetry : TelemetryData)
		{
			// note newline is at the front of every data line to prevent final extraneous newline, per customary for csv
			WriteUTF8(TelemetryCsvHandle, FString::Printf(TEXT("\n%s,%s,%s,%s,%s,%s,"), *Telemetry.TestName, *Telemetry.Context, *Telemetry.DataPoint, *Telemetry.Unit, *Telemetry.Measurement, *Telemetry.Baseline));
		}

		TelemetryCsvHandle->Flush();
		delete TelemetryCsvHandle;
		TelemetryCsvHandle = nullptr;
	}
	else
	{
		UE_LOG(LogSummarizeTrace, Error, TEXT("Unable to open telemetry csv '%s' for write"), *TelemetryCsvFileName);
		return 1;
	}

	return 0;
}