Stats/Profiler - Refactored reading from a stats file (move memorydumpcommand to the profiler, all memory profiler funcionality move to the profiler module, can be accessed from the IProfilerModule, still wip)

[CL 2619747 by Jaroslaw Surowiec in Main branch]

Engine/Source/Developer/Profiler/Private/ProfilerModule.cpp

@@ -24,8 +24,7 @@ public:
 
 	void StatsMemoryDumpCommand( const TCHAR* Filename );
 
-	FRawStatsMemoryProfiler* OpenRawStatForMemoryProfiling( const TCHAR* Filename );
-
+	FRawStatsMemoryProfiler* OpenRawStatsForMemoryProfiling( const TCHAR* Filename );
 
 protected:
 	/** Shutdowns the profiler manager. */
@@ -70,16 +69,68 @@ void FProfilerModule::Shutdown( TSharedRef<SDockTab> TabBeingClosed )
 
 void FProfilerModule::StatsMemoryDumpCommand( const TCHAR* Filename )
 {
+	//FRawStatsMemoryProfiler* Instance = FCreateStatsReader<FRawStatsMemoryProfiler>::ForRawStats( Filename );
+	TUniquePtr<FRawStatsMemoryProfiler> Instance( FCreateStatsReader<FRawStatsMemoryProfiler>::ForRawStats( Filename ) );
+	if (Instance)
+	{
+		Instance->ReadAndProcessAsynchronously();
 
+		while (Instance->IsBusy())
+		{
+			FPlatformProcess::Sleep( 1.0f );
+
+			UE_LOG( LogStats, Log, TEXT( "Async: Stage: %s / %3i%%" ), *Instance->GetProcessingStageAsString(), Instance->GetStageProgress() );
+		}
+		//NewReader->RequestStop();
+
+		//Instance->
+
+		// Frame-240 Frame-120 Frame-060
+		TMap<FString, FCombinedAllocationInfo> FrameBegin_Exit;
+		Instance->CompareSnapshots_FString( TEXT( "BeginSnapshot" ), TEXT( "EngineLoop.Exit" ), FrameBegin_Exit );
+		Instance->DumpScopedAllocations( TEXT( "Begin_Exit" ), FrameBegin_Exit );
+
+#if	UE_BUILD_DEBUG
+		TMap<FString, FCombinedAllocationInfo> Frame060_120;
+		Instance->CompareSnapshots_FString( TEXT( "Frame-060" ), TEXT( "Frame-120" ), Frame060_120 );
+		Instance->DumpScopedAllocations( TEXT( "Frame060_120" ), Frame060_120 );
+
+		TMap<FString, FCombinedAllocationInfo> Frame060_240;
+		Instance->CompareSnapshots_FString( TEXT( "Frame-060" ), TEXT( "Frame-240" ), Frame060_240 );
+		Instance->DumpScopedAllocations( TEXT( "Frame060_240" ), Frame060_240 );
+
+		// Generate scoped tree view.
+		{
+			TMap<FName, FCombinedAllocationInfo> FrameBegin_Exit_FName;
+			Instance->CompareSnapshots_FName( TEXT( "BeginSnapshot" ), TEXT( "EngineLoop.Exit" ), FrameBegin_Exit_FName );
+
+			FNodeAllocationInfo Root;
+			Root.EncodedCallstack = TEXT( "ThreadRoot" );
+			Root.HumanReadableCallstack = TEXT( "ThreadRoot" );
+			Instance->GenerateScopedTreeAllocations( FrameBegin_Exit_FName, Root );
+		}
+
+
+		{
+			TMap<FName, FCombinedAllocationInfo> Frame060_240_FName;
+			Instance->CompareSnapshots_FName( TEXT( "Frame-060" ), TEXT( "Frame-240" ), Frame060_240_FName );
+
+			FNodeAllocationInfo Root;
+			Root.EncodedCallstack = TEXT( "ThreadRoot" );
+			Root.HumanReadableCallstack = TEXT( "ThreadRoot" );
+			Instance->GenerateScopedTreeAllocations( Frame060_240_FName, Root );
+		}
+#endif // UE_BUILD_DEBUG
+	}
 }
 
 /*-----------------------------------------------------------------------------
 	FRawStatsMemoryProfiler
 -----------------------------------------------------------------------------*/
 
-FRawStatsMemoryProfiler* FProfilerModule::OpenRawStatForMemoryProfiling( const TCHAR* Filename )
+FRawStatsMemoryProfiler* FProfilerModule::OpenRawStatsForMemoryProfiling( const TCHAR* Filename )
 {
-	FRawStatsMemoryProfiler* Instance = new FRawStatsMemoryProfiler;
+	FRawStatsMemoryProfiler* Instance = 0;// new FRawStatsMemoryProfiler;
 
 	return Instance;
 }

Engine/Source/Developer/Profiler/Private/ProfilerRawStats.h

@@ -211,8 +211,131 @@ protected:
 -----------------------------------------------------------------------------*/
 
 /** Class for managing the raw stats in the context of memory profiling. */
-class FRawStatsMemoryProfiler
+class FRawStatsMemoryProfiler : public FStatsReadFile
 {
+	friend struct FCreateStatsReader<FRawStatsMemoryProfiler>;
+	typedef FStatsReadFile Super;
 
+protected:
+
+	/** Initialization constructor. */
+	FRawStatsMemoryProfiler( const TCHAR* InFilename );
+
+	/** Called before started processing combined history. */
+	virtual void PreProcessStats() override;
+
+	/** Called after finished processing combined history. */
+	virtual void PostProcessStats() override;
+
+// 	/** Processes special message for advancing the stats frame from the game thread. */
+// 	virtual void ProcessAdvanceFrameEventGameThreadOperation( const FStatMessage& Message, const FStackState& Stack ) override
+// 	{}
+// 
+// 	/** Processes special message for advancing the stats frame from the render thread. */
+// 	virtual void ProcessAdvanceFrameEventRenderThreadOperation( const FStatMessage& Message, const FStackState& Stack ) override
+// 	{}
+// 
+// 	/** ProcessesIndicates begin of the cycle scope. */
+// 	virtual void ProcessCycleScopeStartOperation( const FStatMessage& Message, const FStackState& Stack ) override
+// 	{}
+// 
+// 	/** Indicates end of the cycle scope. */
+// 	virtual void ProcessCycleScopeEndOperation( const FStatMessage& Message, const FStackState& Stack ) override
+// 	{}
+// 
+	/** Processes special message marker used determine that we encountered a special data in the stat file. */
+	virtual void ProcessSpecialMessageMarkerOperation( const FStatMessage& Message, const FStackState& StackState ) override;
+// 
+// 	/** Processes set operation. */
+// 	virtual void ProcessSetOperation( const FStatMessage& Message, const FStackState& Stack ) override
+// 	{}
+// 
+// 	/** Processes clear operation. */
+// 	virtual void ProcessClearOperation( const FStatMessage& Message, const FStackState& Stack ) override
+// 	{}
+// 
+// 	/** Processes add operation. */
+// 	virtual void ProcessAddOperation( const FStatMessage& Message, const FStackState& Stack ) override
+// 	{}
+// 
+// 	/** Processes subtract operation. */
+// 	virtual void ProcessSubtractOperation( const FStatMessage& Message, const FStackState& Stack ) override
+// 	{}
+
+	/** Processes memory operation. @see EMemoryOperation. */
+	virtual void ProcessMemoryOperation( EMemoryOperation MemOp, uint64 Ptr, uint64 NewPtr, int64 Size, uint32 SequenceTag, const FStackState& StackState ) override;
+
+	/**
+	 * Sorts sequence allocations.
+	 *
+	 * @return true, if finished, false otherwise like stopped.
+	 */
+	bool SortSequenceAllocations();
+
+public:
+	/** Generates a basic memory usage report and prints it to the log. */
+	void GenerateMemoryUsageReport( const TMap<uint64, FAllocationInfo>& AllocationMap, const TSet<FName>& UObjectRawNames );
+
+	/** Generates scoped allocation statistics. */
+	void ProcessAndDumpScopedAllocations( const TMap<uint64, FAllocationInfo>& AllocationMap );
+
+	/** Generates UObject allocation statistics. */
+	void ProcessAndDumpUObjectAllocations( const TMap<uint64, FAllocationInfo>& AllocationMap, const TSet<FName>& UObjectRawNames );
+
+	void DumpScopedAllocations( const TCHAR* Name, const TMap<FString, FCombinedAllocationInfo>& CombinedAllocations );
+
+	/** Generates callstack based allocation map. */
+	void GenerateScopedAllocations( const TMap<uint64, FAllocationInfo>& AllocationMap, TMap<FName, FCombinedAllocationInfo>& out_CombinedAllocations, uint64& TotalAllocatedMemory, uint64& NumAllocations );
+
+	void GenerateScopedTreeAllocations( const TMap<FName, FCombinedAllocationInfo>& ScopedAllocations, FNodeAllocationInfo& out_Root );
+
+	/** Prepares data for a snapshot. */
+	void PrepareSnapshot( const FName SnapshotName, const TMap<uint64, FAllocationInfo>& AllocationMap );
+
+	void CompareSnapshots_FName( const FName BeginSnaphotName, const FName EndSnaphotName, TMap<FName, FCombinedAllocationInfo>& out_Result );
+
+	void CompareSnapshots_FString( const FName BeginSnaphotName, const FName EndSnaphotName, TMap<FString, FCombinedAllocationInfo>& out_Result );
+
+
+	/**
+	* @return Platform's name based on the loaded ue4statsraw file
+	*/
+	const FString& GetPlatformName() const
+	{
+		return Header.PlatformName;
+	}
+
+protected:
+	/**
+	 *	 All allocation ordered by the sequence tag.
+	 *	 There is an assumption that the sequence tag will not turn-around.
+	 */
+	TArray<FAllocationInfo> SequenceAllocationArray;
+
+	/** Allocation map. */
+	//TMap<uint64, FAllocationInfo>& AllocationMap;
+
+	/** The sequence tag mapping to the named markers. */
+	TArray<TPair<uint32, FName>> Snapshots;
+
+	/** Unique snapshot names. */
+	TSet<FName> SnapshotNames;
+
+	/** The sequence tag mapping to the named markers that need to processed. */
+	TArray<TPair<uint32, FName>> SnapshotsToBeProcessed;
+
+	/** Snapshots with allocation map. */
+	TMap<FName, TMap<uint64, FAllocationInfo> > SnapshotsWithAllocationMap;
+
+	/** Snapshots with callstack based allocation map. */
+	TMap<FName, TMap<FName, FCombinedAllocationInfo> > SnapshotsWithScopedAllocations;
+
+	TMap<FName, TMap<FString, FCombinedAllocationInfo> > SnapshotsWithDecodedScopedAllocations;
+
+	/** Number of memory operations. */
+	uint64 NumMemoryOperations;
+
+	/** Last sequence tag for named marker. */
+	uint32 LastSequenceTagForNamedMarker;
 };
 

Engine/Source/Developer/Profiler/Public/Interfaces/IProfilerModule.h

@@ -25,5 +25,5 @@ public:
 	virtual void StatsMemoryDumpCommand( const TCHAR* Filename ) = 0;
 
 	/** Creates a new instance of the memory profiler based the raw stats. */
-	virtual FRawStatsMemoryProfiler* OpenRawStatForMemoryProfiling( const TCHAR* Filename ) = 0;
+	virtual FRawStatsMemoryProfiler* OpenRawStatsForMemoryProfiling( const TCHAR* Filename ) = 0;
 };