/** * Copyright 1998-2014 Epic Games, Inc. All Rights Reserved. */ using System; using System.IO; using System.Collections.Generic; using System.Diagnostics; using System.Runtime.InteropServices; using System.Text; using System.Linq; namespace MemoryProfiler2 { /// Helper struct encapsulating information for a callstack and associated allocation size. Shared with stream parser. public struct FCallStackAllocationInfo { /// Size of allocation. public long Size; /// Index of callstack that performed allocation. public int CallStackIndex; /// Number of allocations. public int Count; /// Constructor, initializing all member variables to passed in values. public FCallStackAllocationInfo( long InSize, int InCallStackIndex, int InCount ) { Size = InSize; CallStackIndex = InCallStackIndex; Count = InCount; } /// /// Adds the passed in size and count to this callstack info. /// /// Size to add /// Count to add public FCallStackAllocationInfo Add( long SizeToAdd, int CountToAdd ) { return new FCallStackAllocationInfo( Size + SizeToAdd, CallStackIndex, Count + CountToAdd ); } /// /// Diffs the two passed in callstack infos and returns the difference. /// /// Newer callstack info to subtract older from /// Older callstack info to subtract from older public static FCallStackAllocationInfo Diff( FCallStackAllocationInfo New, FCallStackAllocationInfo Old ) { if( New.CallStackIndex != Old.CallStackIndex ) { throw new InvalidDataException(); } return new FCallStackAllocationInfo( New.Size - Old.Size, New.CallStackIndex, New.Count - Old.Count ); } } /// Additional memory statistics. public enum ESnapshotMetricV3 { /// Memory profiling overhead. MemoryProfilingOverhead, /// Memory allocated from OS. PS3 only. AllocatedFromOS, /// Maximum amount of memory allocated from OS. PS3 only. MaxAllocatedFromOS, /// Memory requested by game. PS3 only. AllocateFromGame, /// Total used chunks by the allocator. TotalUsedChunks, /// Lightmap memory. TextureLightmapMemory, /// Shadowmap memory. TextureShadowmapMemory, /// Last item used as a count. Count, } /// State of total memory at a specific moment in time. public enum ESliceTypesV3 { TotalUsed, TotalAllocated, CPUUsed, CPUSlack, CPUWaste, GPUUsed, GPUSlack, GPUWaste, OSOverhead, ImageSize, HostUsed, HostSlack, HostWaste, OverallAllocatedMemory, Count }; public enum ESliceTypesV4 { PlatformUsedPhysical, BinnedWasteCurrent, BinnedUsedCurrent, BinnedSlackCurrent, MemoryProfilingOverhead, OverallAllocatedMemory, Count }; public class FMemorySlice { public long[] MemoryInfo = null; public FMemorySlice( FStreamSnapshot Snapshot ) { if( FStreamToken.Version >= 4 ) { MemoryInfo = new long[] { Snapshot.MemoryAllocationStats4[FMemoryAllocationStatsV4.PlatformUsedPhysical], Snapshot.MemoryAllocationStats4[FMemoryAllocationStatsV4.BinnedWasteCurrent], Snapshot.MemoryAllocationStats4[FMemoryAllocationStatsV4.BinnedUsedCurrent], Snapshot.MemoryAllocationStats4[FMemoryAllocationStatsV4.BinnedSlackCurrent], Snapshot.MemoryAllocationStats4[FMemoryAllocationStatsV4.MemoryProfilingOverhead], Snapshot.AllocationSize, }; } else { MemoryInfo = new long[] { Snapshot.MemoryAllocationStats3.TotalUsed, Snapshot.MemoryAllocationStats3.TotalAllocated, Snapshot.MemoryAllocationStats3.CPUUsed, Snapshot.MemoryAllocationStats3.CPUSlack, Snapshot.MemoryAllocationStats3.CPUWaste, Snapshot.MemoryAllocationStats3.GPUUsed, Snapshot.MemoryAllocationStats3.GPUSlack, Snapshot.MemoryAllocationStats3.GPUWaste, Snapshot.MemoryAllocationStats3.OSOverhead, Snapshot.MemoryAllocationStats3.ImageSize, Snapshot.MemoryAllocationStats3.HostUsed, Snapshot.MemoryAllocationStats3.HostSlack, Snapshot.MemoryAllocationStats3.HostWaste, Snapshot.AllocationSize, }; } } public long GetSliceInfoV3( ESliceTypesV3 SliceType ) { return ( MemoryInfo[(int)SliceType] ); } public long GetSliceInfoV4( ESliceTypesV4 SliceType ) { return ( MemoryInfo[(int)SliceType] ); } } /// Snapshot of allocation state at a specific time in token stream. public class FStreamSnapshot { /// User defined description of time of snapshot. public string Description; /// List of callstack allocations. public List ActiveCallStackList; /// List of lifetime callstack allocations for memory churn. public List LifetimeCallStackList; /// Position in the stream. public ulong StreamIndex; /// Frame number. public int FrameNumber; /// Current time. public float CurrentTime; /// Current time starting from the previous snapshot marker. public float ElapsedTime; /// Snapshot type. public EProfilingPayloadSubType SubType = EProfilingPayloadSubType.SUBTYPE_SnapshotMarker; /// Index of snapshot. public int SnapshotIndex; /// Platform dependent array of memory metrics. public List MetricArray; /// A list of indices into the name table, one for each loaded level including persistent level. public List LoadedLevels = new List(); /// Array of names of all currently loaded levels formated for usage in details view tab. public List LoadedLevelNames = new List(); /// Generic memory allocation stats. public FMemoryAllocationStatsV3 MemoryAllocationStats3 = new FMemoryAllocationStatsV3(); /// Generic memory allocation stats. public FMemoryAllocationStatsV4 MemoryAllocationStats4 = new FMemoryAllocationStatsV4(); /// Running count of number of allocations. public long AllocationCount = 0; /// Running total of size of allocations. public long AllocationSize = 0; /// Running total of size of allocations. public long AllocationMaxSize = 0; /// True if this snapshot was created as a diff of two snapshots. public bool bIsDiffResult; /// Running total of allocated memory. public List OverallMemorySlice; /// Constructor, naming the snapshot and initializing map. public FStreamSnapshot( string InDescription ) { Description = InDescription; ActiveCallStackList = new List(); // Presize lifetime callstack array and populate. LifetimeCallStackList = new List( FStreamInfo.GlobalInstance.CallStackArray.Count ); for( int CallStackIndex=0; CallStackIndex(); } /// Performs a deep copy of the relevant data structures. public FStreamSnapshot DeepCopy( Dictionary PointerToPointerInfoMap ) { // Create new snapshot object. FStreamSnapshot Snapshot = new FStreamSnapshot( "Copy" ); // Manually perform a deep copy of LifetimeCallstackList foreach( FCallStackAllocationInfo AllocationInfo in LifetimeCallStackList ) { Snapshot.LifetimeCallStackList[ AllocationInfo.CallStackIndex ] = new FCallStackAllocationInfo( AllocationInfo.Size, AllocationInfo.CallStackIndex, AllocationInfo.Count ); } Snapshot.AllocationCount = AllocationCount; Snapshot.AllocationSize = AllocationSize; Snapshot.AllocationMaxSize = AllocationMaxSize; Snapshot.FinalizeSnapshot( PointerToPointerInfoMap ); // Return deep copy of this snapshot. return Snapshot; } public void FillActiveCallStackList( Dictionary PointerToPointerInfoMap ) { ActiveCallStackList.Clear(); ActiveCallStackList.Capacity = LifetimeCallStackList.Count; foreach( KeyValuePair PointerData in PointerToPointerInfoMap ) { int CallStackIndex = PointerData.Value.CallStackIndex; long Size = PointerData.Value.Size; // make sure allocationInfoList is big enough while( CallStackIndex >= ActiveCallStackList.Count ) { ActiveCallStackList.Add( new FCallStackAllocationInfo( 0, ActiveCallStackList.Count, 0 ) ); } FCallStackAllocationInfo NewAllocInfo = ActiveCallStackList[ CallStackIndex ]; NewAllocInfo.Size += Size; NewAllocInfo.Count++; ActiveCallStackList[ CallStackIndex ] = NewAllocInfo; } // strip out any callstacks with no allocations for( int i = ActiveCallStackList.Count - 1; i >= 0; i-- ) { if( ActiveCallStackList[ i ].Count == 0 ) { ActiveCallStackList.RemoveAt( i ); } } ActiveCallStackList.TrimExcess(); } /// Convert "callstack to allocation" mapping (either passed in or generated from pointer map) to callstack info list. public void FinalizeSnapshot( Dictionary PointerToPointerInfoMap ) { FillActiveCallStackList( PointerToPointerInfoMap ); } /// Diffs two snapshots and creates a result one. public static FStreamSnapshot DiffSnapshots( FStreamSnapshot Old, FStreamSnapshot New ) { // Create result snapshot object. FStreamSnapshot ResultSnapshot = new FStreamSnapshot( "Diff " + Old.Description + " <-> " + New.Description ); using( FScopedLogTimer LoadingTime = new FScopedLogTimer( "FStreamSnapshot.DiffSnapshots" ) ) { // Copy over allocation count so we can track where the graph starts ResultSnapshot.AllocationCount = Old.AllocationCount; Debug.Assert( Old.MetricArray.Count == New.MetricArray.Count ); ResultSnapshot.MetricArray = new List( Old.MetricArray.Count ); for( int CallstackIndex = 0; CallstackIndex < Old.MetricArray.Count; CallstackIndex++ ) { ResultSnapshot.MetricArray.Add( New.MetricArray[ CallstackIndex ] - Old.MetricArray[ CallstackIndex ] ); } ResultSnapshot.MemoryAllocationStats3 = FMemoryAllocationStatsV3.Diff( Old.MemoryAllocationStats3, New.MemoryAllocationStats3 ); ResultSnapshot.MemoryAllocationStats4 = FMemoryAllocationStatsV4.Diff( Old.MemoryAllocationStats4, New.MemoryAllocationStats4 ); ResultSnapshot.StreamIndex = New.StreamIndex; ResultSnapshot.bIsDiffResult = true; ResultSnapshot.AllocationMaxSize = New.AllocationMaxSize - Old.AllocationMaxSize; ResultSnapshot.AllocationSize = New.AllocationSize - Old.AllocationSize; ResultSnapshot.CurrentTime = 0; ResultSnapshot.ElapsedTime = New.CurrentTime - Old.CurrentTime; ResultSnapshot.FrameNumber = New.FrameNumber - Old.FrameNumber; ResultSnapshot.LoadedLevels = New.LoadedLevels; // These lists are guaranteed to be sorted by callstack index. List OldActiveCallStackList = Old.ActiveCallStackList; List NewActiveCallStackList = New.ActiveCallStackList; List ResultActiveCallStackList = new List( FStreamInfo.GlobalInstance.CallStackArray.Count ); int OldIndex = 0; int NewIndex = 0; while( true ) { FCallStackAllocationInfo OldAllocInfo = OldActiveCallStackList[ OldIndex ]; FCallStackAllocationInfo NewAllocInfo = NewActiveCallStackList[ NewIndex ]; if( OldAllocInfo.CallStackIndex == NewAllocInfo.CallStackIndex ) { long ResultSize = NewAllocInfo.Size - OldAllocInfo.Size; int ResultCount = NewAllocInfo.Count - OldAllocInfo.Count; if( ResultSize != 0 || ResultCount != 0 ) { ResultActiveCallStackList.Add( new FCallStackAllocationInfo( ResultSize, NewAllocInfo.CallStackIndex, ResultCount ) ); } OldIndex++; NewIndex++; } else if( OldAllocInfo.CallStackIndex > NewAllocInfo.CallStackIndex ) { ResultActiveCallStackList.Add( NewAllocInfo ); NewIndex++; } else // OldAllocInfo.CallStackIndex < NewAllocInfo.CallStackIndex { ResultActiveCallStackList.Add( new FCallStackAllocationInfo( -OldAllocInfo.Size, OldAllocInfo.CallStackIndex, -OldAllocInfo.Count ) ); OldIndex++; } if( OldIndex >= OldActiveCallStackList.Count ) { for( ; NewIndex < NewActiveCallStackList.Count; NewIndex++ ) { ResultActiveCallStackList.Add( NewActiveCallStackList[ NewIndex ] ); } break; } if( NewIndex >= NewActiveCallStackList.Count ) { for( ; OldIndex < OldActiveCallStackList.Count; OldIndex++ ) { ResultActiveCallStackList.Add( OldActiveCallStackList[ OldIndex ] ); } break; } } // Check that list was correctly constructed. for( int CallstackIndex = 0; CallstackIndex < ResultActiveCallStackList.Count - 1; CallstackIndex++ ) { Debug.Assert( ResultActiveCallStackList[ CallstackIndex ].CallStackIndex < ResultActiveCallStackList[ CallstackIndex + 1 ].CallStackIndex ); } ResultActiveCallStackList.TrimExcess(); ResultSnapshot.ActiveCallStackList = ResultActiveCallStackList; // Iterate over new lifetime callstack info and subtract previous one. for( int CallStackIndex = 0; CallStackIndex < New.LifetimeCallStackList.Count; CallStackIndex++ ) { ResultSnapshot.LifetimeCallStackList[ CallStackIndex ] = FCallStackAllocationInfo.Diff( New.LifetimeCallStackList[ CallStackIndex ], Old.LifetimeCallStackList[ CallStackIndex ] ); } // Handle overall memory timeline if( New.OverallMemorySlice.Count > Old.OverallMemorySlice.Count ) { ResultSnapshot.OverallMemorySlice = new List( New.OverallMemorySlice ); ResultSnapshot.OverallMemorySlice.RemoveRange( 0, Old.OverallMemorySlice.Count ); } else { ResultSnapshot.OverallMemorySlice = new List( Old.OverallMemorySlice ); ResultSnapshot.OverallMemorySlice.RemoveRange( 0, New.OverallMemorySlice.Count ); ResultSnapshot.OverallMemorySlice.Reverse(); } } return ResultSnapshot; } /// Exports this snapshot to a CSV file of the passed in name. /// File name to export to /// Whether to export active allocations or lifetime allocations public void ExportToCSV( string FileName, bool bShouldExportActiveAllocations ) { // Create stream writer used to output call graphs to CSV. StreamWriter CSVWriter = new StreamWriter(FileName); // Figure out which list to export. List CallStackList = null; if( bShouldExportActiveAllocations ) { CallStackList = ActiveCallStackList; } else { CallStackList = LifetimeCallStackList; } // Iterate over each unique call graph and spit it out. The sorting is per call stack and not // allocation size. Excel can be used to sort by allocation if needed. This sorting is more robust // and also what the call graph parsing code needs. foreach( FCallStackAllocationInfo AllocationInfo in CallStackList ) { // Skip callstacks with no contribution in this snapshot. if( AllocationInfo.Count > 0 ) { // Dump size first, followed by count. CSVWriter.Write(AllocationInfo.Size + "," + AllocationInfo.Count + ","); // Iterate over ach address in callstack and dump to CSV. FCallStack CallStack = FStreamInfo.GlobalInstance.CallStackArray[AllocationInfo.CallStackIndex]; foreach( int AddressIndex in CallStack.AddressIndices ) { FCallStackAddress Address = FStreamInfo.GlobalInstance.CallStackAddressArray[AddressIndex]; string SymbolFunctionName = FStreamInfo.GlobalInstance.NameArray[Address.FunctionIndex]; string SymbolFileName = FStreamInfo.GlobalInstance.NameArray[Address.FilenameIndex]; // Write out function followed by filename and then line number if valid if( SymbolFunctionName != "" || SymbolFileName != "" ) { if( FStreamInfo.GlobalInstance.Platform == EPlatformType.PS3 ) { CSVWriter.Write( SymbolFunctionName + "," ); } else { CSVWriter.Write( SymbolFunctionName + " @ " + SymbolFileName + ":" + Address.LineNumber + "," ); } } else { CSVWriter.Write("Unknown,"); } } CSVWriter.WriteLine(""); } } // Close the file handle now that we're done writing. CSVWriter.Close(); } /// Exports this snapshots internal properties into a human readable format. public override string ToString() { StringBuilder StrBuilder = new StringBuilder( 1024 ); ///// User defined description of time of snapshot. //public string Description; StrBuilder.AppendLine( "Description: " + Description ); ///// List of callstack allocations. //public List ActiveCallStackList; ///// List of lifetime callstack allocations for memory churn. //public List LifetimeCallStackList; ///// Position in the stream. //public ulong StreamIndex; StrBuilder.AppendLine( "Stream Index: " + StreamIndex ); ///// Frame number. //public int FrameNumber; StrBuilder.AppendLine( "Frame Number: " + FrameNumber ); ///// Current time. //public float CurrentTime; StrBuilder.AppendLine( "Current Time: " + CurrentTime + " seconds" ); ///// Current time starting from the previous snapshot marker. //public float ElapsedTime; StrBuilder.AppendLine( "Elapsed Time: " + ElapsedTime + " seconds" ); ///// Snapshot type. //public EProfilingPayloadSubType SubType = EProfilingPayloadSubType.SUBTYPE_SnapshotMarker; StrBuilder.AppendLine( "Snapshot Type: " + SubType.ToString() ); ///// Index of snapshot. //public int SnapshotIndex; StrBuilder.AppendLine( "Snapshot Index: " + SnapshotIndex ); ///// Platform dependent memory metrics. //public List Metrics; StrBuilder.AppendLine( "Metrics: " ); if( MetricArray.Count > 0 ) { if( FStreamToken.Version >= 4 ) { // Just ignore } else if( FStreamToken.Version > 2 && MetricArray.Count == (int)ESnapshotMetricV3.Count ) { string[] MetricNames = Enum.GetNames( typeof( ESnapshotMetricV3 ) ); for( int MetricIndex = 0; MetricIndex < MetricNames.Length - 1; MetricIndex++ ) { string MetricName = MetricNames[ MetricIndex ]; long MetricData = MetricArray[ MetricIndex ]; if( MetricData != 0 ) { string MetricValue = ( MetricName == "TotalUsedChunks" ) ? MetricData.ToString() : MainWindow.FormatSizeString2( MetricData ); string Metric = " " + MetricName + ": " + MetricValue; StrBuilder.AppendLine( Metric ); } } } } ///// Array of names of all currently loaded levels formated for usage in details view tab. //public List LoadedLevelNames = new List(); StrBuilder.AppendLine( "Loaded Levels: " + LoadedLevels.Count ); foreach( string LevelName in LoadedLevelNames ) { StrBuilder.AppendLine( " " + LevelName ); } ///// Generic memory allocation stats. //public FMemoryAllocationStats MemoryAllocationStats = new FMemoryAllocationStats(); if( FStreamToken.Version >= 4 ) { StrBuilder.AppendLine( "Memory Allocation Stats: " ); StrBuilder.Append( MemoryAllocationStats4.ToString() ); } else { StrBuilder.AppendLine( "Memory Allocation Stats: " ); StrBuilder.Append( MemoryAllocationStats3.ToString() ); } ///// Running count of number of allocations. //public long AllocationCount = 0; StrBuilder.AppendLine( "Allocation Count: " + AllocationCount ); ///// Running total of size of allocations. //public long AllocationSize = 0; StrBuilder.AppendLine( "Allocation Size: " + MainWindow.FormatSizeString2( AllocationSize ) ); ///// Running total of size of allocations. //public long AllocationMaxSize = 0; StrBuilder.AppendLine( "Allocation Max Size: " + MainWindow.FormatSizeString2( AllocationMaxSize ) ); ///// True if this snapshot was created as a diff of two snapshots. //public bool bIsDiffResult; StrBuilder.AppendLine( "Is Diff Result: " + bIsDiffResult ); ///// Running total of allocated memory. //public List OverallMemorySlice; //StrBuilder.AppendLine( "Overall Memory Slice: @TODO" ); return StrBuilder.ToString(); } /// Encapsulates indices to levels in the diff snapshots in relation to start and end snapshot. struct FLevelIndex { public FLevelIndex( int InLeftIndex, int InRightIndex ) { LeftIndex = InLeftIndex; RightIndex = InRightIndex; } public override string ToString() { return LeftIndex + " <-> " + RightIndex; } public int LeftIndex; public int RightIndex; } /// /// Prepares three array with level names. Arrays will be placed into LoadedLevelNames properties of each of snapshots. /// "-" in level name means that level was unloaded. /// "+" in level name means that level was loaded. /// public static void PrepareLevelNamesForDetailsTab( FStreamSnapshot LeftSnapshot, FStreamSnapshot DiffSnapshot, FStreamSnapshot RightSnapshot ) { if( DiffSnapshot != null && LeftSnapshot != null && RightSnapshot != null ) { LeftSnapshot.LoadedLevelNames.Clear(); DiffSnapshot.LoadedLevelNames.Clear(); RightSnapshot.LoadedLevelNames.Clear(); List LevelIndexArray = new List( LeftSnapshot.LoadedLevels.Count + RightSnapshot.LoadedLevels.Count ); List AllLevelIndexArray = new List( LeftSnapshot.LoadedLevels ); AllLevelIndexArray.AddRange( RightSnapshot.LoadedLevels ); IEnumerable AllLevelIndicesDistinct = AllLevelIndexArray.Distinct(); foreach( int LevelIndex in AllLevelIndicesDistinct ) { int StartLevelIndex = LeftSnapshot.LoadedLevels.IndexOf( LevelIndex ); int EndLevelIndex = RightSnapshot.LoadedLevels.IndexOf( LevelIndex ); LevelIndexArray.Add( new FLevelIndex( StartLevelIndex, EndLevelIndex ) ); } foreach( FLevelIndex LevelIndex in LevelIndexArray ) { string LeftLevelName = ""; string DiffLevelName = ""; string RightLevelName = ""; if( LevelIndex.LeftIndex != -1 ) { LeftLevelName = FStreamInfo.GlobalInstance.NameArray[ LeftSnapshot.LoadedLevels[ LevelIndex.LeftIndex ] ]; } if( LevelIndex.RightIndex != -1 ) { RightLevelName = FStreamInfo.GlobalInstance.NameArray[ RightSnapshot.LoadedLevels[ LevelIndex.RightIndex ] ]; } if( LevelIndex.LeftIndex != -1 && LevelIndex.RightIndex == -1 ) { DiffLevelName = "-" + LeftLevelName; } else if( LevelIndex.LeftIndex == -1 && LevelIndex.RightIndex != -1 ) { DiffLevelName = "+" + RightLevelName; } else if( LevelIndex.LeftIndex != -1 && LevelIndex.RightIndex != -1 ) { DiffLevelName = " " + RightLevelName; } LeftSnapshot.LoadedLevelNames.Add( LeftLevelName ); DiffSnapshot.LoadedLevelNames.Add( DiffLevelName ); RightSnapshot.LoadedLevelNames.Add( RightLevelName ); } } else if( LeftSnapshot != null ) { LeftSnapshot.LoadedLevelNames.Clear(); foreach( int LevelIndex in LeftSnapshot.LoadedLevels ) { LeftSnapshot.LoadedLevelNames.Add( FStreamInfo.GlobalInstance.NameArray[ LevelIndex ] ); } } else if( RightSnapshot != null ) { RightSnapshot.LoadedLevelNames.Clear(); foreach( int LevelIndex in RightSnapshot.LoadedLevels ) { RightSnapshot.LoadedLevelNames.Add( FStreamInfo.GlobalInstance.NameArray[ LevelIndex ] ); } } } }; }