You've already forked UnrealEngineUWP
mirror of
https://github.com/izzy2lost/UnrealEngineUWP.git
synced 2026-03-26 18:15:20 -07:00
394 lines
15 KiB
C++
394 lines
15 KiB
C++
// Copyright 1998-2019 Epic Games, Inc. All Rights Reserved.
|
|
|
|
#include "CrashDebugPDBCache.h"
|
|
#include "HAL/PlatformTime.h"
|
|
#include "HAL/FileManager.h"
|
|
#include "Misc/Parse.h"
|
|
#include "Templates/ScopedPointer.h"
|
|
#include "Misc/CommandLine.h"
|
|
#include "Misc/Paths.h"
|
|
#include "Misc/ConfigCacheIni.h"
|
|
#include "CrashDebugHelperPrivate.h"
|
|
#include "Templates/UniquePtr.h"
|
|
|
|
/*-----------------------------------------------------------------------------
|
|
PDB Cache implementation
|
|
-----------------------------------------------------------------------------*/
|
|
|
|
const TCHAR* FPDBCache::PDBTimeStampFileNoMeta = TEXT( "PDBTimeStamp.txt" );
|
|
const TCHAR* FPDBCache::PDBTimeStampFile = TEXT( "PDBTimeStamp.bin" );
|
|
|
|
void FPDBCache::Init()
|
|
{
|
|
// PDB Cache
|
|
// Default configuration
|
|
//PDBCachePath=G:/CrashReportPDBCache/
|
|
//DepotRoot=F:/depot
|
|
//DaysToDeleteUnusedFilesFromPDBCache=3
|
|
//PDBCacheSizeGB=300
|
|
//MinDiskFreeSpaceGB=256
|
|
|
|
// Can be enabled only through the command line.
|
|
FParse::Bool( FCommandLine::Get(), TEXT( "bUsePDBCache=" ), bUsePDBCache );
|
|
|
|
UE_LOG( LogCrashDebugHelper, Warning, TEXT( "bUsePDBCache is %s" ), bUsePDBCache ? TEXT( "enabled" ) : TEXT( "disabled" ) );
|
|
|
|
if (bUsePDBCache)
|
|
{
|
|
if (!FParse::Value(FCommandLine::Get(), TEXT("PDBCacheDepotRoot="), DepotRoot))
|
|
{
|
|
GConfig->GetString(TEXT("Engine.CrashDebugHelper"), TEXT("DepotRoot"), DepotRoot, GEngineIni);
|
|
}
|
|
|
|
ICrashDebugHelper::SetDepotIndex( DepotRoot );
|
|
|
|
const bool bHasDepotRoot = IFileManager::Get().DirectoryExists( *DepotRoot );
|
|
UE_CLOG( !bHasDepotRoot, LogCrashDebugHelper, Warning, TEXT( "DepotRoot: %s is not valid" ), *DepotRoot );
|
|
UE_LOG( LogCrashDebugHelper, Log, TEXT( "DepotRoot: %s" ), *DepotRoot );
|
|
|
|
bUsePDBCache = bHasDepotRoot;
|
|
}
|
|
|
|
// Get the rest of the PDB cache configuration.
|
|
if( bUsePDBCache )
|
|
{
|
|
if (!FParse::Value(FCommandLine::Get(), TEXT("PDBCachePath="), PDBCachePath))
|
|
{
|
|
if (!GConfig->GetString(TEXT("Engine.CrashDebugHelper"), TEXT("PDBCachePath"), PDBCachePath, GEngineIni))
|
|
{
|
|
UE_LOG(LogCrashDebugHelper, Warning, TEXT("Failed to get PDBCachePath from ini file or command line"));
|
|
bUsePDBCache = false;
|
|
}
|
|
}
|
|
|
|
ICrashDebugHelper::SetDepotIndex( PDBCachePath );
|
|
}
|
|
|
|
if( bUsePDBCache )
|
|
{
|
|
if (!FParse::Value(FCommandLine::Get(), TEXT("PDBCacheSizeGB="), PDBCacheSizeGB))
|
|
{
|
|
if (!GConfig->GetInt(TEXT("Engine.CrashDebugHelper"), TEXT("PDBCacheSizeGB"), PDBCacheSizeGB, GEngineIni))
|
|
{
|
|
UE_LOG(LogCrashDebugHelper, Warning, TEXT("Failed to get PDBCachePath from ini file or command line"));
|
|
}
|
|
}
|
|
|
|
if (!FParse::Value(FCommandLine::Get(), TEXT("PDBCacheMinFreeSpaceGB="), MinDiskFreeSpaceGB))
|
|
{
|
|
if (!GConfig->GetInt(TEXT("Engine.CrashDebugHelper"), TEXT("MinDiskFreeSpaceGB"), MinDiskFreeSpaceGB, GEngineIni))
|
|
{
|
|
UE_LOG(LogCrashDebugHelper, Warning, TEXT("Failed to get MinDiskFreeSpaceGB from ini file or command line"));
|
|
}
|
|
}
|
|
|
|
if (!FParse::Value(FCommandLine::Get(), TEXT("PDBCacheFileDeleteDays="), DaysToDeleteUnusedFilesFromPDBCache))
|
|
{
|
|
if (!GConfig->GetInt(TEXT("Engine.CrashDebugHelper"), TEXT("DaysToDeleteUnusedFilesFromPDBCache"), DaysToDeleteUnusedFilesFromPDBCache, GEngineIni))
|
|
{
|
|
UE_LOG(LogCrashDebugHelper, Warning, TEXT("Failed to get DaysToDeleteUnusedFilesFromPDBCache from ini file or command line"));
|
|
}
|
|
}
|
|
|
|
InitializePDBCache();
|
|
CleanPDBCache( DaysToDeleteUnusedFilesFromPDBCache );
|
|
|
|
// Verify that we have enough space to enable the PDB Cache.
|
|
uint64 TotalNumberOfBytes = 0;
|
|
uint64 NumberOfFreeBytes = 0;
|
|
FPlatformMisc::GetDiskTotalAndFreeSpace( PDBCachePath, TotalNumberOfBytes, NumberOfFreeBytes );
|
|
|
|
const int32 TotalDiscSpaceGB = int32( TotalNumberOfBytes >> 30 );
|
|
const int32 DiskFreeSpaceGB = int32( NumberOfFreeBytes >> 30 );
|
|
|
|
if( DiskFreeSpaceGB < MinDiskFreeSpaceGB || TotalNumberOfBytes == 0 )
|
|
{
|
|
// There is not enough free space, calculate the current PDB cache usage and try removing the old data.
|
|
const int32 CurrentPDBCacheSizeGB = GetPDBCacheSizeGB();
|
|
const int32 DiskFreeSpaceAfterCleanGB = DiskFreeSpaceGB + CurrentPDBCacheSizeGB;
|
|
if( DiskFreeSpaceAfterCleanGB < MinDiskFreeSpaceGB )
|
|
{
|
|
UE_LOG( LogCrashDebugHelper, Error, TEXT( "There is not enough free space. PDB Cache disabled." ) );
|
|
UE_LOG( LogCrashDebugHelper, Error, TEXT( "Current disk free space is %i GBs." ), DiskFreeSpaceGB );
|
|
UE_LOG( LogCrashDebugHelper, Error, TEXT( "To enable the PDB Cache you need to free %i GB of space" ), MinDiskFreeSpaceGB - DiskFreeSpaceAfterCleanGB );
|
|
bUsePDBCache = false;
|
|
// Remove all data.
|
|
CleanPDBCache( 0 );
|
|
}
|
|
else
|
|
{
|
|
// Clean the PDB cache until we get enough free space.
|
|
const int32 MinSpaceRequirement = FMath::Max( MinDiskFreeSpaceGB - DiskFreeSpaceGB, 0 );
|
|
const int32 CacheSpaceRequirement = FMath::Max( CurrentPDBCacheSizeGB - PDBCacheSizeGB, 0 );
|
|
CleanPDBCache( DaysToDeleteUnusedFilesFromPDBCache, FMath::Max( MinSpaceRequirement, CacheSpaceRequirement ) );
|
|
}
|
|
}
|
|
}
|
|
|
|
if( bUsePDBCache )
|
|
{
|
|
UE_LOG( LogCrashDebugHelper, Log, TEXT( "PDBCachePath: %s" ), *PDBCachePath );
|
|
UE_LOG( LogCrashDebugHelper, Log, TEXT( "PDBCacheSizeGB: %i" ), PDBCacheSizeGB );
|
|
UE_LOG( LogCrashDebugHelper, Log, TEXT( "MinDiskFreeSpaceGB: %i" ), MinDiskFreeSpaceGB );
|
|
UE_LOG( LogCrashDebugHelper, Log, TEXT( "DaysToDeleteUnusedFilesFromPDBCache: %i" ), DaysToDeleteUnusedFilesFromPDBCache );
|
|
}
|
|
}
|
|
|
|
|
|
void FPDBCache::InitializePDBCache()
|
|
{
|
|
const double StartTime = FPlatformTime::Seconds();
|
|
IFileManager::Get().MakeDirectory( *PDBCachePath, true );
|
|
|
|
TArray<FString> PDBCacheEntryDirectories;
|
|
IFileManager::Get().FindFiles( PDBCacheEntryDirectories, *(PDBCachePath / TEXT( "*" )), false, true );
|
|
|
|
for( const auto& Directory : PDBCacheEntryDirectories )
|
|
{
|
|
FPDBCacheEntryPtr Entry = ReadPDBCacheEntry( Directory );
|
|
if (Entry.IsValid())
|
|
{
|
|
PDBCacheEntries.Add( Directory, Entry.ToSharedRef() );
|
|
}
|
|
}
|
|
|
|
SortPDBCache();
|
|
|
|
const double TotalTime = FPlatformTime::Seconds() - StartTime;
|
|
UE_LOG( LogCrashDebugHelper, Log, TEXT( "PDB Cache initialized in %.2f ms" ), TotalTime*1000.0f );
|
|
UE_LOG( LogCrashDebugHelper, Log, TEXT( "Found %i entries which occupy %i GBs" ), PDBCacheEntries.Num(), GetPDBCacheSizeGB() );
|
|
}
|
|
|
|
void FPDBCache::CleanPDBCache( int32 DaysToDelete, int32 NumberOfGBsToBeCleaned /*= 0 */ )
|
|
{
|
|
// Not very efficient, but should do the trick.
|
|
// Revisit it later.
|
|
const double StartTime = FPlatformTime::Seconds();
|
|
|
|
TSet<FString> EntriesToBeRemoved;
|
|
|
|
// Find all outdated PDB Cache entries and mark them for removal.
|
|
const double DaysToDeleteAsSeconds = FTimespan( DaysToDelete, 0, 0, 0 ).GetTotalSeconds();
|
|
int32 NumGBsCleaned = 0;
|
|
for( const auto& It : PDBCacheEntries )
|
|
{
|
|
const FPDBCacheEntryRef& Entry = It.Value;
|
|
const FString EntryDirectory = PDBCachePath / Entry->Directory;
|
|
const FString EntryTimeStampFilename = EntryDirectory / PDBTimeStampFile;
|
|
|
|
const double EntryFileAge = IFileManager::Get().GetFileAgeSeconds( *EntryTimeStampFilename );
|
|
if( EntryFileAge > DaysToDeleteAsSeconds )
|
|
{
|
|
EntriesToBeRemoved.Add( Entry->Directory );
|
|
NumGBsCleaned += Entry->SizeGB;
|
|
}
|
|
}
|
|
|
|
if( NumberOfGBsToBeCleaned > 0 && NumGBsCleaned < NumberOfGBsToBeCleaned )
|
|
{
|
|
// Do the second pass if we need to remove more PDB Cache entries due to the free disk space restriction.
|
|
for( const auto& It : PDBCacheEntries )
|
|
{
|
|
const FPDBCacheEntryRef& Entry = It.Value;
|
|
|
|
if( !EntriesToBeRemoved.Contains( Entry->Directory ) )
|
|
{
|
|
EntriesToBeRemoved.Add( Entry->Directory );
|
|
NumGBsCleaned += Entry->SizeGB;
|
|
|
|
if( NumGBsCleaned > NumberOfGBsToBeCleaned )
|
|
{
|
|
// Break the loop, we are done.
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Remove all marked PDB Cache entries.
|
|
for( const auto& EntryDirectory : EntriesToBeRemoved )
|
|
{
|
|
RemovePDBCacheEntry( EntryDirectory );
|
|
}
|
|
|
|
const double TotalTime = FPlatformTime::Seconds() - StartTime;
|
|
UE_LOG( LogCrashDebugHelper, Log, TEXT( "PDB Cache cleaned %i GBs in %.2f ms" ), NumGBsCleaned, TotalTime*1000.0f );
|
|
}
|
|
|
|
FPDBCacheEntryRef FPDBCache::CreateAndAddPDBCacheEntry( const FString& OriginalLabelName, const FString& DepotName, const TArray<FString>& FilesToBeCached )
|
|
{
|
|
const FString CleanedLabelName = EscapePath( OriginalLabelName );
|
|
const FString EntryDirectory = PDBCachePath / CleanedLabelName;
|
|
const FString EntryTimeStampFilename = EntryDirectory / PDBTimeStampFile;
|
|
|
|
const FString LocalDepotDir = EscapePath( DepotRoot / DepotName );
|
|
|
|
UE_LOG( LogCrashDebugHelper, Warning, TEXT( "PDB Cache entry: %s is being copied from: %s, it will take some time" ), *CleanedLabelName, *OriginalLabelName );
|
|
for( const auto& Filename : FilesToBeCached )
|
|
{
|
|
const FString SourceDirectoryWithSearch = Filename.Replace( *DepotName, *LocalDepotDir );
|
|
|
|
TArray<FString> MatchedFiles;
|
|
IFileManager::Get().FindFiles( MatchedFiles, *SourceDirectoryWithSearch, true, false );
|
|
|
|
for( const auto& MatchedFilename : MatchedFiles )
|
|
{
|
|
const FString SrcFilename = FPaths::GetPath( SourceDirectoryWithSearch ) / MatchedFilename;
|
|
const FString DestFilename = EntryDirectory / SrcFilename.Replace( *LocalDepotDir, TEXT( "" ) );
|
|
IFileManager::Get().Copy( *DestFilename, *SrcFilename );
|
|
}
|
|
}
|
|
|
|
|
|
TArray<FString> CachedFiles;
|
|
IFileManager::Get().FindFilesRecursive( CachedFiles, *EntryDirectory, TEXT( "*.*" ), true, false );
|
|
|
|
// Calculate the size of this PDB Cache entry.
|
|
int64 TotalSize = 0;
|
|
for( const auto& Filename : CachedFiles )
|
|
{
|
|
const int64 FileSize = IFileManager::Get().FileSize( *Filename );
|
|
TotalSize += FileSize;
|
|
}
|
|
|
|
// Round-up the size.
|
|
int32 SizeGB = (int32)FMath::DivideAndRoundUp( TotalSize, (int64)NUM_BYTES_PER_GB );
|
|
FPDBCacheEntryRef NewCacheEntry = MakeShareable( new FPDBCacheEntry( CachedFiles, CleanedLabelName, FDateTime::Now(), SizeGB ) );
|
|
|
|
// Verify there is an entry timestamp file, write the size of a PDB cache to avoid time consuming FindFilesRecursive during initialization.
|
|
TUniquePtr<FArchive> FileWriter( IFileManager::Get().CreateFileWriter( *EntryTimeStampFilename ) );
|
|
UE_CLOG( !FileWriter, LogCrashDebugHelper, Fatal, TEXT( "Couldn't save the timestamp for a file: %s" ), *EntryTimeStampFilename );
|
|
*FileWriter << *NewCacheEntry;
|
|
|
|
PDBCacheEntries.Add( CleanedLabelName, NewCacheEntry );
|
|
SortPDBCache();
|
|
|
|
return NewCacheEntry;
|
|
}
|
|
|
|
FPDBCacheEntryRef FPDBCache::CreateAndAddPDBCacheEntryMixed( const FString& ProductVersion, const TMap<FString, FString>& FilesToBeCached )
|
|
{
|
|
// Enable MDD to parse all minidumps regardless the branch, to fix the issue with the missing executables on the P4 due to the build system changes.
|
|
const FString EntryDirectory = PDBCachePath / ProductVersion;
|
|
const FString EntryTimeStampFilename = EntryDirectory / PDBTimeStampFile;
|
|
|
|
UE_LOG( LogCrashDebugHelper, Warning, TEXT( "PDB Cache entry: %s is being created from %i files, it will take some time" ), *ProductVersion, FilesToBeCached.Num() );
|
|
for( const auto& It : FilesToBeCached )
|
|
{
|
|
const FString& SrcFilename = It.Key;
|
|
const FString DestFilename = EntryDirectory / It.Value;
|
|
IFileManager::Get().Copy( *DestFilename, *SrcFilename );
|
|
}
|
|
|
|
TArray<FString> CachedFiles;
|
|
IFileManager::Get().FindFilesRecursive( CachedFiles, *EntryDirectory, TEXT( "*.*" ), true, false );
|
|
|
|
// Calculate the size of this PDB Cache entry.
|
|
int64 TotalSize = 0;
|
|
for( const auto& Filename : CachedFiles )
|
|
{
|
|
const int64 FileSize = IFileManager::Get().FileSize( *Filename );
|
|
TotalSize += FileSize;
|
|
}
|
|
|
|
// Round-up the size.
|
|
int32 SizeGB = (int32)FMath::DivideAndRoundUp( TotalSize, (int64)NUM_BYTES_PER_GB );
|
|
FPDBCacheEntryRef NewCacheEntry = MakeShareable( new FPDBCacheEntry( CachedFiles, ProductVersion, FDateTime::Now(), SizeGB ) );
|
|
|
|
// Verify there is an entry timestamp file, write the size of a PDB cache to avoid time consuming FindFilesRecursive during initialization.
|
|
TUniquePtr<FArchive> FileWriter( IFileManager::Get().CreateFileWriter( *EntryTimeStampFilename ) );
|
|
UE_CLOG( !FileWriter, LogCrashDebugHelper, Fatal, TEXT( "Couldn't save the timestamp for a file: %s" ), *EntryTimeStampFilename );
|
|
*FileWriter << *NewCacheEntry;
|
|
|
|
PDBCacheEntries.Add( ProductVersion, NewCacheEntry );
|
|
SortPDBCache();
|
|
|
|
return NewCacheEntry;
|
|
}
|
|
|
|
FPDBCacheEntryPtr FPDBCache::ReadPDBCacheEntry( const FString& Directory )
|
|
{
|
|
const FString EntryDirectory = PDBCachePath / Directory;
|
|
const FString EntryTimeStampFilenameNoMeta = EntryDirectory / PDBTimeStampFileNoMeta;
|
|
const FString EntryTimeStampFilename = EntryDirectory / PDBTimeStampFile;
|
|
|
|
// Verify there is an entry timestamp file.
|
|
const FDateTime LastAccessTimeNoMeta = IFileManager::Get().GetTimeStamp( *EntryTimeStampFilenameNoMeta );
|
|
const FDateTime LastAccessTime = IFileManager::Get().GetTimeStamp( *EntryTimeStampFilename );
|
|
|
|
FPDBCacheEntryPtr NewEntry;
|
|
|
|
if( LastAccessTime != FDateTime::MinValue() )
|
|
{
|
|
// Read the metadata
|
|
TUniquePtr<FArchive> FileReader( IFileManager::Get().CreateFileReader( *EntryTimeStampFilename ) );
|
|
NewEntry = MakeShareable( new FPDBCacheEntry( LastAccessTime ) );
|
|
*FileReader << *NewEntry;
|
|
}
|
|
else if( LastAccessTimeNoMeta != FDateTime::MinValue() )
|
|
{
|
|
// Calculate the size of this PDB Cache entry and update to the new version.
|
|
TArray<FString> PDBFiles;
|
|
IFileManager::Get().FindFilesRecursive( PDBFiles, *EntryDirectory, TEXT( "*.*" ), true, false );
|
|
|
|
// Calculate the size of this PDB Cache entry.
|
|
int64 TotalSize = 0;
|
|
for( const auto& Filename : PDBFiles )
|
|
{
|
|
const int64 FileSize = IFileManager::Get().FileSize( *Filename );
|
|
TotalSize += FileSize;
|
|
}
|
|
|
|
// Round-up the size.
|
|
const int32 SizeGB = (int32)FMath::DivideAndRoundUp( TotalSize, (int64)NUM_BYTES_PER_GB );
|
|
NewEntry = MakeShareable( new FPDBCacheEntry( PDBFiles, Directory, LastAccessTimeNoMeta, SizeGB ) );
|
|
|
|
// Remove the old file and save the metadata.
|
|
TUniquePtr<FArchive> FileWriter( IFileManager::Get().CreateFileWriter( *EntryTimeStampFilename ) );
|
|
*FileWriter << *NewEntry;
|
|
|
|
IFileManager::Get().Delete( *EntryTimeStampFilenameNoMeta );
|
|
}
|
|
else
|
|
{
|
|
// Something wrong.
|
|
ensureMsgf( 0, TEXT( "Invalid symbol cache entry: %s" ), *EntryDirectory );
|
|
}
|
|
|
|
return NewEntry;
|
|
}
|
|
|
|
void FPDBCache::TouchPDBCacheEntry( const FString& Directory )
|
|
{
|
|
const FString EntryDirectory = PDBCachePath / Directory;
|
|
const FString EntryTimeStampFilename = EntryDirectory / PDBTimeStampFile;
|
|
|
|
FPDBCacheEntryRef& Entry = PDBCacheEntries.FindChecked( Directory );
|
|
Entry->SetLastAccessTimeToNow();
|
|
|
|
const bool bResult = IFileManager::Get().SetTimeStamp( *EntryTimeStampFilename, Entry->LastAccessTime );
|
|
SortPDBCache();
|
|
}
|
|
|
|
void FPDBCache::RemovePDBCacheEntry( const FString& Directory )
|
|
{
|
|
const double StartTime = FPlatformTime::Seconds();
|
|
|
|
const FString EntryDirectory = PDBCachePath / Directory;
|
|
|
|
FPDBCacheEntryRef& Entry = PDBCacheEntries.FindChecked( Directory );
|
|
IFileManager::Get().DeleteDirectory( *EntryDirectory, true, true );
|
|
|
|
const double TotalTime = FPlatformTime::Seconds() - StartTime;
|
|
UE_LOG( LogCrashDebugHelper, Warning, TEXT( "PDB Cache entry %s removed in %.2f ms, restored %i GBs" ), *Directory, TotalTime*1000.0f, Entry->SizeGB );
|
|
|
|
PDBCacheEntries.Remove( Directory );
|
|
}
|
|
|
|
FPDBCacheEntryRef FPDBCache::FindAndTouchPDBCacheEntry( const FString& PathOrLabel )
|
|
{
|
|
FPDBCacheEntryRef CacheEntry = PDBCacheEntries.FindChecked( EscapePath( PathOrLabel ) );
|
|
TouchPDBCacheEntry( CacheEntry->Directory );
|
|
return CacheEntry;
|
|
}
|