Removed unneeded dependencies from CrashReportClient

Legacy code in CrashDebugHelper was dragging in SourceControl and AssetRegistry as dependencies for CrashReportClient.

This dependency is no longer needed, as internal crashes now always perform local symbolification (either via symbols built locally, or synced via UGS). Syncing symbols from Perforce or a network drive is no longer needed or used.

#rb Ben.Marsh
#rnx

[CL 9422370 by Jamie Dale in Dev-Core branch]
This commit is contained in:
Jamie Dale
2019-10-04 16:17:40 -04:00
parent f1cbbaa76a
commit 2ad3997cc5
18 changed files with 117 additions and 1781 deletions

View File

@@ -22,7 +22,6 @@ public class CrashDebugHelper : ModuleRules
PublicDependencyModuleNames.AddRange(
new string[] {
"Core",
"SourceControl"
}
);
}

View File

@@ -1,392 +0,0 @@
// Copyright 1998-2019 Epic Games, Inc. All Rights Reserved.
#include "CrashDebugPDBCache.h"
#include "HAL/PlatformTime.h"
#include "HAL/FileManager.h"
#include "Misc/Parse.h"
#include "Misc/CommandLine.h"
#include "Misc/Paths.h"
#include "Misc/ConfigCacheIni.h"
#include "CrashDebugHelperPrivate.h"
#include "Templates/UniquePtr.h"
/*-----------------------------------------------------------------------------
PDB Cache implementation
-----------------------------------------------------------------------------*/
const TCHAR* FPDBCache::PDBTimeStampFileNoMeta = TEXT( "PDBTimeStamp.txt" );
const TCHAR* FPDBCache::PDBTimeStampFile = TEXT( "PDBTimeStamp.bin" );
void FPDBCache::Init()
{
// PDB Cache
// Default configuration
//PDBCachePath=G:/CrashReportPDBCache/
//DepotRoot=F:/depot
//DaysToDeleteUnusedFilesFromPDBCache=3
//PDBCacheSizeGB=300
//MinDiskFreeSpaceGB=256
// Can be enabled only through the command line.
FParse::Bool( FCommandLine::Get(), TEXT( "bUsePDBCache=" ), bUsePDBCache );
UE_LOG( LogCrashDebugHelper, Warning, TEXT( "bUsePDBCache is %s" ), bUsePDBCache ? TEXT( "enabled" ) : TEXT( "disabled" ) );
if (bUsePDBCache)
{
if (!FParse::Value(FCommandLine::Get(), TEXT("PDBCacheDepotRoot="), DepotRoot))
{
GConfig->GetString(TEXT("Engine.CrashDebugHelper"), TEXT("DepotRoot"), DepotRoot, GEngineIni);
}
ICrashDebugHelper::SetDepotIndex( DepotRoot );
const bool bHasDepotRoot = IFileManager::Get().DirectoryExists( *DepotRoot );
UE_CLOG( !bHasDepotRoot, LogCrashDebugHelper, Warning, TEXT( "DepotRoot: %s is not valid" ), *DepotRoot );
UE_LOG( LogCrashDebugHelper, Log, TEXT( "DepotRoot: %s" ), *DepotRoot );
bUsePDBCache = bHasDepotRoot;
}
// Get the rest of the PDB cache configuration.
if( bUsePDBCache )
{
if (!FParse::Value(FCommandLine::Get(), TEXT("PDBCachePath="), PDBCachePath))
{
if (!GConfig->GetString(TEXT("Engine.CrashDebugHelper"), TEXT("PDBCachePath"), PDBCachePath, GEngineIni))
{
UE_LOG(LogCrashDebugHelper, Warning, TEXT("Failed to get PDBCachePath from ini file or command line"));
bUsePDBCache = false;
}
}
ICrashDebugHelper::SetDepotIndex( PDBCachePath );
}
if( bUsePDBCache )
{
if (!FParse::Value(FCommandLine::Get(), TEXT("PDBCacheSizeGB="), PDBCacheSizeGB))
{
if (!GConfig->GetInt(TEXT("Engine.CrashDebugHelper"), TEXT("PDBCacheSizeGB"), PDBCacheSizeGB, GEngineIni))
{
UE_LOG(LogCrashDebugHelper, Warning, TEXT("Failed to get PDBCachePath from ini file or command line"));
}
}
if (!FParse::Value(FCommandLine::Get(), TEXT("PDBCacheMinFreeSpaceGB="), MinDiskFreeSpaceGB))
{
if (!GConfig->GetInt(TEXT("Engine.CrashDebugHelper"), TEXT("MinDiskFreeSpaceGB"), MinDiskFreeSpaceGB, GEngineIni))
{
UE_LOG(LogCrashDebugHelper, Warning, TEXT("Failed to get MinDiskFreeSpaceGB from ini file or command line"));
}
}
if (!FParse::Value(FCommandLine::Get(), TEXT("PDBCacheFileDeleteDays="), DaysToDeleteUnusedFilesFromPDBCache))
{
if (!GConfig->GetInt(TEXT("Engine.CrashDebugHelper"), TEXT("DaysToDeleteUnusedFilesFromPDBCache"), DaysToDeleteUnusedFilesFromPDBCache, GEngineIni))
{
UE_LOG(LogCrashDebugHelper, Warning, TEXT("Failed to get DaysToDeleteUnusedFilesFromPDBCache from ini file or command line"));
}
}
InitializePDBCache();
CleanPDBCache( DaysToDeleteUnusedFilesFromPDBCache );
// Verify that we have enough space to enable the PDB Cache.
uint64 TotalNumberOfBytes = 0;
uint64 NumberOfFreeBytes = 0;
FPlatformMisc::GetDiskTotalAndFreeSpace( PDBCachePath, TotalNumberOfBytes, NumberOfFreeBytes );
const int32 TotalDiscSpaceGB = int32( TotalNumberOfBytes >> 30 );
const int32 DiskFreeSpaceGB = int32( NumberOfFreeBytes >> 30 );
if( DiskFreeSpaceGB < MinDiskFreeSpaceGB || TotalNumberOfBytes == 0 )
{
// There is not enough free space, calculate the current PDB cache usage and try removing the old data.
const int32 CurrentPDBCacheSizeGB = GetPDBCacheSizeGB();
const int32 DiskFreeSpaceAfterCleanGB = DiskFreeSpaceGB + CurrentPDBCacheSizeGB;
if( DiskFreeSpaceAfterCleanGB < MinDiskFreeSpaceGB )
{
UE_LOG( LogCrashDebugHelper, Error, TEXT( "There is not enough free space. PDB Cache disabled." ) );
UE_LOG( LogCrashDebugHelper, Error, TEXT( "Current disk free space is %i GBs." ), DiskFreeSpaceGB );
UE_LOG( LogCrashDebugHelper, Error, TEXT( "To enable the PDB Cache you need to free %i GB of space" ), MinDiskFreeSpaceGB - DiskFreeSpaceAfterCleanGB );
bUsePDBCache = false;
// Remove all data.
CleanPDBCache( 0 );
}
else
{
// Clean the PDB cache until we get enough free space.
const int32 MinSpaceRequirement = FMath::Max( MinDiskFreeSpaceGB - DiskFreeSpaceGB, 0 );
const int32 CacheSpaceRequirement = FMath::Max( CurrentPDBCacheSizeGB - PDBCacheSizeGB, 0 );
CleanPDBCache( DaysToDeleteUnusedFilesFromPDBCache, FMath::Max( MinSpaceRequirement, CacheSpaceRequirement ) );
}
}
}
if( bUsePDBCache )
{
UE_LOG( LogCrashDebugHelper, Log, TEXT( "PDBCachePath: %s" ), *PDBCachePath );
UE_LOG( LogCrashDebugHelper, Log, TEXT( "PDBCacheSizeGB: %i" ), PDBCacheSizeGB );
UE_LOG( LogCrashDebugHelper, Log, TEXT( "MinDiskFreeSpaceGB: %i" ), MinDiskFreeSpaceGB );
UE_LOG( LogCrashDebugHelper, Log, TEXT( "DaysToDeleteUnusedFilesFromPDBCache: %i" ), DaysToDeleteUnusedFilesFromPDBCache );
}
}
void FPDBCache::InitializePDBCache()
{
const double StartTime = FPlatformTime::Seconds();
IFileManager::Get().MakeDirectory( *PDBCachePath, true );
TArray<FString> PDBCacheEntryDirectories;
IFileManager::Get().FindFiles( PDBCacheEntryDirectories, *(PDBCachePath / TEXT( "*" )), false, true );
for( const auto& Directory : PDBCacheEntryDirectories )
{
FPDBCacheEntryPtr Entry = ReadPDBCacheEntry( Directory );
if (Entry.IsValid())
{
PDBCacheEntries.Add( Directory, Entry.ToSharedRef() );
}
}
SortPDBCache();
const double TotalTime = FPlatformTime::Seconds() - StartTime;
UE_LOG( LogCrashDebugHelper, Log, TEXT( "PDB Cache initialized in %.2f ms" ), TotalTime*1000.0f );
UE_LOG( LogCrashDebugHelper, Log, TEXT( "Found %i entries which occupy %i GBs" ), PDBCacheEntries.Num(), GetPDBCacheSizeGB() );
}
void FPDBCache::CleanPDBCache( int32 DaysToDelete, int32 NumberOfGBsToBeCleaned /*= 0 */ )
{
// Not very efficient, but should do the trick.
// Revisit it later.
const double StartTime = FPlatformTime::Seconds();
TSet<FString> EntriesToBeRemoved;
// Find all outdated PDB Cache entries and mark them for removal.
const double DaysToDeleteAsSeconds = FTimespan( DaysToDelete, 0, 0, 0 ).GetTotalSeconds();
int32 NumGBsCleaned = 0;
for( const auto& It : PDBCacheEntries )
{
const FPDBCacheEntryRef& Entry = It.Value;
const FString EntryDirectory = PDBCachePath / Entry->Directory;
const FString EntryTimeStampFilename = EntryDirectory / PDBTimeStampFile;
const double EntryFileAge = IFileManager::Get().GetFileAgeSeconds( *EntryTimeStampFilename );
if( EntryFileAge > DaysToDeleteAsSeconds )
{
EntriesToBeRemoved.Add( Entry->Directory );
NumGBsCleaned += Entry->SizeGB;
}
}
if( NumberOfGBsToBeCleaned > 0 && NumGBsCleaned < NumberOfGBsToBeCleaned )
{
// Do the second pass if we need to remove more PDB Cache entries due to the free disk space restriction.
for( const auto& It : PDBCacheEntries )
{
const FPDBCacheEntryRef& Entry = It.Value;
if( !EntriesToBeRemoved.Contains( Entry->Directory ) )
{
EntriesToBeRemoved.Add( Entry->Directory );
NumGBsCleaned += Entry->SizeGB;
if( NumGBsCleaned > NumberOfGBsToBeCleaned )
{
// Break the loop, we are done.
break;
}
}
}
}
// Remove all marked PDB Cache entries.
for( const auto& EntryDirectory : EntriesToBeRemoved )
{
RemovePDBCacheEntry( EntryDirectory );
}
const double TotalTime = FPlatformTime::Seconds() - StartTime;
UE_LOG( LogCrashDebugHelper, Log, TEXT( "PDB Cache cleaned %i GBs in %.2f ms" ), NumGBsCleaned, TotalTime*1000.0f );
}
FPDBCacheEntryRef FPDBCache::CreateAndAddPDBCacheEntry( const FString& OriginalLabelName, const FString& DepotName, const TArray<FString>& FilesToBeCached )
{
const FString CleanedLabelName = EscapePath( OriginalLabelName );
const FString EntryDirectory = PDBCachePath / CleanedLabelName;
const FString EntryTimeStampFilename = EntryDirectory / PDBTimeStampFile;
const FString LocalDepotDir = EscapePath( DepotRoot / DepotName );
UE_LOG( LogCrashDebugHelper, Warning, TEXT( "PDB Cache entry: %s is being copied from: %s, it will take some time" ), *CleanedLabelName, *OriginalLabelName );
for( const auto& Filename : FilesToBeCached )
{
const FString SourceDirectoryWithSearch = Filename.Replace( *DepotName, *LocalDepotDir );
TArray<FString> MatchedFiles;
IFileManager::Get().FindFiles( MatchedFiles, *SourceDirectoryWithSearch, true, false );
for( const auto& MatchedFilename : MatchedFiles )
{
const FString SrcFilename = FPaths::GetPath( SourceDirectoryWithSearch ) / MatchedFilename;
const FString DestFilename = EntryDirectory / SrcFilename.Replace( *LocalDepotDir, TEXT( "" ) );
IFileManager::Get().Copy( *DestFilename, *SrcFilename );
}
}
TArray<FString> CachedFiles;
IFileManager::Get().FindFilesRecursive( CachedFiles, *EntryDirectory, TEXT( "*.*" ), true, false );
// Calculate the size of this PDB Cache entry.
int64 TotalSize = 0;
for( const auto& Filename : CachedFiles )
{
const int64 FileSize = IFileManager::Get().FileSize( *Filename );
TotalSize += FileSize;
}
// Round-up the size.
int32 SizeGB = (int32)FMath::DivideAndRoundUp( TotalSize, (int64)NUM_BYTES_PER_GB );
FPDBCacheEntryRef NewCacheEntry = MakeShareable( new FPDBCacheEntry( CachedFiles, CleanedLabelName, FDateTime::Now(), SizeGB ) );
// Verify there is an entry timestamp file, write the size of a PDB cache to avoid time consuming FindFilesRecursive during initialization.
TUniquePtr<FArchive> FileWriter( IFileManager::Get().CreateFileWriter( *EntryTimeStampFilename ) );
UE_CLOG( !FileWriter, LogCrashDebugHelper, Fatal, TEXT( "Couldn't save the timestamp for a file: %s" ), *EntryTimeStampFilename );
*FileWriter << *NewCacheEntry;
PDBCacheEntries.Add( CleanedLabelName, NewCacheEntry );
SortPDBCache();
return NewCacheEntry;
}
FPDBCacheEntryRef FPDBCache::CreateAndAddPDBCacheEntryMixed( const FString& ProductVersion, const TMap<FString, FString>& FilesToBeCached )
{
// Enable MDD to parse all minidumps regardless the branch, to fix the issue with the missing executables on the P4 due to the build system changes.
const FString EntryDirectory = PDBCachePath / ProductVersion;
const FString EntryTimeStampFilename = EntryDirectory / PDBTimeStampFile;
UE_LOG( LogCrashDebugHelper, Warning, TEXT( "PDB Cache entry: %s is being created from %i files, it will take some time" ), *ProductVersion, FilesToBeCached.Num() );
for( const auto& It : FilesToBeCached )
{
const FString& SrcFilename = It.Key;
const FString DestFilename = EntryDirectory / It.Value;
IFileManager::Get().Copy( *DestFilename, *SrcFilename );
}
TArray<FString> CachedFiles;
IFileManager::Get().FindFilesRecursive( CachedFiles, *EntryDirectory, TEXT( "*.*" ), true, false );
// Calculate the size of this PDB Cache entry.
int64 TotalSize = 0;
for( const auto& Filename : CachedFiles )
{
const int64 FileSize = IFileManager::Get().FileSize( *Filename );
TotalSize += FileSize;
}
// Round-up the size.
int32 SizeGB = (int32)FMath::DivideAndRoundUp( TotalSize, (int64)NUM_BYTES_PER_GB );
FPDBCacheEntryRef NewCacheEntry = MakeShareable( new FPDBCacheEntry( CachedFiles, ProductVersion, FDateTime::Now(), SizeGB ) );
// Verify there is an entry timestamp file, write the size of a PDB cache to avoid time consuming FindFilesRecursive during initialization.
TUniquePtr<FArchive> FileWriter( IFileManager::Get().CreateFileWriter( *EntryTimeStampFilename ) );
UE_CLOG( !FileWriter, LogCrashDebugHelper, Fatal, TEXT( "Couldn't save the timestamp for a file: %s" ), *EntryTimeStampFilename );
*FileWriter << *NewCacheEntry;
PDBCacheEntries.Add( ProductVersion, NewCacheEntry );
SortPDBCache();
return NewCacheEntry;
}
FPDBCacheEntryPtr FPDBCache::ReadPDBCacheEntry( const FString& Directory )
{
const FString EntryDirectory = PDBCachePath / Directory;
const FString EntryTimeStampFilenameNoMeta = EntryDirectory / PDBTimeStampFileNoMeta;
const FString EntryTimeStampFilename = EntryDirectory / PDBTimeStampFile;
// Verify there is an entry timestamp file.
const FDateTime LastAccessTimeNoMeta = IFileManager::Get().GetTimeStamp( *EntryTimeStampFilenameNoMeta );
const FDateTime LastAccessTime = IFileManager::Get().GetTimeStamp( *EntryTimeStampFilename );
FPDBCacheEntryPtr NewEntry;
if( LastAccessTime != FDateTime::MinValue() )
{
// Read the metadata
TUniquePtr<FArchive> FileReader( IFileManager::Get().CreateFileReader( *EntryTimeStampFilename ) );
NewEntry = MakeShareable( new FPDBCacheEntry( LastAccessTime ) );
*FileReader << *NewEntry;
}
else if( LastAccessTimeNoMeta != FDateTime::MinValue() )
{
// Calculate the size of this PDB Cache entry and update to the new version.
TArray<FString> PDBFiles;
IFileManager::Get().FindFilesRecursive( PDBFiles, *EntryDirectory, TEXT( "*.*" ), true, false );
// Calculate the size of this PDB Cache entry.
int64 TotalSize = 0;
for( const auto& Filename : PDBFiles )
{
const int64 FileSize = IFileManager::Get().FileSize( *Filename );
TotalSize += FileSize;
}
// Round-up the size.
const int32 SizeGB = (int32)FMath::DivideAndRoundUp( TotalSize, (int64)NUM_BYTES_PER_GB );
NewEntry = MakeShareable( new FPDBCacheEntry( PDBFiles, Directory, LastAccessTimeNoMeta, SizeGB ) );
// Remove the old file and save the metadata.
TUniquePtr<FArchive> FileWriter( IFileManager::Get().CreateFileWriter( *EntryTimeStampFilename ) );
*FileWriter << *NewEntry;
IFileManager::Get().Delete( *EntryTimeStampFilenameNoMeta );
}
else
{
// Something wrong.
ensureMsgf( 0, TEXT( "Invalid symbol cache entry: %s" ), *EntryDirectory );
}
return NewEntry;
}
void FPDBCache::TouchPDBCacheEntry( const FString& Directory )
{
const FString EntryDirectory = PDBCachePath / Directory;
const FString EntryTimeStampFilename = EntryDirectory / PDBTimeStampFile;
FPDBCacheEntryRef& Entry = PDBCacheEntries.FindChecked( Directory );
Entry->SetLastAccessTimeToNow();
const bool bResult = IFileManager::Get().SetTimeStamp( *EntryTimeStampFilename, Entry->LastAccessTime );
SortPDBCache();
}
void FPDBCache::RemovePDBCacheEntry( const FString& Directory )
{
const double StartTime = FPlatformTime::Seconds();
const FString EntryDirectory = PDBCachePath / Directory;
FPDBCacheEntryRef& Entry = PDBCacheEntries.FindChecked( Directory );
IFileManager::Get().DeleteDirectory( *EntryDirectory, true, true );
const double TotalTime = FPlatformTime::Seconds() - StartTime;
UE_LOG( LogCrashDebugHelper, Warning, TEXT( "PDB Cache entry %s removed in %.2f ms, restored %i GBs" ), *Directory, TotalTime*1000.0f, Entry->SizeGB );
PDBCacheEntries.Remove( Directory );
}
FPDBCacheEntryRef FPDBCache::FindAndTouchPDBCacheEntry( const FString& PathOrLabel )
{
FPDBCacheEntryRef CacheEntry = PDBCacheEntries.FindChecked( EscapePath( PathOrLabel ) );
TouchPDBCacheEntry( CacheEntry->Directory );
return CacheEntry;
}

View File

@@ -599,14 +599,6 @@ bool FCrashDebugHelperIOS::ParseCrashDump(const FString& InCrashDumpName, FCrash
bool FCrashDebugHelperIOS::CreateMinidumpDiagnosticReport( const FString& InCrashDumpName )
{
bool bOK = false;
const bool bSyncSymbols = FParse::Param( FCommandLine::Get(), TEXT( "SyncSymbols" ) );
const bool bAnnotate = FParse::Param( FCommandLine::Get(), TEXT( "Annotate" ) );
const bool bUseSCC = bSyncSymbols || bAnnotate;
if( bUseSCC )
{
InitSourceControl( false );
}
FString CrashDump;
if ( FFileHelper::LoadFileToString( CrashDump, *InCrashDumpName ) )
@@ -645,14 +637,6 @@ bool FCrashDebugHelperIOS::CreateMinidumpDiagnosticReport( const FString& InCras
if(Result == 5 && Branch.Len() > 0)
{
CrashInfo.LabelName = Branch;
if( bSyncSymbols )
{
FindSymbolsAndBinariesStorage();
bool bPDBCacheEntryValid = false;
SyncModules(bPDBCacheEntryValid);
}
}
Result = ParseOS(*CrashDump, CrashInfo.SystemInfo.OSMajor, CrashInfo.SystemInfo.OSMinor, CrashInfo.SystemInfo.OSBuild, CrashInfo.SystemInfo.OSRevision);
@@ -767,24 +751,8 @@ bool FCrashDebugHelperIOS::CreateMinidumpDiagnosticReport( const FString& InCras
CrashInfo.SourceFile = ExtractRelativePath( TEXT( "source" ), *FileName );
CrashInfo.SourceLineNumber = LineNumber;
if( bSyncSymbols && CrashInfo.BuiltFromCL > 0 )
{
UE_LOG( LogCrashDebugHelper, Log, TEXT( "Using CL %i to sync crash source file" ), CrashInfo.BuiltFromCL );
SyncSourceFile();
}
// Try to annotate the file if requested
bool bAnnotationSuccessful = false;
if( bAnnotate )
{
bAnnotationSuccessful = AddAnnotatedSourceToReport();
}
// If annotation is not requested, or failed, add the standard source context
if( !bAnnotationSuccessful )
{
AddSourceToReport();
}
// Add the standard source context
AddSourceToReport();
}
}
@@ -811,10 +779,5 @@ bool FCrashDebugHelperIOS::CreateMinidumpDiagnosticReport( const FString& InCras
}
}
if( bUseSCC )
{
ShutdownSourceControl();
}
return bOK;
}

View File

@@ -606,14 +606,6 @@ bool FCrashDebugHelperMac::ParseCrashDump(const FString& InCrashDumpName, FCrash
bool FCrashDebugHelperMac::CreateMinidumpDiagnosticReport( const FString& InCrashDumpName )
{
bool bOK = false;
const bool bSyncSymbols = FParse::Param( FCommandLine::Get(), TEXT( "SyncSymbols" ) );
const bool bAnnotate = FParse::Param( FCommandLine::Get(), TEXT( "Annotate" ) );
const bool bUseSCC = bSyncSymbols || bAnnotate;
if( bUseSCC )
{
InitSourceControl( false );
}
FString CrashDump;
@@ -671,14 +663,6 @@ bool FCrashDebugHelperMac::CreateMinidumpDiagnosticReport( const FString& InCras
if(Result == 5 && Branch.Len() > 0)
{
CrashInfo.LabelName = Branch;
if( bSyncSymbols )
{
FindSymbolsAndBinariesStorage();
bool bPDBCacheEntryValid = false;
SyncModules(bPDBCacheEntryValid);
}
}
Result = ParseOS(*CrashDump, CrashInfo.SystemInfo.OSMajor, CrashInfo.SystemInfo.OSMinor, CrashInfo.SystemInfo.OSBuild, CrashInfo.SystemInfo.OSRevision);
@@ -793,24 +777,8 @@ bool FCrashDebugHelperMac::CreateMinidumpDiagnosticReport( const FString& InCras
CrashInfo.SourceFile = ExtractRelativePath( TEXT( "source" ), *FileName );
CrashInfo.SourceLineNumber = LineNumber;
if( bSyncSymbols && CrashInfo.BuiltFromCL > 0 )
{
UE_LOG( LogCrashDebugHelper, Log, TEXT( "Using CL %i to sync crash source file" ), CrashInfo.BuiltFromCL );
SyncSourceFile();
}
// Try to annotate the file if requested
bool bAnnotationSuccessful = false;
if( bAnnotate )
{
bAnnotationSuccessful = AddAnnotatedSourceToReport();
}
// If annotation is not requested, or failed, add the standard source context
if( !bAnnotationSuccessful )
{
AddSourceToReport();
}
// Add the standard source context
AddSourceToReport();
}
}
@@ -837,10 +805,5 @@ bool FCrashDebugHelperMac::CreateMinidumpDiagnosticReport( const FString& InCras
}
}
if( bUseSCC )
{
ShutdownSourceControl();
}
return bOK;
}

View File

@@ -7,7 +7,6 @@
#include "Misc/CommandLine.h"
#include "Misc/EngineVersion.h"
#include "ISourceControlModule.h"
#include "Windows/WindowsHWrapper.h"
#include "Windows/AllowWindowsPlatformTypes.h"
@@ -15,16 +14,6 @@
bool FCrashDebugHelperWindows::CreateMinidumpDiagnosticReport( const FString& InCrashDumpFilename )
{
const bool bSyncSymbols = FParse::Param( FCommandLine::Get(), TEXT( "SyncSymbols" ) );
const bool bAnnotate = FParse::Param( FCommandLine::Get(), TEXT( "Annotate" ) );
const bool bNoTrim = FParse::Param(FCommandLine::Get(), TEXT("NoTrimCallstack"));
const bool bUseSCC = bSyncSymbols || bAnnotate;
if( bUseSCC )
{
InitSourceControl( false );
}
FWindowsPlatformStackWalkExt WindowsStackWalkExt( CrashInfo );
const bool bReady = WindowsStackWalkExt.InitStackWalking();
@@ -39,72 +28,31 @@ bool FCrashDebugHelperWindows::CreateMinidumpDiagnosticReport( const FString& In
WindowsStackWalkExt.GetExeFileVersionAndModuleList(ExeFileVersion);
// Init Symbols
bool bInitSymbols = false;
if (CrashInfo.bMutexPDBCache && !CrashInfo.PDBCacheLockName.IsEmpty())
{
// Scoped lock
UE_LOG(LogCrashDebugHelper, Log, TEXT("Locking for InitSymbols()"));
FSystemWideCriticalSection PDBCacheLock(CrashInfo.PDBCacheLockName, FTimespan::FromMinutes(10.0));
if (PDBCacheLock.IsValid())
{
bInitSymbols = InitSymbols(WindowsStackWalkExt, bSyncSymbols);
}
UE_LOG(LogCrashDebugHelper, Log, TEXT("Unlocking after InitSymbols()"));
}
else
{
bInitSymbols = InitSymbols(WindowsStackWalkExt, bSyncSymbols);
}
WindowsStackWalkExt.InitSymbols();
if (bInitSymbols)
{
// Get all the info we should ever need about the modules
WindowsStackWalkExt.GetModuleInfoDetailed();
// Set the symbol path based on the loaded modules
WindowsStackWalkExt.SetSymbolPathsFromModules();
// Get info about the system that created the minidump
WindowsStackWalkExt.GetSystemInfo();
// Get all the info we should ever need about the modules
WindowsStackWalkExt.GetModuleInfoDetailed();
// Get all the thread info
WindowsStackWalkExt.GetThreadInfo();
// Get info about the system that created the minidump
WindowsStackWalkExt.GetSystemInfo();
// Get exception info
WindowsStackWalkExt.GetExceptionInfo();
// Get all the thread info
WindowsStackWalkExt.GetThreadInfo();
// Get the callstacks for each thread
WindowsStackWalkExt.GetCallstacks();
// Get exception info
WindowsStackWalkExt.GetExceptionInfo();
// Sync the source file where the crash occurred
if (CrashInfo.SourceFile.Len() > 0)
{
const bool bMutexSourceSync = FParse::Param(FCommandLine::Get(), TEXT("MutexSourceSync"));
FString SourceSyncLockName;
FParse::Value(FCommandLine::Get(), TEXT("SourceSyncLock="), SourceSyncLockName);
// Get the callstacks for each thread
WindowsStackWalkExt.GetCallstacks();
if (bMutexSourceSync && !SourceSyncLockName.IsEmpty())
{
// Scoped lock
UE_LOG(LogCrashDebugHelper, Log, TEXT("Locking for SyncAndReadSourceFile()"));
const FTimespan GlobalLockWaitTimeout = FTimespan::FromSeconds(30.0);
FSystemWideCriticalSection SyncSourceLock(SourceSyncLockName, GlobalLockWaitTimeout);
if (SyncSourceLock.IsValid())
{
SyncAndReadSourceFile(bSyncSymbols, bAnnotate, CrashInfo.BuiltFromCL);
}
UE_LOG(LogCrashDebugHelper, Log, TEXT("Unlocking after SyncAndReadSourceFile()"));
}
else
{
SyncAndReadSourceFile(bSyncSymbols, bAnnotate, CrashInfo.BuiltFromCL);
}
}
// Add the source file where the crash occurred
AddSourceToReport();
// Set the result
bResult = true;
}
else
{
UE_LOG(LogCrashDebugHelper, Warning, TEXT("InitSymbols failed"));
}
// Set the result
bResult = true;
}
else
{
@@ -116,70 +64,7 @@ bool FCrashDebugHelperWindows::CreateMinidumpDiagnosticReport( const FString& In
UE_LOG( LogCrashDebugHelper, Warning, TEXT( "Failed to open crash dump file: %s" ), *InCrashDumpFilename );
}
if( bUseSCC )
{
ShutdownSourceControl();
}
return bResult;
}
bool FCrashDebugHelperWindows::InitSymbols(FWindowsPlatformStackWalkExt& WindowsStackWalkExt, bool bSyncSymbols)
{
// CrashInfo now contains a changelist to lookup a label for
if (bSyncSymbols)
{
FindSymbolsAndBinariesStorage();
bool bPDBCacheEntryValid = false;
const bool bSynced = SyncModules(bPDBCacheEntryValid);
// Without symbols we can't decode the provided minidump.
if (!bSynced)
{
return false;
}
if (!bPDBCacheEntryValid)
{
// early-out option
const bool bForceUsePDBCache = FParse::Param(FCommandLine::Get(), TEXT("ForceUsePDBCache"));
if (bForceUsePDBCache)
{
UE_LOG(LogCrashDebugHelper, Log, TEXT("No cached symbols available. Exiting due to -ForceUsePDBCache."));
return false;
}
}
}
// Initialise the symbol options
WindowsStackWalkExt.InitSymbols();
// Set the symbol path based on the loaded modules
WindowsStackWalkExt.SetSymbolPathsFromModules();
return true;
}
void FCrashDebugHelperWindows::SyncAndReadSourceFile(bool bSyncSymbols, bool bAnnotate, int32 BuiltFromCL)
{
if (bSyncSymbols && BuiltFromCL > 0)
{
UE_LOG(LogCrashDebugHelper, Log, TEXT("Using CL %i to sync crash source file"), BuiltFromCL);
SyncSourceFile();
}
// Try to annotate the file if requested
bool bAnnotationSuccessful = false;
if (bAnnotate)
{
bAnnotationSuccessful = AddAnnotatedSourceToReport();
}
// If annotation is not requested, or failed, add the standard source context
if (!bAnnotationSuccessful)
{
AddSourceToReport();
}
}
#include "Windows/HideWindowsPlatformTypes.h"

View File

@@ -15,10 +15,6 @@ public:
* @return bool true if successful, false if not
*/
virtual bool CreateMinidumpDiagnosticReport( const FString& InCrashDumpName ) override;
private:
bool InitSymbols(struct FWindowsPlatformStackWalkExt& WindowsStackWalkExt, bool bSyncSymbols);
void SyncAndReadSourceFile(bool bSyncSymbols, bool bAnnotate, int32 BuiltFromCL);
};
typedef FCrashDebugHelperWindows FCrashDebugHelper;

View File

@@ -1,10 +1,10 @@
// Copyright 1998-2019 Epic Games, Inc. All Rights Reserved.
#include "WindowsPlatformStackWalkExt.h"
#include "CrashDebugHelper.h"
#include "CrashDebugHelperPrivate.h"
#include "GenericPlatform/GenericPlatformStackWalk.h"
#include "GenericPlatform/GenericPlatformCrashContext.h"
#include "CrashDebugPDBCache.h"
#include "Misc/Parse.h"
#include "Misc/CommandLine.h"
#include "Misc/MemStack.h"
@@ -179,59 +179,33 @@ void FWindowsPlatformStackWalkExt::GetExeFileVersionAndModuleList( FCrashModuleI
void FWindowsPlatformStackWalkExt::SetSymbolPathsFromModules()
{
const bool bUseCachedData = CrashInfo.PDBCacheEntry.IsValid();
FString CombinedPath = TEXT( "" );
// Use symbol cache from command line
FString DebugSymbols;
if (FParse::Value(FCommandLine::Get(), TEXT("DebugSymbols="), DebugSymbols))
{
CombinedPath += TEXT("SRV*");
CombinedPath += DebugSymbols;
CombinedPath += TEXT(";");
}
// For externally launched minidump diagnostics.
if( bUseCachedData )
{
TSet<FString> SymbolPaths;
for( const auto& Filename : CrashInfo.PDBCacheEntry->Files )
// Use symbol cache from command line
FString DebugSymbols;
if (FParse::Value(FCommandLine::Get(), TEXT("DebugSymbols="), DebugSymbols))
{
const FString SymbolPath = FPaths::GetPath( Filename );
if( SymbolPath.Len() > 0 )
{
SymbolPaths.Add( SymbolPath );
}
CombinedPath += TEXT("SRV*");
CombinedPath += DebugSymbols;
CombinedPath += TEXT(";");
}
for( const auto& SymbolPath : SymbolPaths )
{
CombinedPath += SymbolPath;
CombinedPath += TEXT( ";" );
}
// Set the symbol path
Symbol->SetImagePathWide( *CombinedPath );
Symbol->SetSymbolPathWide( *CombinedPath );
}
// For locally launched minidump diagnostics.
else
{
// Iterate over all loaded modules.
TSet<FString> SymbolPaths;
for (const auto& Filename : CrashInfo.ModuleNames)
{
const FString Path = FPaths::GetPath( Filename );
if( Path.Len() > 0 )
const FString Path = FPaths::GetPath(Filename);
if (Path.Len() > 0)
{
SymbolPaths.Add( Path );
SymbolPaths.Add(Path);
}
}
for( const auto& SymbolPath : SymbolPaths )
for (const auto& SymbolPath : SymbolPaths)
{
CombinedPath += SymbolPath;
CombinedPath += TEXT( ";" );
CombinedPath += TEXT(";");
}
#if ALLOW_UNREAL_ACCESS_TO_NT_SYMBOL_PATH
@@ -242,12 +216,12 @@ void FWindowsPlatformStackWalkExt::SetSymbolPathsFromModules()
CombinedPath += ";";
}
#endif
// Set the symbol path
Symbol->SetImagePathWide( *CombinedPath );
Symbol->SetSymbolPathWide( *CombinedPath );
}
// Set the symbol path
Symbol->SetImagePathWide( *CombinedPath );
Symbol->SetSymbolPathWide( *CombinedPath );
// Add in syncing of the Microsoft symbol servers if requested
if( FParse::Param( FCommandLine::Get(), TEXT( "SyncMicrosoftSymbols" ) ) )
{

View File

@@ -7,7 +7,6 @@
#include "Templates/SharedPointer.h"
class FArchive;
struct FPDBCacheEntry;
enum EProcessorArchitecture
{
@@ -27,22 +26,12 @@ public:
FString Name;
FString Extension;
uint64 BaseOfImage;
uint32 SizeOfImage;
uint16 Major;
uint16 Minor;
uint16 Patch;
uint16 Revision;
FCrashModuleInfo()
: BaseOfImage( 0 )
, SizeOfImage( 0 )
, Major( 0 )
, Minor( 0 )
, Patch( 0 )
, Revision( 0 )
{
}
uint64 BaseOfImage = 0;
uint32 SizeOfImage = 0;
uint16 Major = 0;
uint16 Minor = 0;
uint16 Patch = 0;
uint16 Revision = 0;
};
/**
@@ -53,20 +42,10 @@ class FCrashThreadInfo
public:
FString Report;
uint32 ThreadId;
uint32 SuspendCount;
uint32 ThreadId = 0;
uint32 SuspendCount = 0;
TArray<uint64> CallStack;
FCrashThreadInfo()
: ThreadId( 0 )
, SuspendCount( 0 )
{
}
~FCrashThreadInfo()
{
}
};
/**
@@ -77,23 +56,12 @@ class FCrashExceptionInfo
public:
FString Report;
uint32 ProcessId;
uint32 ThreadId;
uint32 Code;
uint32 ProcessId = 0;
uint32 ThreadId = 0;
uint32 Code = 0;
FString ExceptionString;
TArray<FString> CallStackString;
FCrashExceptionInfo()
: ProcessId( 0 )
, ThreadId( 0 )
, Code( 0 )
{
}
~FCrashExceptionInfo()
{
}
};
/**
@@ -104,23 +72,13 @@ class FCrashSystemInfo
public:
FString Report;
EProcessorArchitecture ProcessorArchitecture;
uint32 ProcessorCount;
EProcessorArchitecture ProcessorArchitecture = PA_UNKNOWN;
uint32 ProcessorCount = 0;
uint16 OSMajor;
uint16 OSMinor;
uint16 OSBuild;
uint16 OSRevision;
FCrashSystemInfo()
: ProcessorArchitecture( PA_UNKNOWN )
, ProcessorCount( 0 )
, OSMajor( 0 )
, OSMinor( 0 )
, OSBuild( 0 )
, OSRevision( 0 )
{
}
uint16 OSMajor = 0;
uint16 OSMinor = 0;
uint16 OSBuild = 0;
uint16 OSRevision = 0;
};
// #TODO 2015-07-24 Refactor
@@ -147,7 +105,7 @@ public:
FString BuildVersion;
/** CL built from. */
int32 BuiltFromCL;
int32 BuiltFromCL = INVALID_CHANGELIST;
/** The label the describes the executables and symbols. */
FString LabelName;
@@ -159,7 +117,7 @@ public:
FString SymbolsPath;
FString SourceFile;
uint32 SourceLineNumber;
uint32 SourceLineNumber = 0;
TArray<FString> SourceContext;
/** Only modules names, retrieved from the minidump file. */
@@ -170,28 +128,9 @@ public:
TArray<FCrashThreadInfo> Threads;
TArray<FCrashModuleInfo> Modules;
/** Shared pointer to the PDB Cache entry, if valid contains all information about synced PDBs. */
TSharedPtr<FPDBCacheEntry> PDBCacheEntry;
FString PlatformName;
FString PlatformVariantName;
/** If we are using a PDBCache, this is whether we should use a system-wide lock to access it. */
bool bMutexPDBCache;
/** If we are using a PDBCache, this is the name of the system-wide lock we should use to access it. */
FString PDBCacheLockName;
FCrashInfo()
: BuiltFromCL( INVALID_CHANGELIST )
, SourceLineNumber( 0 )
{
}
~FCrashInfo()
{
}
/**
* Generate a report for the crash in the requested path
*/
@@ -206,20 +145,15 @@ private:
/**
* Convert the processor architecture to a human readable string
*/
const TCHAR* GetProcessorArchitecture( EProcessorArchitecture PA );
/**
* Calculate the byte size of a UTF-8 string
*/
int64 StringSize( const ANSICHAR* Line );
static const TCHAR* GetProcessorArchitecture( EProcessorArchitecture PA );
/**
* Write a line of UTF-8 to a file
* Write a line as UTF-8 to a file
*/
void WriteLine( FArchive* ReportFile, const ANSICHAR* Line = NULL );
static void WriteLine(FArchive* ReportFile, const TCHAR* Line = nullptr);
static void WriteLine(FArchive* ReportFile, const FString& Line);
};
/** Helper structure for tracking crash debug information */
struct FCrashDebugInfo
{
@@ -237,28 +171,8 @@ struct FCrashDebugInfo
class CRASHDEBUGHELPER_API ICrashDebugHelper
{
public:
/** Replaces %DEPOT_INDEX% with the command line DepotIndex in the specified path. */
static void SetDepotIndex( FString& PathToChange );
protected:
/**
* Pattern to search in source control for the label.
* This somehow works for older crashes, before 4.2 and for the newest one,
* bur for the newest we also need to look for the executables on the network drive.
* This may change in future.
*/
FString SourceControlBuildLabelPattern;
/** Indicates that the crash handler is ready to do work */
bool bInitialized;
public:
/** A platform independent representation of a crash */
FCrashInfo CrashInfo;
/** Virtual destructor */
virtual ~ICrashDebugHelper()
{}
virtual ~ICrashDebugHelper() = default;
/**
* Initialize the helper
@@ -294,35 +208,12 @@ public:
return false;
}
/**
* Sync the branch root relative file names to the requested label
*
* @param bOutPDBCacheEntryValid Returns whether the PDB cache entry was found or created and whether it contains files.
*
* @return bool true if successful, false if not
*/
virtual bool SyncModules(bool& bOutPDBCacheEntryValid);
/**
* Sync a single source file to the requested CL.
*/
virtual bool SyncSourceFile();
/**
* Extract lines from a source file, and add to the crash report.
*/
virtual void AddSourceToReport();
/**
* Extract annotated lines from a source file stored in Perforce, and add to the crash report.
*/
virtual bool AddAnnotatedSourceToReport();
protected:
/** Finds the storage of the symbols and the executables for the specified changelist and the depot name, it can be Perforce, network drive or stored locally. */
void FindSymbolsAndBinariesStorage();
/**
* Load the given ANSI text file to an array of strings - one FString per line of the file.
* Intended for use in simple text parsing actions
@@ -334,6 +225,10 @@ protected:
bool ReadSourceFile( TArray<FString>& OutStrings );
public:
bool InitSourceControl(bool bShowLogin);
void ShutdownSourceControl();
/** A platform independent representation of a crash */
FCrashInfo CrashInfo;
protected:
/** Indicates that the crash handler is ready to do work */
bool bInitialized = false;
};

View File

@@ -1,247 +0,0 @@
// Copyright 1998-2019 Epic Games, Inc. All Rights Reserved.
#pragma once
#include "CoreMinimal.h"
#include "Misc/DateTime.h"
#include "Containers/UnrealString.h"
#include "Containers/Map.h"
#include "CrashDebugHelper.h"
class FArchive;
typedef TSharedRef<FPDBCacheEntry> FPDBCacheEntryRef;
typedef TSharedPtr<FPDBCacheEntry> FPDBCacheEntryPtr;
/** Helper struct that holds various information about one PDB Cache entry. */
struct FPDBCacheEntry
{
/** Default constructor. */
FPDBCacheEntry( const FDateTime InLastAccessTime )
: LastAccessTime( InLastAccessTime )
, SizeGB( 0 )
{}
/** Initialization constructor. */
FPDBCacheEntry( const TArray<FString>& InFiles, const FString& InDirectory, const FDateTime InLastAccessTime, const int32 InSizeGB )
: Files( InFiles )
, Directory( InDirectory )
, LastAccessTime( InLastAccessTime )
, SizeGB( InSizeGB )
{}
void SetLastAccessTimeToNow()
{
LastAccessTime = FDateTime::UtcNow();
}
/** Paths to files associated with this PDB Cache entry. */
TArray<FString> Files;
/** The path associated with this PDB Cache entry. */
FString Directory;
/** Last access time, changed every time this PDB cache entry is used. */
FDateTime LastAccessTime;
/** Size of the cache entry, in GBs. Rounded-up. */
const int32 SizeGB;
/**
* Serializer.
*/
friend FArchive& operator<<(FArchive& Ar, FPDBCacheEntry& Entry)
{
return Ar << Entry.Files << (FString&)Entry.Directory << (int32&)Entry.SizeGB;
}
};
struct FPDBCacheEntryByAccessTime
{
FORCEINLINE bool operator()( const FPDBCacheEntryRef& A, const FPDBCacheEntryRef& B ) const
{
return A->LastAccessTime.GetTicks() < B->LastAccessTime.GetTicks();
}
};
/** Implements PDB cache. */
struct FPDBCache
{
protected:
// Defaults for the PDB cache.
enum
{
/** Size of the PDB cache, in GBs. */
PDB_CACHE_SIZE_GB = 300,
MIN_FREESPACE_GB = 64,
/** Age of file when it should be deleted from the PDB cache. */
DAYS_TO_DELETE_UNUSED_FILES = 14,
/**
* Number of iterations inside the CleanPDBCache method.
* Mostly to verify that MinDiskFreeSpaceGB requirement is met.
*/
CLEAN_PDBCACHE_NUM_ITERATIONS = 2,
/** Number of bytes per one gigabyte. */
NUM_BYTES_PER_GB = 1024 * 1024 * 1024
};
/** Dummy file used to read/set the file timestamp. */
static const TCHAR* PDBTimeStampFileNoMeta;
/** Data file used to read/set the file timestamp, contains all metadata. */
static const TCHAR* PDBTimeStampFile;
/** Map of the PDB Cache entries. */
TMap<FString, FPDBCacheEntryRef> PDBCacheEntries;
/** Path to the folder where the PDB cache is stored. */
FString PDBCachePath;
/** Depot root. */
FString DepotRoot;
/** Age of file when it should be deleted from the PDB cache. */
int32 DaysToDeleteUnusedFilesFromPDBCache;
/** Size of the PDB cache, in GBs. */
int32 PDBCacheSizeGB;
/**
* Minimum disk free space that should be available on the disk where the PDB cache is stored, in GBs.
* Minidump diagnostics runs usually on the same drive as the crash reports drive, so we need to leave some space for the crash receiver.
* If there is not enough disk free space, we will run the clean-up process.
*/
int32 MinDiskFreeSpaceGB;
/** Whether to use the PDB cache. */
bool bUsePDBCache;
public:
/** Default constructor. */
FPDBCache()
: DaysToDeleteUnusedFilesFromPDBCache( DAYS_TO_DELETE_UNUSED_FILES )
, PDBCacheSizeGB( PDB_CACHE_SIZE_GB )
, MinDiskFreeSpaceGB( MIN_FREESPACE_GB )
, bUsePDBCache( false )
{}
/** Basic initialization, reading config etc.. */
void Init();
/**
* @return whether to use the PDB cache.
*/
bool UsePDBCache() const
{
return bUsePDBCache;
}
/** @return the path the depot root. */
const FString& GetDepotRoot() const
{
return DepotRoot;
}
/** Accesses the singleton. */
CRASHDEBUGHELPER_API static FPDBCache& Get()
{
static FPDBCache Instance;
return Instance;
}
/**
* @return true, if the PDB Cache contains the specified label.
*/
bool ContainsPDBCacheEntry( const FString& PathOrLabel ) const
{
return PDBCacheEntries.Contains( EscapePath( PathOrLabel ) );
}
/**
* Touches a PDB Cache entry by updating the timestamp.
*/
void TouchPDBCacheEntry( const FString& Directory );
/**
* @return a PDB Cache entry for the specified label and touches it at the same time
*/
FPDBCacheEntryRef FindAndTouchPDBCacheEntry( const FString& PathOrLabel );
/**
* Creates a new PDB Cache entry, initializes it and adds to the database.
*/
FPDBCacheEntryRef CreateAndAddPDBCacheEntry( const FString& OriginalLabelName, const FString& DepotName, const TArray<FString>& FilesToBeCached );
/**
* Creates a new PDB Cache entry, initializes it and adds to the database.
*/
FPDBCacheEntryRef CreateAndAddPDBCacheEntryMixed( const FString& ProductVersion, const TMap<FString, FString>& FilesToBeCached );
protected:
/** Initializes the PDB Cache. */
void InitializePDBCache();
/**
* @return the size of the PDB cache entry, in GBs.
*/
int32 GetPDBCacheEntrySizeGB( const FString& PathOrLabel ) const
{
return PDBCacheEntries.FindChecked( PathOrLabel )->SizeGB;
}
/**
* @return the size of the PDB cache directory, in GBs.
*/
int32 GetPDBCacheSizeGB() const
{
int32 Result = 0;
if( bUsePDBCache )
{
for( const auto& It : PDBCacheEntries )
{
Result += It.Value->SizeGB;
}
}
return Result;
}
/**
* Cleans the PDB Cache.
*
* @param DaysToKeep - removes all PDB Cache entries that are older than this value
* @param NumberOfGBsToBeCleaned - if specifies, will try to remove as many PDB Cache entries as needed
* to free disk space specified by this value
*
*/
void CleanPDBCache( int32 DaysToKeep, int32 NumberOfGBsToBeCleaned = 0 );
/**
* Reads an existing PDB Cache entry.
*/
FPDBCacheEntryPtr ReadPDBCacheEntry( const FString& Directory );
/**
* Sort PDB Cache entries by last access time.
*/
void SortPDBCache()
{
PDBCacheEntries.ValueSort( FPDBCacheEntryByAccessTime() );
}
/**
* Removes a PDB Cache entry from the database.
* Also removes all external files associated with this PDB Cache entry.
*/
void RemovePDBCacheEntry( const FString& Directory );
/** Replaces all invalid chars with + for the specified name. */
FString EscapePath( const FString& PathOrLabel ) const
{
// @see AutomationTool.CommandUtils.EscapePath
return PathOrLabel.Replace( TEXT( ":" ), TEXT( "" ) ).Replace( TEXT( "/" ), TEXT( "+" ) ).Replace( TEXT( "\\" ), TEXT( "+" ) ).Replace( TEXT( " " ), TEXT( "+" ) );
}
};