Files
UnrealEngineUWP/Engine/Source/Programs/UnrealPak/Private/SignedArchiveWriter.cpp
Robert Manuszewski 1ff335d148 Copying //UE4/Dev-Core to //UE4/Main
==========================
MAJOR FEATURES + CHANGES
==========================

Change 2877442 on 2016/02/23 by Graeme.Thornton@GThornton_DevCore

	Allow 'dumpnonframe' stat command to also take an optional group name which will filter the output

Change 2879289 on 2016/02/24 by Graeme.Thornton@GThornton_DevCore

	Removed "-" in switch string for load time tracker. Makes -NoLoadTrackClear cmd line param actually work

Change 2879323 on 2016/02/24 by Graeme.Thornton@GThornton_DevCore

	PC builds now log engine initialization time with the load time tracker

Change 2879418 on 2016/02/24 by Graeme.Thornton@GThornton_DevCore

	Optimization for FSignedArchiveReader
	 - Compute incoming data hash in parallel on another thread while we serialize and decrypt the signature on the chunk cache worker thread

Change 2879419 on 2016/02/24 by Graeme.Thornton@GThornton_DevCore

	Extra stats for PakFile group
	 - Time reading from pak handles
	 - Number of open pak handles

Change 2881199 on 2016/02/25 by Graeme.Thornton@GThornton_DevCore

	Use 32 bit CRCs to hash chunk contents rather than SHA1. Much faster to compute than SHA1, and the decrypting 4 bytes rather than 20 is also a lot faster

Change 2885687 on 2016/02/29 by Robert.Manuszewski@Robert.Manuszewski_NCL_Stream1

	Close disregard for GC pool when disabling it to leave it in a good state.

Change 2887358 on 2016/03/01 by Steve.Robb@Dev-Core

	FlushCurrentThreadCache calls forwarded on from FMalloc wrappers.

Change 2887398 on 2016/03/01 by Steve.Robb@Dev-Core

	Stats for MallocBinned2.

Change 2887682 on 2016/03/01 by Robert.Manuszewski@Robert.Manuszewski_NCL_Stream1

	Added code to catch game hangs. If game thread is stalled for more than 5 seconds we will catch that, report its callstack and crash.

Change 2889573 on 2016/03/02 by Robert.Manuszewski@Robert.Manuszewski_NCL_Stream1

	Renamed the new StackWalkAndDump to ThreadStackWalkAndDump to avoid name collisions and build errors.

Change 2889687 on 2016/03/02 by Robert.Manuszewski@Robert.Manuszewski_NCL_Stream1

	Send thread heartbeats during potentially slow tasks.

Change 2889963 on 2016/03/02 by Robert.Manuszewski@Robert.Manuszewski_NCL_Stream1

	Added the ability to query the name of a thread given its TLS id.

Change 2890439 on 2016/03/02 by Steve.Robb@Dev-Core

	MallocBinned redirection header, to allow easy switching between MallocBinned and MallocBinned2.
	Switch back to MallocBinned by default.

Change 2891799 on 2016/03/03 by Steve.Robb@Dev-Core

	Shrunken GMallocBinned2MemSizeToIndex array.
	Improvement of some shared templated alignment-related functions.

Change 2891812 on 2016/03/03 by Graeme.Thornton@GThornton_DevCore

	Refactoring of pak signing to be more compatible with patching
	 - Store signatures in a block at the end of the pak file
	 - Optimize decryption of signatures by serializing them all when the pak is opened, and decrypting on the chunk cache worker thread between read requests.

Change 2891813 on 2016/03/03 by Robert.Manuszewski@Robert.Manuszewski_NCL_Stream2

	Don't compile UnrealPak with ICU

Change 2892386 on 2016/03/03 by Steve.Robb@Dev-Core

	GitHub #2070 - Fix incorrect FMallocDebug::Malloc alignment checking

Change 2894189 on 2016/03/04 by Steve.Robb@Dev-Core

	Memswap fixes for sizes of 3, 5, 6 and 7, and early out on 0.

Change 2894427 on 2016/03/04 by Robert.Manuszewski@Robert.Manuszewski_NCL_Stream1

	Don't display message boxes in unattended mode when D3D device is removed.

Change 2896912 on 2016/03/07 by Graeme.Thornton@GThornton_DevCore

	Disable reading of anything outside the pak file when in shipping builds

Change 2897053 on 2016/03/07 by Robert.Manuszewski@Robert.Manuszewski_NCL_Stream1

	A few improvements to thread heartbeat thread:
	- Don't check for heartbeat in debug builds
	- Don't spawn the thread heartbeat thread for programs
	- Ignore stalls when exiting

Change 2898752 on 2016/03/08 by Robert.Manuszewski@Robert.Manuszewski_NCL_Stream1

	Fixing race conditions when using malloc verify

Change 2899075 on 2016/03/08 by Robert.Manuszewski@Robert.Manuszewski_NCL_Stream1

	UnrealPak will now accept hex values for p and q. Added the command line param to trust p and q.

Change 2899091 on 2016/03/08 by Steve.Robb@Dev-Core

	Fix for TQueue::Peek const-correctness.

Change 2902775 on 2016/03/10 by Robert.Manuszewski@Robert.Manuszewski_NCL_Stream1

	Adding code to test signature keys.

Change 2905040 on 2016/03/11 by Robert.Manuszewski@Robert.Manuszewski_NCL_Stream1

	TBigInt Multiply is now 30x faster

Change 2905218 on 2016/03/11 by Steve.Robb@Dev-Core

	New FCString::Strspn and FCString::Strcspn functions.

[CL 2921423 by Robert Manuszewski in Main branch]
2016-03-24 13:53:55 -04:00

173 lines
4.6 KiB
C++

// Copyright 1998-2016 Epic Games, Inc. All Rights Reserved.
#include "UnrealPak.h"
#include "IPlatformFilePak.h"
#include "SecureHash.h"
#include "BigInt.h"
#include "SignedArchiveWriter.h"
FSignedArchiveWriter::FSignedArchiveWriter(FArchive& InPak, const FString& InPakFilename, const FEncryptionKey& InPublicKey, const FEncryptionKey& InPrivateKey)
: BufferArchive(Buffer)
, PakWriter(InPak)
, PakSignaturesFilename(FPaths::ChangeExtension(InPakFilename, TEXT("sig")))
, SizeOnDisk(0)
, PakSize(0)
, PublicKey(InPublicKey)
, PrivateKey(InPrivateKey)
{
Buffer.Reserve(FPakInfo::MaxChunkDataSize);
}
FSignedArchiveWriter::~FSignedArchiveWriter()
{
if (BufferArchive.Tell() > 0)
{
SerializeBufferAndSign();
}
delete &PakWriter;
}
void FSignedArchiveWriter::SerializeBufferAndSign()
{
// Hash the buffer
FDecryptedSignature SourceSignature;
// Compute a hash for this buffer data
SourceSignature.Data = FCrc::MemCrc32(&Buffer[0], Buffer.Num());
// Encrypt the signature
FEncryptedSignature EncryptedSignature;
FEncryption::EncryptSignature(SourceSignature, EncryptedSignature, PrivateKey);
// Flush the buffer
PakWriter.Serialize(&Buffer[0], Buffer.Num());
BufferArchive.Seek(0);
Buffer.Empty(FPakInfo::MaxChunkDataSize);
// Collect the signature so we can write it out at the end
ChunkSignatures.Add(EncryptedSignature);
#if 0
FDecryptedSignature<GPakFileChunkHashSize> TestSignature;
FEncryption::DecryptBytes(TestSignature.Data, Signature.Data, GPakFileChunkHashSize, PublicKey);
check(FMemory::Memcmp(TestSignature.Data, Hash, sizeof(Hash)) == 0);
#endif
}
bool FSignedArchiveWriter::Close()
{
if (BufferArchive.Tell() > 0)
{
SerializeBufferAndSign();
}
FArchive* SignatureWriter = IFileManager::Get().CreateFileWriter(*PakSignaturesFilename);
*SignatureWriter << ChunkSignatures;
delete SignatureWriter;
return FArchive::Close();
}
void FSignedArchiveWriter::Serialize(void* Data, int64 Length)
{
// Serialize data to a buffer. When the max buffer size is reached, the buffer is signed and
// serialized to disk with its signature
uint8* DataToWrite = (uint8*)Data;
int64 RemainingSize = Length;
while (RemainingSize > 0)
{
int64 BufferPos = BufferArchive.Tell();
int64 SizeToWrite = RemainingSize;
if (BufferPos + SizeToWrite > FPakInfo::MaxChunkDataSize)
{
SizeToWrite = FPakInfo::MaxChunkDataSize - BufferPos;
}
BufferArchive.Serialize(DataToWrite, SizeToWrite);
if (BufferArchive.Tell() == FPakInfo::MaxChunkDataSize)
{
SerializeBufferAndSign();
}
SizeOnDisk += SizeToWrite;
PakSize += SizeToWrite;
RemainingSize -= SizeToWrite;
DataToWrite += SizeToWrite;
}
}
int64 FSignedArchiveWriter::Tell()
{
return PakSize;
}
int64 FSignedArchiveWriter::TotalSize()
{
return PakSize;
}
void FSignedArchiveWriter::Seek(int64 InPos)
{
UE_LOG(LogPakFile, Fatal, TEXT("Seek is not supported in FSignedArchiveWriter."));
}
/**
* Useful code for testing the encryption methods
*/
void TestEncryption()
{
FEncryptionKey PublicKey;
FEncryptionKey PrivateKey;
TEncryptionInt P(TEXT("0x21443BD2DD63E995403"));
TEncryptionInt Q(TEXT("0x28CBB6E5749AC65749"));
FEncryption::GenerateKeyPair(P, Q, PublicKey, PrivateKey);
// Generate random data
const int32 DataSize = 1024;
uint8* Data = (uint8*)FMemory::Malloc(DataSize);
for (int32 Index = 0; Index < DataSize; ++Index)
{
Data[Index] = (uint8)(Index % 255);
}
// Generate signature
FDecryptedSignature OriginalSignature;
FEncryptedSignature EncryptedSignature;
FDecryptedSignature DecryptedSignature;
OriginalSignature.Data = FCrc::MemCrc32(Data, DataSize);
// Encrypt signature with the private key
FEncryption::EncryptSignature(OriginalSignature, EncryptedSignature, PrivateKey);
FEncryption::DecryptSignature(EncryptedSignature, DecryptedSignature, PublicKey);
// Check
if (OriginalSignature == DecryptedSignature)
{
UE_LOG(LogPakFile, Display, TEXT("Keys match"));
}
else
{
UE_LOG(LogPakFile, Fatal, TEXT("Keys mismatched!"));
}
double OverallTime = 0.0;
double OverallNumTests = 0.0;
for (int32 TestIndex = 0; TestIndex < 10; ++TestIndex)
{
static const int64 NumTests = 500;
double Timer = FPlatformTime::Seconds();
{
for (int64 i = 0; i < NumTests; ++i)
{
FEncryption::DecryptSignature(EncryptedSignature, DecryptedSignature, PublicKey);
}
}
Timer = FPlatformTime::Seconds() - Timer;
OverallTime += Timer;
OverallNumTests += (double)NumTests;
UE_LOG(LogPakFile, Display, TEXT("%i signatures decrypted in %.4fs, Avg = %.4fs, OverallAvg = %.4fs"), NumTests, Timer, Timer / (double)NumTests, OverallTime / OverallNumTests);
}
FMemory::Free(Data);
}