You've already forked UnrealEngineUWP
mirror of
https://github.com/izzy2lost/UnrealEngineUWP.git
synced 2026-03-26 18:15:20 -07:00
c27625b056
Non-DevIteration_ShooterGame #ROBOMERGE-SOURCE: CL 16678907 in //UE5/Main/... #ROBOMERGE-BOT: STARSHIP (Main -> Release-Engine-Test) (v835-16672529) [CL 16678928 by zousar shaker in ue5-release-engine-test branch]
1124 lines
32 KiB
C++
1124 lines
32 KiB
C++
// Copyright Epic Games, Inc. All Rights Reserved.
|
|
|
|
#include "ZenKvDerivedDataBackend.h"
|
|
|
|
#if WITH_ZEN_DDC_BACKEND
|
|
|
|
#if PLATFORM_WINDOWS || PLATFORM_HOLOLENS
|
|
# include "Windows/WindowsHWrapper.h"
|
|
# include "Windows/AllowWindowsPlatformTypes.h"
|
|
#endif
|
|
|
|
#include "curl/curl.h"
|
|
|
|
#if PLATFORM_WINDOWS || PLATFORM_HOLOLENS
|
|
# include "Windows/HideWindowsPlatformTypes.h"
|
|
#endif
|
|
|
|
#include "Containers/StaticArray.h"
|
|
#include "Containers/Ticker.h"
|
|
#include "DerivedDataCacheRecord.h"
|
|
#include "Dom/JsonObject.h"
|
|
#include "GenericPlatform/GenericPlatformFile.h"
|
|
#include "HAL/PlatformFileManager.h"
|
|
#include "Misc/FileHelper.h"
|
|
#include "Misc/ScopeLock.h"
|
|
#include "Misc/SecureHash.h"
|
|
#include "ProfilingDebugging/CpuProfilerTrace.h"
|
|
#include "ProfilingDebugging/CountersTrace.h"
|
|
#include "Serialization/JsonReader.h"
|
|
#include "Serialization/JsonSerializer.h"
|
|
|
|
#define UE_ZENHTTPDDC_BACKEND_WAIT_INTERVAL 0.01f
|
|
#define UE_ZENHTTPDDC_HTTP_REQUEST_TIMEOUT_SECONDS 30L
|
|
#define UE_ZENHTTPDDC_HTTP_REQUEST_TIMOUT_ENABLED 1
|
|
#define UE_ZENHTTPDDC_HTTP_DEBUG 0
|
|
#define UE_ZENHTTPDDC_REQUEST_POOL_SIZE 16
|
|
#define UE_ZENHTTPDDC_MAX_FAILED_LOGIN_ATTEMPTS 16
|
|
#define UE_ZENHTTPDDC_MAX_ATTEMPTS 4
|
|
#define UE_ZENHTTPDDC_MAX_BUFFER_RESERVE 104857600u
|
|
|
|
TRACE_DECLARE_INT_COUNTER(ZenHTTPDDC_Exist, TEXT("ZenHTTPDDC Exist"));
|
|
TRACE_DECLARE_INT_COUNTER(ZenHTTPDDC_ExistHit, TEXT("ZenHTTPDDC Exist Hit"));
|
|
TRACE_DECLARE_INT_COUNTER(ZenHTTPDDC_Get, TEXT("ZenHTTPDDC Get"));
|
|
TRACE_DECLARE_INT_COUNTER(ZenHTTPDDC_GetHit, TEXT("ZenHTTPDDC Get Hit"));
|
|
TRACE_DECLARE_INT_COUNTER(ZenHTTPDDC_Put, TEXT("ZenHTTPDDC Put"));
|
|
TRACE_DECLARE_INT_COUNTER(ZenHTTPDDC_PutHit, TEXT("ZenHTTPDDC Put Hit"));
|
|
TRACE_DECLARE_INT_COUNTER(ZenHTTPDDC_BytesReceived, TEXT("ZenHTTPDDC Bytes Received"));
|
|
TRACE_DECLARE_INT_COUNTER(ZenHTTPDDC_BytesSent, TEXT("ZenHTTPDDC Bytes Sent"));
|
|
|
|
namespace zenhttp {
|
|
|
|
/**
|
|
* Encapsulation for access token shared by all requests.
|
|
*/
|
|
|
|
/**
|
|
* Minimal HTTP request type wrapping CURL without the need for managers. This request
|
|
* is written to allow reuse of request objects, in order to allow connections to be reused.
|
|
*
|
|
* CURL has a global library initialization (curl_global_init). We rely on this happening in
|
|
* the Online/HTTP library which is a dependency on this module.
|
|
*/
|
|
class FRequest
|
|
{
|
|
public:
|
|
/**
|
|
* Supported request verbs
|
|
*/
|
|
enum RequestVerb
|
|
{
|
|
Get,
|
|
Put,
|
|
Post,
|
|
PostJson,
|
|
Delete,
|
|
Head
|
|
};
|
|
|
|
/**
|
|
* Convenience result type interpreted from HTTP response code.
|
|
*/
|
|
enum Result
|
|
{
|
|
Success,
|
|
Failed,
|
|
FailedTimeout
|
|
};
|
|
|
|
FRequest(const TCHAR* InDomain, bool bInLogErrors)
|
|
: bLogErrors(bInLogErrors)
|
|
, Domain(InDomain)
|
|
{
|
|
Curl = curl_easy_init();
|
|
Reset();
|
|
}
|
|
|
|
~FRequest()
|
|
{
|
|
curl_easy_cleanup(Curl);
|
|
}
|
|
|
|
/**
|
|
* Resets all options on the request except those that should always be set.
|
|
*/
|
|
void Reset()
|
|
{
|
|
Headers.Reset();
|
|
ResponseHeader.Reset();
|
|
ResponseBuffer.Reset();
|
|
ResponseCode = 0;
|
|
ReadDataView = TArrayView<const uint8>();
|
|
WriteDataBufferPtr = nullptr;
|
|
WriteHeaderBufferPtr = nullptr;
|
|
BytesSent = 0;
|
|
BytesReceived = 0;
|
|
CurlResult = CURL_LAST;
|
|
|
|
curl_easy_reset(Curl);
|
|
|
|
// Options that are always set for all connections.
|
|
#if UE_ZENHTTPDDC_HTTP_REQUEST_TIMOUT_ENABLED
|
|
curl_easy_setopt(Curl, CURLOPT_CONNECTTIMEOUT, UE_ZENHTTPDDC_HTTP_REQUEST_TIMEOUT_SECONDS);
|
|
#endif
|
|
curl_easy_setopt(Curl, CURLOPT_FOLLOWLOCATION, 1L);
|
|
curl_easy_setopt(Curl, CURLOPT_NOSIGNAL, 1L);
|
|
// Response functions
|
|
curl_easy_setopt(Curl, CURLOPT_HEADERDATA, this);
|
|
curl_easy_setopt(Curl, CURLOPT_HEADERFUNCTION, &FRequest::StaticWriteHeaderFn);
|
|
curl_easy_setopt(Curl, CURLOPT_WRITEDATA, this);
|
|
curl_easy_setopt(Curl, CURLOPT_WRITEFUNCTION, StaticWriteBodyFn);
|
|
// Allow compressed data
|
|
curl_easy_setopt(Curl, CURLOPT_ACCEPT_ENCODING, "gzip");
|
|
// Rewind method, handle special error case where request need to rewind data stream
|
|
curl_easy_setopt(Curl, CURLOPT_SEEKFUNCTION, StaticSeekFn);
|
|
curl_easy_setopt(Curl, CURLOPT_SEEKDATA, this);
|
|
// Debug hooks
|
|
#if UE_ZENHTTPDDC_HTTP_DEBUG
|
|
curl_easy_setopt(Curl, CURLOPT_DEBUGDATA, this);
|
|
curl_easy_setopt(Curl, CURLOPT_DEBUGFUNCTION, StaticDebugCallback);
|
|
curl_easy_setopt(Curl, CURLOPT_VERBOSE, 1L);
|
|
#endif
|
|
}
|
|
|
|
/** Gets the domain name for this request */
|
|
const FString& GetDomain() const
|
|
{
|
|
return Domain;
|
|
}
|
|
|
|
/** Returns the HTTP response code.*/
|
|
const int64 GetResponseCode() const
|
|
{
|
|
return ResponseCode;
|
|
}
|
|
|
|
/** Returns the number of bytes received this request (headers withstanding). */
|
|
const size_t GetBytesReceived() const
|
|
{
|
|
return BytesReceived;
|
|
}
|
|
|
|
/** Returns the number of bytes sent during this request (headers withstanding). */
|
|
const size_t GetBytesSent() const
|
|
{
|
|
return BytesSent;
|
|
}
|
|
|
|
/**
|
|
* Upload buffer using the request, using either "Put" or "Post" verbs.
|
|
* @param Uri Url to use.
|
|
* @param Buffer Data to upload
|
|
* @return Result of the request
|
|
*/
|
|
template<RequestVerb V>
|
|
Result PerformBlockingUpload(const TCHAR* Uri, TArrayView<const uint8> Buffer)
|
|
{
|
|
static_assert(V == Put || V == Post || V == PostJson, "Upload should use either Put or Post verbs.");
|
|
|
|
uint32 ContentLength = 0u;
|
|
|
|
if (V == Put)
|
|
{
|
|
curl_easy_setopt(Curl, CURLOPT_UPLOAD, 1L);
|
|
curl_easy_setopt(Curl, CURLOPT_INFILESIZE, Buffer.Num());
|
|
curl_easy_setopt(Curl, CURLOPT_READDATA, this);
|
|
curl_easy_setopt(Curl, CURLOPT_READFUNCTION, StaticReadFn);
|
|
Headers.Add(FString(TEXT("Content-Type: application/octet-stream")));
|
|
ContentLength = Buffer.Num();
|
|
ReadDataView = Buffer;
|
|
}
|
|
else if (V == Post || V == PostJson)
|
|
{
|
|
curl_easy_setopt(Curl, CURLOPT_POST, 1L);
|
|
curl_easy_setopt(Curl, CURLOPT_INFILESIZE, Buffer.Num());
|
|
curl_easy_setopt(Curl, CURLOPT_READDATA, this);
|
|
curl_easy_setopt(Curl, CURLOPT_READFUNCTION, StaticReadFn);
|
|
Headers.Add(V == Post ? FString(TEXT("Content-Type: application/x-www-form-urlencoded")) : FString(TEXT("Content-Type: application/json")));
|
|
ContentLength = Buffer.Num();
|
|
ReadDataView = Buffer;
|
|
}
|
|
|
|
return PerformBlocking(Uri, V, ContentLength);
|
|
}
|
|
|
|
/**
|
|
* Download an url into a buffer using the request.
|
|
* @param Uri Url to use.
|
|
* @param Buffer Optional buffer where data should be downloaded to. If empty downloaded data will
|
|
* be stored in an internal buffer and accessed GetResponse* methods.
|
|
* @return Result of the request
|
|
*/
|
|
Result PerformBlockingDownload(const TCHAR* Uri, TArray<uint8>* Buffer)
|
|
{
|
|
curl_easy_setopt(Curl, CURLOPT_HTTPGET, 1L);
|
|
WriteDataBufferPtr = Buffer;
|
|
|
|
return PerformBlocking(Uri, Get, 0u);
|
|
}
|
|
|
|
/**
|
|
* Query an url using the request. Queries can use either "Head" or "Delete" verbs.
|
|
* @param Uri Url to use.
|
|
* @return Result of the request
|
|
*/
|
|
template<RequestVerb V>
|
|
Result PerformBlockingQuery(const TCHAR* Uri)
|
|
{
|
|
static_assert(V == Head || V == Delete, "Queries should use either Head or Delete verbs.");
|
|
|
|
if (V == Delete)
|
|
{
|
|
curl_easy_setopt(Curl, CURLOPT_POST, 1L);
|
|
curl_easy_setopt(Curl, CURLOPT_CUSTOMREQUEST, "DELETE");
|
|
}
|
|
else if (V == Head)
|
|
{
|
|
curl_easy_setopt(Curl, CURLOPT_NOBODY, 1L);
|
|
}
|
|
|
|
return PerformBlocking(Uri, V, 0u);
|
|
}
|
|
|
|
/**
|
|
* Set a header to send with the request.
|
|
*/
|
|
void SetHeader(const TCHAR* Header, const TCHAR* Value)
|
|
{
|
|
check(CurlResult == CURL_LAST); // Cannot set header after request is sent
|
|
Headers.Add(FString::Printf(TEXT("%s: %s"), Header, Value));
|
|
}
|
|
|
|
/**
|
|
* Attempts to find the header from the response. Returns false if header is not present.
|
|
*/
|
|
bool GetHeader(const ANSICHAR* Header, FString& OutValue) const
|
|
{
|
|
check(CurlResult != CURL_LAST); // Cannot query headers before request is sent
|
|
|
|
const ANSICHAR* HeadersBuffer = (const ANSICHAR*)ResponseHeader.GetData();
|
|
size_t HeaderLen = strlen(Header);
|
|
|
|
// Find the header key in the (ANSI) response buffer. If not found we can exist immediately
|
|
if (const ANSICHAR* Found = strstr(HeadersBuffer, Header))
|
|
{
|
|
const ANSICHAR* Linebreak = strchr(Found, '\r');
|
|
const ANSICHAR* ValueStart = Found + HeaderLen + 2; //colon and space
|
|
const size_t ValueSize = Linebreak - ValueStart;
|
|
FUTF8ToTCHAR TCHARData(ValueStart, ValueSize);
|
|
OutValue = FString(TCHARData.Length(), TCHARData.Get());
|
|
return true;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
/**
|
|
* Returns the response buffer. Note that is the request is performed
|
|
* with an external buffer as target buffer this string will be empty.
|
|
*/
|
|
const TArray<uint8>& GetResponseBuffer() const
|
|
{
|
|
return ResponseBuffer;
|
|
}
|
|
|
|
/**
|
|
* Returns the response buffer as a string. Note that is the request is performed
|
|
* with an external buffer as target buffer this string will be empty.
|
|
*/
|
|
FString GetResponseAsString() const
|
|
{
|
|
return GetAnsiBufferAsString(ResponseBuffer);
|
|
}
|
|
|
|
/**
|
|
* Returns the response header as a string.
|
|
*/
|
|
FString GetResponseHeaderAsString()
|
|
{
|
|
return GetAnsiBufferAsString(ResponseHeader);
|
|
}
|
|
|
|
/**
|
|
* Tries to parse the response buffer as a JsonObject. Return empty pointer if
|
|
* parse error occurs.
|
|
*/
|
|
TSharedPtr<FJsonObject> GetResponseAsJsonObject() const
|
|
{
|
|
FString Response = GetAnsiBufferAsString(ResponseBuffer);
|
|
|
|
TSharedPtr<FJsonObject> JsonObject;
|
|
TSharedRef<TJsonReader<> > JsonReader = TJsonReaderFactory<>::Create(Response);
|
|
if (!FJsonSerializer::Deserialize(JsonReader, JsonObject) || !JsonObject.IsValid())
|
|
{
|
|
return TSharedPtr<FJsonObject>(nullptr);
|
|
}
|
|
|
|
return JsonObject;
|
|
}
|
|
|
|
private:
|
|
|
|
CURL* Curl;
|
|
CURLcode CurlResult;
|
|
long ResponseCode;
|
|
size_t BytesSent;
|
|
size_t BytesReceived;
|
|
bool bLogErrors;
|
|
|
|
TArrayView<const uint8> ReadDataView;
|
|
TArray<uint8>* WriteDataBufferPtr;
|
|
TArray<uint8>* WriteHeaderBufferPtr;
|
|
|
|
TArray<uint8> ResponseHeader;
|
|
TArray<uint8> ResponseBuffer;
|
|
TArray<FString> Headers;
|
|
FString Domain;
|
|
|
|
/**
|
|
* Performs the request, blocking until finished.
|
|
* @param Uri Address on the domain to query
|
|
* @param Verb HTTP verb to use
|
|
* @param Buffer Optional buffer to directly receive the result of the request.
|
|
* If unset the response body will be stored in the request.
|
|
*/
|
|
Result PerformBlocking(const TCHAR* Uri, RequestVerb Verb, uint32 ContentLength)
|
|
{
|
|
static const char* CommonHeaders[] = {
|
|
"User-Agent: UE4",
|
|
nullptr
|
|
};
|
|
|
|
TRACE_CPUPROFILER_EVENT_SCOPE(ZenHTTPDDC_CurlPerform);
|
|
|
|
// Setup request options
|
|
FString Url = FString::Printf(TEXT("%s%s"), *Domain, Uri);
|
|
curl_easy_setopt(Curl, CURLOPT_URL, TCHAR_TO_ANSI(*Url));
|
|
|
|
// Setup response header buffer. If caller has not setup a response data buffer, use interal.
|
|
WriteHeaderBufferPtr = &ResponseHeader;
|
|
if (WriteDataBufferPtr == nullptr)
|
|
{
|
|
WriteDataBufferPtr = &ResponseBuffer;
|
|
}
|
|
|
|
// Content-Length should always be set
|
|
Headers.Add(FString::Printf(TEXT("Content-Length: %d"), ContentLength));
|
|
|
|
// Build headers list
|
|
curl_slist* CurlHeaders = nullptr;
|
|
// Add common headers
|
|
for (uint8 i = 0; CommonHeaders[i] != nullptr; ++i)
|
|
{
|
|
CurlHeaders = curl_slist_append(CurlHeaders, CommonHeaders[i]);
|
|
}
|
|
// Setup added headers
|
|
for (const FString& Header : Headers)
|
|
{
|
|
CurlHeaders = curl_slist_append(CurlHeaders, TCHAR_TO_ANSI(*Header));
|
|
}
|
|
curl_easy_setopt(Curl, CURLOPT_HTTPHEADER, CurlHeaders);
|
|
|
|
// Shots fired!
|
|
CurlResult = curl_easy_perform(Curl);
|
|
|
|
// Get response code
|
|
bool bRedirected = false;
|
|
if (CURLE_OK == curl_easy_getinfo(Curl, CURLINFO_RESPONSE_CODE, &ResponseCode))
|
|
{
|
|
bRedirected = (ResponseCode >= 300 && ResponseCode < 400);
|
|
}
|
|
|
|
LogResult(CurlResult, Uri, Verb);
|
|
|
|
// Clean up
|
|
curl_slist_free_all(CurlHeaders);
|
|
|
|
return CurlResult == CURLE_OK ? Success : Failed;
|
|
}
|
|
|
|
void LogResult(CURLcode Result, const TCHAR* Uri, RequestVerb Verb) const
|
|
{
|
|
if (Result == CURLE_OK)
|
|
{
|
|
bool bSuccess = false;
|
|
const TCHAR* VerbStr = nullptr;
|
|
FString AdditionalInfo;
|
|
|
|
const bool Is400 = (ResponseCode >= 400) && (ResponseCode <= 499);
|
|
const bool Is200 = (ResponseCode >= 200) && (ResponseCode <= 299);
|
|
|
|
switch (Verb)
|
|
{
|
|
case Head:
|
|
bSuccess = Is400 || Is200;
|
|
VerbStr = TEXT("querying");
|
|
break;
|
|
case Get:
|
|
bSuccess = Is400 || Is200;
|
|
VerbStr = TEXT("fetching");
|
|
AdditionalInfo = FString::Printf(TEXT("Received: %d bytes."), BytesReceived);
|
|
break;
|
|
case Put:
|
|
bSuccess = Is200;
|
|
VerbStr = TEXT("updating");
|
|
AdditionalInfo = FString::Printf(TEXT("Sent: %d bytes."), BytesSent);
|
|
break;
|
|
case Post:
|
|
case PostJson:
|
|
bSuccess = Is200;
|
|
VerbStr = TEXT("posting");
|
|
break;
|
|
case Delete:
|
|
bSuccess = Is200;
|
|
VerbStr = TEXT("deleting");
|
|
break;
|
|
}
|
|
|
|
if (bSuccess)
|
|
{
|
|
UE_LOG(
|
|
LogDerivedDataCache,
|
|
Verbose,
|
|
TEXT("Finished %s HTTP cache entry (response %d) from %s. %s"),
|
|
VerbStr,
|
|
ResponseCode,
|
|
Uri,
|
|
*AdditionalInfo
|
|
);
|
|
}
|
|
else if (bLogErrors)
|
|
{
|
|
// Print the response body if we got one, otherwise print header.
|
|
FString Response = GetAnsiBufferAsString(ResponseBuffer.Num() > 0 ? ResponseBuffer : ResponseHeader);
|
|
Response.ReplaceCharInline('\n', ' ');
|
|
Response.ReplaceCharInline('\r', ' ');
|
|
UE_LOG(
|
|
LogDerivedDataCache,
|
|
Error,
|
|
TEXT("Failed %s HTTP cache entry (response %d) from %s. Response: %s"),
|
|
VerbStr,
|
|
ResponseCode,
|
|
Uri,
|
|
*Response
|
|
);
|
|
}
|
|
}
|
|
else if (bLogErrors)
|
|
{
|
|
UE_LOG(
|
|
LogDerivedDataCache,
|
|
Error,
|
|
TEXT("Error while connecting to %s: %s"),
|
|
*Domain,
|
|
ANSI_TO_TCHAR(curl_easy_strerror(Result))
|
|
);
|
|
}
|
|
}
|
|
|
|
FString GetAnsiBufferAsString(const TArray<uint8>& Buffer) const
|
|
{
|
|
// Content is NOT null-terminated; we need to specify lengths here
|
|
FUTF8ToTCHAR TCHARData(reinterpret_cast<const ANSICHAR*>(Buffer.GetData()), Buffer.Num());
|
|
return FString(TCHARData.Length(), TCHARData.Get());
|
|
}
|
|
|
|
static size_t StaticDebugCallback(CURL* Handle, curl_infotype DebugInfoType, char* DebugInfo, size_t DebugInfoSize, void* UserData)
|
|
{
|
|
FRequest* Request = static_cast<FRequest*>(UserData);
|
|
|
|
switch (DebugInfoType)
|
|
{
|
|
case CURLINFO_TEXT:
|
|
{
|
|
// Truncate at 1023 characters. This is just an arbitrary number based on a buffer size seen in
|
|
// the libcurl code.
|
|
DebugInfoSize = FMath::Min(DebugInfoSize, (size_t)1023);
|
|
|
|
// Calculate the actual length of the string due to incorrect use of snprintf() in lib/vtls/openssl.c.
|
|
char* FoundNulPtr = (char*)memchr(DebugInfo, 0, DebugInfoSize);
|
|
int CalculatedSize = FoundNulPtr != nullptr ? FoundNulPtr - DebugInfo : DebugInfoSize;
|
|
|
|
auto ConvertedString = StringCast<TCHAR>(static_cast<const ANSICHAR*>(DebugInfo), CalculatedSize);
|
|
FString DebugText(ConvertedString.Length(), ConvertedString.Get());
|
|
DebugText.ReplaceInline(TEXT("\n"), TEXT(""), ESearchCase::CaseSensitive);
|
|
DebugText.ReplaceInline(TEXT("\r"), TEXT(""), ESearchCase::CaseSensitive);
|
|
UE_LOG(LogDerivedDataCache, VeryVerbose, TEXT("%p: '%s'"), Request, *DebugText);
|
|
}
|
|
break;
|
|
|
|
case CURLINFO_HEADER_IN:
|
|
UE_LOG(LogDerivedDataCache, VeryVerbose, TEXT("%p: Received header (%d bytes)"), Request, DebugInfoSize);
|
|
break;
|
|
|
|
case CURLINFO_DATA_IN:
|
|
UE_LOG(LogDerivedDataCache, VeryVerbose, TEXT("%p: Received data (%d bytes)"), Request, DebugInfoSize);
|
|
break;
|
|
|
|
case CURLINFO_DATA_OUT:
|
|
UE_LOG(LogDerivedDataCache, VeryVerbose, TEXT("%p: Sent data (%d bytes)"), Request, DebugInfoSize);
|
|
break;
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
static size_t StaticReadFn(void* Ptr, size_t SizeInBlocks, size_t BlockSizeInBytes, void* UserData)
|
|
{
|
|
FRequest* Request = static_cast<FRequest*>(UserData);
|
|
TArrayView<const uint8>& ReadDataView = Request->ReadDataView;
|
|
|
|
const size_t Offset = Request->BytesSent;
|
|
const size_t ReadSize = FMath::Min((size_t)ReadDataView.Num() - Offset, SizeInBlocks * BlockSizeInBytes);
|
|
check(ReadDataView.Num() >= Offset + ReadSize);
|
|
|
|
FMemory::Memcpy(Ptr, ReadDataView.GetData() + Offset, ReadSize);
|
|
Request->BytesSent += ReadSize;
|
|
return ReadSize;
|
|
|
|
return 0;
|
|
}
|
|
|
|
static size_t StaticWriteHeaderFn(void* Ptr, size_t SizeInBlocks, size_t BlockSizeInBytes, void* UserData)
|
|
{
|
|
FRequest* Request = static_cast<FRequest*>(UserData);
|
|
const size_t WriteSize = SizeInBlocks * BlockSizeInBytes;
|
|
TArray<uint8>* WriteHeaderBufferPtr = Request->WriteHeaderBufferPtr;
|
|
if (WriteHeaderBufferPtr && WriteSize > 0)
|
|
{
|
|
const size_t CurrentBufferLength = WriteHeaderBufferPtr->Num();
|
|
if (CurrentBufferLength > 0)
|
|
{
|
|
// Remove the previous zero termination
|
|
(*WriteHeaderBufferPtr)[CurrentBufferLength - 1] = ' ';
|
|
}
|
|
|
|
// Write the header
|
|
WriteHeaderBufferPtr->Append((const uint8*)Ptr, WriteSize + 1);
|
|
(*WriteHeaderBufferPtr)[WriteHeaderBufferPtr->Num() - 1] = 0; // Zero terminate string
|
|
return WriteSize;
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
static size_t StaticWriteBodyFn(void* Ptr, size_t SizeInBlocks, size_t BlockSizeInBytes, void* UserData)
|
|
{
|
|
FRequest* Request = static_cast<FRequest*>(UserData);
|
|
const size_t WriteSize = SizeInBlocks * BlockSizeInBytes;
|
|
TArray<uint8>* WriteDataBufferPtr = Request->WriteDataBufferPtr;
|
|
|
|
if (WriteDataBufferPtr && WriteSize > 0)
|
|
{
|
|
// If this is the first part of the body being received, try to reserve
|
|
// memory if content length is defined in the header.
|
|
if (Request->BytesReceived == 0 && Request->WriteHeaderBufferPtr)
|
|
{
|
|
static const ANSICHAR* ContentLengthHeaderStr = "Content-Length: ";
|
|
const ANSICHAR* Header = (const ANSICHAR*)Request->WriteHeaderBufferPtr->GetData();
|
|
|
|
if (const ANSICHAR* ContentLengthHeader = FCStringAnsi::Strstr(Header, ContentLengthHeaderStr))
|
|
{
|
|
size_t ContentLength = (size_t)FCStringAnsi::Atoi64(ContentLengthHeader + strlen(ContentLengthHeaderStr));
|
|
if (ContentLength > 0u && ContentLength < UE_ZENHTTPDDC_MAX_BUFFER_RESERVE)
|
|
{
|
|
WriteDataBufferPtr->Reserve(ContentLength);
|
|
}
|
|
}
|
|
}
|
|
|
|
// Write to the target buffer
|
|
WriteDataBufferPtr->Append((const uint8*)Ptr, WriteSize);
|
|
Request->BytesReceived += WriteSize;
|
|
return WriteSize;
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
static size_t StaticSeekFn(void* UserData, curl_off_t Offset, int Origin)
|
|
{
|
|
FRequest* Request = static_cast<FRequest*>(UserData);
|
|
size_t NewPosition = 0;
|
|
|
|
switch (Origin)
|
|
{
|
|
case SEEK_SET: NewPosition = Offset; break;
|
|
case SEEK_CUR: NewPosition = Request->BytesSent + Offset; break;
|
|
case SEEK_END: NewPosition = Request->ReadDataView.Num() + Offset; break;
|
|
}
|
|
|
|
// Make sure we don't seek outside of the buffer
|
|
if (NewPosition < 0 || NewPosition >= Request->ReadDataView.Num())
|
|
{
|
|
return CURL_SEEKFUNC_FAIL;
|
|
}
|
|
|
|
// Update the used offset
|
|
Request->BytesSent = NewPosition;
|
|
return CURL_SEEKFUNC_OK;
|
|
}
|
|
|
|
};
|
|
|
|
//----------------------------------------------------------------------------------------------------------
|
|
// Request pool
|
|
//----------------------------------------------------------------------------------------------------------
|
|
|
|
/**
|
|
* Pool that manages a fixed set of requests. Users are required to release requests that have been
|
|
* acquired. Usable with \ref zenhttp::FScopedRequestPtr which handles this automatically.
|
|
*/
|
|
struct FRequestPool
|
|
{
|
|
FRequestPool(const TCHAR* InServiceUrl)
|
|
{
|
|
for (uint8 i = 0; i < Pool.Num(); ++i)
|
|
{
|
|
Pool[i].Usage = 0u;
|
|
Pool[i].Request = new FRequest(InServiceUrl, true);
|
|
}
|
|
}
|
|
|
|
~FRequestPool()
|
|
{
|
|
for (uint8 i = 0; i < Pool.Num(); ++i)
|
|
{
|
|
// No requests should be in use by now.
|
|
check(Pool[i].Usage.Load(EMemoryOrder::Relaxed) == 0u);
|
|
delete Pool[i].Request;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Block until a request is free. Once a request has been returned it is
|
|
* "owned by the caller and need to release it to the pool when work has been completed.
|
|
* @return Usable request instance.
|
|
*/
|
|
FRequest* WaitForFreeRequest()
|
|
{
|
|
TRACE_CPUPROFILER_EVENT_SCOPE(ZenHTTPDDC_WaitForConnPool);
|
|
while (true)
|
|
{
|
|
for (uint8 i = 0; i < Pool.Num(); ++i)
|
|
{
|
|
if (!Pool[i].Usage.Load(EMemoryOrder::Relaxed))
|
|
{
|
|
uint8 Expected = 0u;
|
|
if (Pool[i].Usage.CompareExchange(Expected, 1u))
|
|
{
|
|
return Pool[i].Request;
|
|
}
|
|
}
|
|
}
|
|
FPlatformProcess::Sleep(UE_ZENHTTPDDC_BACKEND_WAIT_INTERVAL);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Release request to the pool.
|
|
* @param Request Request that should be freed. Note that any buffer owened by the request can now be reset.
|
|
*/
|
|
void ReleaseRequestToPool(FRequest* Request)
|
|
{
|
|
for (uint8 i = 0; i < Pool.Num(); ++i)
|
|
{
|
|
if (Pool[i].Request == Request)
|
|
{
|
|
Request->Reset();
|
|
uint8 Expected = 1u;
|
|
Pool[i].Usage.CompareExchange(Expected, 0u);
|
|
return;
|
|
}
|
|
}
|
|
check(false);
|
|
}
|
|
|
|
private:
|
|
|
|
struct FEntry
|
|
{
|
|
TAtomic<uint8> Usage;
|
|
FRequest* Request;
|
|
};
|
|
|
|
TStaticArray<FEntry, UE_ZENHTTPDDC_REQUEST_POOL_SIZE> Pool;
|
|
|
|
FRequestPool() {}
|
|
};
|
|
|
|
//----------------------------------------------------------------------------------------------------------
|
|
// FScopedRequestPtr
|
|
//----------------------------------------------------------------------------------------------------------
|
|
|
|
/**
|
|
* Utility class to manage requesting and releasing requests from the \ref FRequestPool.
|
|
*/
|
|
struct FScopedRequestPtr
|
|
{
|
|
public:
|
|
FScopedRequestPtr(FRequestPool* InPool)
|
|
: Request(InPool->WaitForFreeRequest())
|
|
, Pool(InPool)
|
|
{}
|
|
|
|
~FScopedRequestPtr()
|
|
{
|
|
Pool->ReleaseRequestToPool(Request);
|
|
}
|
|
|
|
bool IsValid() const
|
|
{
|
|
return Request != nullptr;
|
|
}
|
|
|
|
FRequest* Get() const
|
|
{
|
|
check(IsValid());
|
|
return Request;
|
|
}
|
|
|
|
FRequest* operator->()
|
|
{
|
|
check(IsValid());
|
|
return Request;
|
|
}
|
|
|
|
private:
|
|
FRequest* Request;
|
|
FRequestPool* Pool;
|
|
};
|
|
|
|
//----------------------------------------------------------------------------------------------------------
|
|
// Content parsing and checking
|
|
//----------------------------------------------------------------------------------------------------------
|
|
|
|
|
|
/**
|
|
* Verifies the integrity of the recieved data using supplied checksum.
|
|
* @param Hash Recieved hash value.
|
|
* @param Payload Payload recieved.
|
|
* @return True if the data is correct, false if checksums doesn't match.
|
|
*/
|
|
bool VerifyPayload(FSHAHash Hash, const TArray<uint8>& Payload)
|
|
{
|
|
FSHAHash PayloadHash;
|
|
FSHA1::HashBuffer(Payload.GetData(), Payload.Num(), PayloadHash.Hash);
|
|
|
|
if (Hash != PayloadHash)
|
|
{
|
|
UE_LOG(LogDerivedDataCache,
|
|
Warning,
|
|
TEXT("Checksum from server did not match recieved data (%s vs %s). Discarding cached result."),
|
|
*Hash.ToString(),
|
|
*PayloadHash.ToString()
|
|
);
|
|
return false;
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
|
|
/**
|
|
* Verifies the integrity of the recieved data using supplied checksum.
|
|
* @param Request Request that the data was be recieved with.
|
|
* @param Payload Payload recieved.
|
|
* @return True if the data is correct, false if checksums doesn't match.
|
|
*/
|
|
bool VerifyRequest(const FRequest* Request, const TArray<uint8>& Payload)
|
|
{
|
|
#if 0
|
|
FString RecievedHashStr;
|
|
FSHAHash PayloadHash;
|
|
FSHA1::HashBuffer(Payload.GetData(), Payload.Num(), PayloadHash.Hash);
|
|
if (Request->GetHeader("X-Jupiter-Sha1", RecievedHashStr))
|
|
{
|
|
FSHAHash RecievedHash;
|
|
RecievedHash.FromString(RecievedHashStr);
|
|
return VerifyPayload(RecievedHash, Payload);
|
|
}
|
|
UE_LOG(LogDerivedDataCache, Warning, TEXT("HTTP server did not send a content hash. Wrong server version?"));
|
|
#endif
|
|
return true;
|
|
}
|
|
|
|
/**
|
|
* Adds a checksum (as request header) for a given payload. Jupiter will use this to verify the integrity
|
|
* of the recieved data.
|
|
* @param Request Request that the data will be sent with.
|
|
* @param Payload Payload that will be sent.
|
|
* @return True on success, false on failure.
|
|
*/
|
|
bool HashPayload(FRequest* Request, const TArrayView<const uint8> Payload)
|
|
{
|
|
FSHAHash PayloadHash;
|
|
FSHA1::HashBuffer(Payload.GetData(), Payload.Num(), PayloadHash.Hash);
|
|
Request->SetHeader(TEXT("X-Jupiter-Sha1"), *PayloadHash.ToString());
|
|
return true;
|
|
}
|
|
|
|
}
|
|
|
|
//----------------------------------------------------------------------------------------------------------
|
|
// FZenHttpDerivedDataBackend
|
|
//----------------------------------------------------------------------------------------------------------
|
|
|
|
namespace UE::DerivedData::Backends {
|
|
|
|
FZenHttpDerivedDataBackend::FZenHttpDerivedDataBackend(
|
|
ICacheFactory& InFactory,
|
|
const TCHAR* InServiceUrl,
|
|
const TCHAR* InNamespace)
|
|
: Factory(InFactory)
|
|
, Domain(InServiceUrl)
|
|
, Namespace(InNamespace)
|
|
, DefaultBucket(TEXT("default"))
|
|
, bIsUsable(false)
|
|
, FailedLoginAttempts(0)
|
|
{
|
|
if (IsServiceReady())
|
|
{
|
|
RequestPool = MakeUnique<zenhttp::FRequestPool>(InServiceUrl);
|
|
bIsUsable = true;
|
|
}
|
|
}
|
|
|
|
FZenHttpDerivedDataBackend::~FZenHttpDerivedDataBackend()
|
|
{
|
|
}
|
|
|
|
FString FZenHttpDerivedDataBackend::GetName() const
|
|
{
|
|
return Domain;
|
|
}
|
|
|
|
bool FZenHttpDerivedDataBackend::IsServiceReady()
|
|
{
|
|
zenhttp::FRequest Request(*Domain, false);
|
|
zenhttp::FRequest::Result Result = Request.PerformBlockingDownload(TEXT("/health/ready"), nullptr);
|
|
|
|
if (Result == zenhttp::FRequest::Success && Request.GetResponseCode() == 200)
|
|
{
|
|
UE_LOG(LogDerivedDataCache, Display, TEXT("HTTP DDC service status: %s."), *Request.GetResponseAsString());
|
|
return true;
|
|
}
|
|
else
|
|
{
|
|
UE_LOG(LogDerivedDataCache, Warning, TEXT("Unable to reach HTTP DDC service at %s. Status: %d . Response: %s"), *Domain, Request.GetResponseCode(), *Request.GetResponseAsString());
|
|
}
|
|
|
|
return false;
|
|
}
|
|
|
|
bool FZenHttpDerivedDataBackend::ShouldRetryOnError(int64 ResponseCode)
|
|
{
|
|
// Access token might have expired, request a new token and try again.
|
|
if (ResponseCode == 401)
|
|
{
|
|
return true;
|
|
}
|
|
|
|
// Too many requests, make a new attempt
|
|
if (ResponseCode == 429)
|
|
{
|
|
return true;
|
|
}
|
|
|
|
return false;
|
|
}
|
|
|
|
bool FZenHttpDerivedDataBackend::CachedDataProbablyExists(const TCHAR* CacheKey)
|
|
{
|
|
TRACE_CPUPROFILER_EVENT_SCOPE(ZenHTTPDDC_Exist);
|
|
TRACE_COUNTER_ADD(ZenHTTPDDC_Exist, int64(1));
|
|
COOK_STAT(auto Timer = UsageStats.TimeProbablyExists());
|
|
|
|
FString Uri = FString::Printf(TEXT("/cache/%s"), CacheKey);
|
|
long ResponseCode = 0; uint32 Attempts = 0;
|
|
|
|
// Retry request until we get an accepted response or exhaust allowed number of attempts.
|
|
while (ResponseCode == 0 && ++Attempts < UE_ZENHTTPDDC_MAX_ATTEMPTS)
|
|
{
|
|
zenhttp::FScopedRequestPtr Request(RequestPool.Get());
|
|
zenhttp::FRequest::Result Result = Request->PerformBlockingQuery<zenhttp::FRequest::Head>(*Uri);
|
|
ResponseCode = Request->GetResponseCode();
|
|
|
|
if (ResponseCode == 200 || ResponseCode == 400)
|
|
{
|
|
const bool bIsHit = (Result == zenhttp::FRequest::Success && ResponseCode == 200);
|
|
if (bIsHit)
|
|
{
|
|
TRACE_COUNTER_ADD(ZenHTTPDDC_ExistHit, int64(1));
|
|
}
|
|
return bIsHit;
|
|
}
|
|
|
|
if (!ShouldRetryOnError(ResponseCode))
|
|
{
|
|
return false;
|
|
}
|
|
|
|
ResponseCode = 0;
|
|
}
|
|
|
|
return false;
|
|
}
|
|
|
|
bool FZenHttpDerivedDataBackend::GetCachedData(const TCHAR* CacheKey, TArray<uint8>& OutData)
|
|
{
|
|
TRACE_CPUPROFILER_EVENT_SCOPE(ZenHTTPDDC_Get);
|
|
TRACE_COUNTER_ADD(ZenHTTPDDC_Get, int64(1));
|
|
COOK_STAT(auto Timer = UsageStats.TimeGet());
|
|
|
|
FString Uri = FString::Printf(TEXT("/cache/%s"), CacheKey);
|
|
int64 ResponseCode = 0; uint32 Attempts = 0;
|
|
|
|
// Retry request until we get an accepted response or exhaust allowed number of attempts.
|
|
while (ResponseCode == 0 && ++Attempts < UE_ZENHTTPDDC_MAX_ATTEMPTS)
|
|
{
|
|
zenhttp::FScopedRequestPtr Request(RequestPool.Get());
|
|
if (Request.IsValid())
|
|
{
|
|
zenhttp::FRequest::Result Result = Request->PerformBlockingDownload(*Uri, &OutData);
|
|
ResponseCode = Request->GetResponseCode();
|
|
|
|
// Request was successful, make sure we got all the expected data.
|
|
if (ResponseCode == 200 && VerifyRequest(Request.Get(), OutData))
|
|
{
|
|
TRACE_COUNTER_ADD(ZenHTTPDDC_GetHit, int64(1));
|
|
TRACE_COUNTER_ADD(ZenHTTPDDC_BytesReceived, int64(Request->GetBytesReceived()));
|
|
COOK_STAT(Timer.AddHit(Request->GetBytesReceived()));
|
|
return true;
|
|
}
|
|
|
|
if (!ShouldRetryOnError(ResponseCode))
|
|
{
|
|
return false;
|
|
}
|
|
|
|
ResponseCode = 0;
|
|
}
|
|
}
|
|
|
|
return false;
|
|
}
|
|
|
|
FDerivedDataBackendInterface::EPutStatus FZenHttpDerivedDataBackend::PutCachedData(const TCHAR* CacheKey, TArrayView<const uint8> InData, bool bPutEvenIfExists)
|
|
{
|
|
TRACE_CPUPROFILER_EVENT_SCOPE(ZenHTTPDDC_Put);
|
|
COOK_STAT(auto Timer = UsageStats.TimePut());
|
|
|
|
FString Uri = FString::Printf(TEXT("/cache/%s"), CacheKey);
|
|
int64 ResponseCode = 0; uint32 Attempts = 0;
|
|
|
|
// Retry request until we get an accepted response or exhaust allowed number of attempts.
|
|
while (ResponseCode == 0 && ++Attempts < UE_ZENHTTPDDC_MAX_ATTEMPTS)
|
|
{
|
|
zenhttp::FScopedRequestPtr Request(RequestPool.Get());
|
|
if (Request.IsValid())
|
|
{
|
|
// Append the content hash to the header
|
|
HashPayload(Request.Get(), InData);
|
|
|
|
zenhttp::FRequest::Result Result = Request->PerformBlockingUpload<zenhttp::FRequest::Put>(*Uri, InData);
|
|
ResponseCode = Request->GetResponseCode();
|
|
|
|
if (ResponseCode >= 200 && ResponseCode < 300)
|
|
{
|
|
TRACE_COUNTER_ADD(ZenHTTPDDC_BytesSent, int64(Request->GetBytesSent()));
|
|
COOK_STAT(Timer.AddHit(Request->GetBytesSent()));
|
|
|
|
return EPutStatus::Cached;
|
|
}
|
|
|
|
if (!ShouldRetryOnError(ResponseCode))
|
|
{
|
|
return EPutStatus::NotCached;
|
|
}
|
|
|
|
ResponseCode = 0;
|
|
}
|
|
}
|
|
|
|
return EPutStatus::NotCached;
|
|
}
|
|
|
|
void FZenHttpDerivedDataBackend::RemoveCachedData(const TCHAR* CacheKey, bool bTransient)
|
|
{
|
|
TRACE_CPUPROFILER_EVENT_SCOPE(ZenHTTPDDC_Remove);
|
|
FString Uri = FString::Printf(TEXT("/cache/%s"), CacheKey);
|
|
int64 ResponseCode = 0; uint32 Attempts = 0;
|
|
|
|
// Retry request until we get an accepted response or exhaust allowed number of attempts.
|
|
while (ResponseCode == 0 && ++Attempts < UE_ZENHTTPDDC_MAX_ATTEMPTS)
|
|
{
|
|
zenhttp::FScopedRequestPtr Request(RequestPool.Get());
|
|
if (Request.IsValid())
|
|
{
|
|
zenhttp::FRequest::Result Result = Request->PerformBlockingQuery<zenhttp::FRequest::Delete>(*Uri);
|
|
ResponseCode = Request->GetResponseCode();
|
|
|
|
if (ResponseCode == 200)
|
|
{
|
|
return;
|
|
}
|
|
|
|
if (!ShouldRetryOnError(ResponseCode))
|
|
{
|
|
return;
|
|
}
|
|
|
|
ResponseCode = 0;
|
|
}
|
|
}
|
|
}
|
|
|
|
bool FZenHttpDerivedDataBackend::IsWritable() const
|
|
{
|
|
return true;
|
|
}
|
|
|
|
FDerivedDataBackendInterface::ESpeedClass
|
|
FZenHttpDerivedDataBackend::GetSpeedClass() const
|
|
{
|
|
return ESpeedClass::Fast;
|
|
}
|
|
|
|
TSharedRef<FDerivedDataCacheStatsNode> FZenHttpDerivedDataBackend::GatherUsageStats() const
|
|
{
|
|
return MakeShared<FDerivedDataCacheStatsNode>(this, "ZenHttp");
|
|
}
|
|
|
|
bool FZenHttpDerivedDataBackend::TryToPrefetch(TConstArrayView<FString> CacheKeys)
|
|
{
|
|
return false;
|
|
}
|
|
|
|
bool FZenHttpDerivedDataBackend::WouldCache(const TCHAR* CacheKey, TArrayView<const uint8> InData)
|
|
{
|
|
return true;
|
|
}
|
|
|
|
bool FZenHttpDerivedDataBackend::ApplyDebugOptions(FBackendDebugOptions& InOptions)
|
|
{
|
|
return false;
|
|
}
|
|
|
|
FRequest FZenHttpDerivedDataBackend::Put(
|
|
TConstArrayView<UE::DerivedData::FCacheRecord> Records,
|
|
FStringView Context,
|
|
ECachePolicy Policy,
|
|
EPriority Priority,
|
|
FOnCachePutComplete&& OnComplete)
|
|
{
|
|
if (OnComplete)
|
|
{
|
|
for (const FCacheRecord& Record : Records)
|
|
{
|
|
OnComplete({ Record.GetKey(), EStatus::Error });
|
|
}
|
|
}
|
|
return FRequest();
|
|
}
|
|
|
|
FRequest FZenHttpDerivedDataBackend::Get(
|
|
TConstArrayView<UE::DerivedData::FCacheKey> Keys,
|
|
FStringView Context,
|
|
UE::DerivedData::ECachePolicy Policy,
|
|
UE::DerivedData::EPriority Priority,
|
|
UE::DerivedData::FOnCacheGetComplete&& OnComplete)
|
|
{
|
|
if (OnComplete)
|
|
{
|
|
for (const FCacheKey& Key : Keys)
|
|
{
|
|
OnComplete({ Factory.CreateRecord(Key).Build(), EStatus::Error });
|
|
}
|
|
}
|
|
return FRequest();
|
|
}
|
|
|
|
FRequest FZenHttpDerivedDataBackend::GetPayload(
|
|
TConstArrayView<UE::DerivedData::FCachePayloadKey> Keys,
|
|
FStringView Context,
|
|
UE::DerivedData::ECachePolicy Policy,
|
|
UE::DerivedData::EPriority Priority,
|
|
UE::DerivedData::FOnCacheGetPayloadComplete&& OnComplete)
|
|
{
|
|
if (OnComplete)
|
|
{
|
|
for (const FCachePayloadKey& Key : Keys)
|
|
{
|
|
OnComplete({ Key.CacheKey, FPayload(Key.Id), EStatus::Error });
|
|
}
|
|
}
|
|
return FRequest();
|
|
}
|
|
|
|
void FZenHttpDerivedDataBackend::CancelAll()
|
|
{
|
|
}
|
|
|
|
}
|
|
|
|
#endif //WITH_HTTP_DDC_BACKEND
|