Files
UnrealEngineUWP/Engine/Source/Programs/WebTests/Private/TestHttp.cpp
lorry li 329d5447bc Make sure it's safe to retry http request in http thread, by keep the shared ptr of retry manager in the request. (Caught by the random failure of a test case in WebTests)
NOTE: This change requires existing holder of FHttpRetrySystem::FManager be converted to shared ptr.

#jira UE-163631
[REVIEW] [at]michael.kirzinger [at]michael.atchison
#rb [at]michael.kirzinger [at]michael.atchison

[CL 29698310 by lorry li in ue5-main branch]
2023-11-13 17:55:00 -05:00

1044 lines
34 KiB
C++

// Copyright Epic Games, Inc. All Rights Reserved.
#include "CoreMinimal.h"
#include "HAL/FileManager.h"
#include "HAL/IConsoleManager.h"
#include "HAL/PlatformProcess.h"
#include "HAL/Runnable.h"
#include "HAL/RunnableThread.h"
#include "HttpManager.h"
#include "HttpRetrySystem.h"
#include "Http.h"
#include "Misc/CommandLine.h"
#include "TestHarness.h"
#include "Serialization/JsonSerializerMacros.h"
/**
* HTTP Tests
* -----------------------------------------------------------------------------------------------
*
* PURPOSE:
*
* Integration Tests to make sure all kinds of HTTP client features in C++ work well on different platforms,
* including but not limited to error handing, retrying, threading, streaming, SSL and profiling.
*
* Refer to WebTests/README.md for more info about how to run these tests
*
* -----------------------------------------------------------------------------------------------
*/
#define HTTP_TAG "[HTTP]"
#define HTTP_TIME_DIFF_TOLERANCE 0.5f
extern TAutoConsoleVariable<bool> CVarHttpInsecureProtocolEnabled;
extern TAutoConsoleVariable<bool> CVarHttpRetrySystemNonGameThreadSupportEnabled;
class FHttpModuleTestFixture
{
public:
FHttpModuleTestFixture()
: WebServerIp(TEXT("127.0.0.1"))
, WebServerHttpPort(8000)
, bRunHeavyTests(false)
, bRetryEnabled(true)
, OldVerbosity(LogHttp.GetVerbosity())
{
ParseSettingsFromCommandLine();
bRetryEnabled &= CVarHttpRetrySystemNonGameThreadSupportEnabled.GetValueOnAnyThread();
HttpModule = new FHttpModule();
IModuleInterface* Module = HttpModule;
Module->StartupModule();
CVarHttpInsecureProtocolEnabled->Set(true);
}
virtual ~FHttpModuleTestFixture()
{
IModuleInterface* Module = HttpModule;
Module->ShutdownModule();
delete Module;
if (OldVerbosity != LogHttp.GetVerbosity())
{
LogHttp.SetVerbosity(OldVerbosity);
}
}
void ParseSettingsFromCommandLine()
{
FParse::Value(FCommandLine::Get(), TEXT("web_server_ip"), WebServerIp);
FParse::Bool(FCommandLine::Get(), TEXT("run_heavy_tests"), bRunHeavyTests);
FParse::Bool(FCommandLine::Get(), TEXT("retry_enabled"), bRetryEnabled);
}
void DisableWarningsInThisTest()
{
LogHttp.SetVerbosity(ELogVerbosity::Error);
}
const FString UrlWithInvalidPortToTestConnectTimeout() const { return FString::Format(TEXT("http://{0}:{1}"), { *WebServerIp, 8765 }); }
const FString UrlBase() const { return FString::Format(TEXT("http://{0}:{1}"), { *WebServerIp, WebServerHttpPort }); }
const FString UrlHttpTests() const { return FString::Format(TEXT("{0}/webtests/httptests"), { *UrlBase() }); }
const FString UrlToTestMethods() const { return FString::Format(TEXT("{0}/methods"), { *UrlHttpTests() }); }
const FString UrlStreamDownload(uint32 Chunks, uint32 ChunkSize) { return FString::Format(TEXT("{0}/streaming_download/{1}/{2}/"), { *UrlHttpTests(), Chunks, ChunkSize }); }
FString WebServerIp;
uint32 WebServerHttpPort;
FHttpModule* HttpModule;
bool bRunHeavyTests;
bool bRetryEnabled;
ELogVerbosity::Type OldVerbosity;
};
TEST_CASE_METHOD(FHttpModuleTestFixture, "Shutdown http module without issue when there are ongoing http requests.", HTTP_TAG)
{
DisableWarningsInThisTest();
uint32 ChunkSize = 1024 * 1024;
TArray<uint8> DataChunk;
DataChunk.SetNum(ChunkSize);
FMemory::Memset(DataChunk.GetData(), 'd', ChunkSize);
for (int32 i = 0; i < 10; ++i)
{
IHttpRequest* LeakingHttpRequest = FPlatformHttp::ConstructRequest(); // Leaking in purpose to make sure it's ok
TSharedRef<IHttpRequest> HttpRequest = HttpModule->CreateRequest();
HttpRequest->SetURL(UrlToTestMethods());
HttpRequest->SetVerb(TEXT("PUT"));
// TODO: Use some shared data, like cookie, openssl session etc.
HttpRequest->SetContent(DataChunk);
HttpRequest->OnProcessRequestComplete().BindLambda([](FHttpRequestPtr HttpRequest, FHttpResponsePtr HttpResponse, bool bSucceeded) {
CHECK(bSucceeded);
});
HttpRequest->ProcessRequest();
}
HttpModule->GetHttpManager().Tick(0.0f);
}
class FMockRetryManager : public FHttpRetrySystem::FManager
{
public:
using FHttpRetrySystem::FManager::FManager;
using FHttpRetrySystem::FManager::RequestList;
using FHttpRetrySystem::FManager::FHttpRetryRequestEntry;
using FHttpRetrySystem::FManager::RetryTimeoutRelativeSecondsDefault;
bool IsEmpty()
{
FScopeLock ScopeLock(&RequestListLock);
return RequestList.IsEmpty();
}
};
class FWaitUntilCompleteHttpFixture : public FHttpModuleTestFixture
{
public:
FWaitUntilCompleteHttpFixture()
{
HttpModule->GetHttpManager().SetRequestAddedDelegate(FHttpManagerRequestAddedDelegate::CreateRaw(this, &FWaitUntilCompleteHttpFixture::OnRequestAdded));
HttpModule->GetHttpManager().SetRequestCompletedDelegate(FHttpManagerRequestCompletedDelegate::CreateRaw(this, &FWaitUntilCompleteHttpFixture::OnRequestCompleted));
if (bRetryEnabled)
{
HttpRetryManager = MakeShared<FMockRetryManager>(FHttpRetrySystem::FRetryLimitCountSetting(RetryLimitCount), FHttpRetrySystem::FRetryTimeoutRelativeSecondsSetting(/*RetryTimeoutRelativeSeconds*/));
HttpRetryManager->RetryTimeoutRelativeSecondsDefault = 2.0f; // Value is set so that retry system will use it at high level, will no longer wait for ever for HTTP code
}
}
~FWaitUntilCompleteHttpFixture()
{
WaitUntilAllHttpRequestsComplete();
HttpModule->GetHttpManager().SetRequestAddedDelegate(FHttpManagerRequestAddedDelegate());
HttpModule->GetHttpManager().SetRequestCompletedDelegate(FHttpManagerRequestCompletedDelegate());
}
void OnRequestAdded(const FHttpRequestRef& Request)
{
++OngoingRequests;
}
void OnRequestCompleted(const FHttpRequestRef& Request)
{
ensure(--OngoingRequests >= 0);
}
void WaitUntilAllHttpRequestsComplete()
{
while (OngoingRequests != 0 || (bRetryEnabled && !HttpRetryManager->IsEmpty()) )
{
HttpModule->GetHttpManager().Tick(TickFrequency);
FPlatformProcess::Sleep(TickFrequency);
}
// In case in http thread the http request complete and set OngoingRequests to 0, http manager never
// had chance to Tick and remove the request
HttpModule->GetHttpManager().Tick(TickFrequency);
}
TSharedRef<IHttpRequest> CreateRequest()
{
return bRetryEnabled ? HttpRetryManager->CreateRequest() : HttpModule->CreateRequest();
}
std::atomic<int32> OngoingRequests = 0;
float TickFrequency = 1.0f / 60; /*60 FPS*/;
uint32 RetryLimitCount = 0;
TSharedPtr<FMockRetryManager> HttpRetryManager;
};
TEST_CASE_METHOD(FWaitUntilCompleteHttpFixture, "Http Methods", HTTP_TAG)
{
TSharedRef<IHttpRequest> HttpRequest = CreateRequest();
CHECK(HttpRequest->GetVerb() == TEXT("GET"));
HttpRequest->SetURL(UrlToTestMethods());
SECTION("Default GET")
{
}
SECTION("GET")
{
HttpRequest->SetVerb(TEXT("GET"));
}
SECTION("POST")
{
HttpRequest->SetVerb(TEXT("POST"));
}
SECTION("PUT")
{
HttpRequest->SetVerb(TEXT("PUT"));
}
SECTION("DELETE")
{
HttpRequest->SetVerb(TEXT("DELETE"));
}
HttpRequest->OnProcessRequestComplete().BindLambda([](FHttpRequestPtr HttpRequest, FHttpResponsePtr HttpResponse, bool bSucceeded) {
CHECK(bSucceeded);
REQUIRE(HttpResponse != nullptr);
CHECK(HttpResponse->GetResponseCode() == 200);
});
HttpRequest->ProcessRequest();
}
TEST_CASE_METHOD(FWaitUntilCompleteHttpFixture, "Get large response content without chunks", HTTP_TAG)
{
TSharedRef<IHttpRequest> HttpRequest = CreateRequest();
HttpRequest->SetURL(FString::Format(TEXT("{0}/get_large_response_without_chunks/{1}/"), { *UrlHttpTests(), 1024 * 1024/*bytes_number*/}));
HttpRequest->SetVerb(TEXT("GET"));
HttpRequest->OnProcessRequestComplete().BindLambda([](FHttpRequestPtr HttpRequest, FHttpResponsePtr HttpResponse, bool bSucceeded) {
CHECK(bSucceeded);
REQUIRE(HttpResponse != nullptr);
CHECK(HttpResponse->GetResponseCode() == 200);
});
HttpRequest->ProcessRequest();
}
// TODO: Enable this after finding a more reliable way of simulate timeout
//TEST_CASE_METHOD(FWaitUntilCompleteHttpFixture, "Http request connect timeout", HTTP_TAG)
//{
// TSharedRef<IHttpRequest> HttpRequest = CreateRequest();
// HttpRequest->SetURL(UrlWithInvalidPortToTestConnectTimeout());
// HttpRequest->SetVerb(TEXT("GET"));
// HttpRequest->SetTimeout(7);
// FDateTime StartTime = FDateTime::Now();
// HttpRequest->OnProcessRequestComplete().BindLambda([StartTime](FHttpRequestPtr HttpRequest, FHttpResponsePtr HttpResponse, bool bSucceeded) {
// CHECK(!bSucceeded);
// CHECK(HttpResponse == nullptr);
// // TODO: For now curl impl is using customized timeout instead of relying on native http timeout,
// // which doesn't get CURLE_COULDNT_CONNECT. Enable this after switching to native http timeout
// //CHECK(HttpRequest->GetStatus() == EHttpRequestStatus::Failed_ConnectionError);
// FTimespan Timespan = FDateTime::Now() - StartTime;
// float DurationInSeconds = Timespan.GetTotalSeconds();
// CHECK(FMath::IsNearlyEqual(DurationInSeconds, 7, HTTP_TIME_DIFF_TOLERANCE));
// });
// HttpRequest->ProcessRequest();
//}
TEST_CASE_METHOD(FWaitUntilCompleteHttpFixture, "Streaming http download", HTTP_TAG)
{
uint32 Chunks = 3;
uint32 ChunkSize = 1024*1024;
TSharedRef<IHttpRequest> HttpRequest = CreateRequest();
HttpRequest->SetURL(UrlStreamDownload(Chunks, ChunkSize));
HttpRequest->SetVerb(TEXT("GET"));
TSharedRef<int64> TotalBytesReceived = MakeShared<int64>(0);
SECTION("Success without stream provided")
{
HttpRequest->OnProcessRequestComplete().BindLambda([Chunks, ChunkSize](FHttpRequestPtr HttpRequest, FHttpResponsePtr HttpResponse, bool bSucceeded) {
CHECK(bSucceeded);
REQUIRE(HttpResponse != nullptr);
CHECK(HttpResponse->GetResponseCode() == 200);
CHECK(!HttpResponse->GetAllHeaders().IsEmpty());
CHECK(HttpResponse->GetContentLength() == Chunks * ChunkSize);
});
}
SECTION("Success with customized stream")
{
class FTestHttpReceiveStream final : public FArchive
{
public:
FTestHttpReceiveStream(TSharedRef<int64> InTotalBytesReceived)
: TotalBytesReceived(InTotalBytesReceived)
{
}
virtual void Serialize(void* V, int64 Length) override
{
*TotalBytesReceived += Length;
}
TSharedRef<int64> TotalBytesReceived;
};
TSharedRef<FTestHttpReceiveStream> Stream = MakeShared<FTestHttpReceiveStream>(TotalBytesReceived);
CHECK(HttpRequest->SetResponseBodyReceiveStream(Stream));
HttpRequest->OnProcessRequestComplete().BindLambda([Chunks, ChunkSize, TotalBytesReceived](FHttpRequestPtr HttpRequest, FHttpResponsePtr HttpResponse, bool bSucceeded) {
CHECK(bSucceeded);
REQUIRE(HttpResponse != nullptr);
CHECK(HttpResponse->GetResponseCode() == 200);
CHECK(!HttpResponse->GetAllHeaders().IsEmpty());
CHECK(HttpResponse->GetContentLength() == Chunks * ChunkSize);
CHECK(HttpResponse->GetContent().IsEmpty());
CHECK(*TotalBytesReceived == Chunks * ChunkSize);
});
}
SECTION("Success with customized stream delegate")
{
FHttpRequestStreamDelegate Delegate;
Delegate.BindLambda([TotalBytesReceived](void* Ptr, int64 Length) {
*TotalBytesReceived += Length;
return true;
});
CHECK(HttpRequest->SetResponseBodyReceiveStreamDelegate(Delegate));
HttpRequest->OnProcessRequestComplete().BindLambda([Chunks, ChunkSize, TotalBytesReceived](FHttpRequestPtr HttpRequest, FHttpResponsePtr HttpResponse, bool bSucceeded) {
CHECK(bSucceeded);
REQUIRE(HttpResponse != nullptr);
CHECK(HttpResponse->GetResponseCode() == 200);
CHECK(!HttpResponse->GetAllHeaders().IsEmpty());
CHECK(HttpResponse->GetContentLength() == Chunks * ChunkSize);
CHECK(HttpResponse->GetContent().IsEmpty());
CHECK(*TotalBytesReceived == Chunks * ChunkSize);
});
}
SECTION("Use customized stream to receive response body but failed when serialize")
{
DisableWarningsInThisTest();
class FTestHttpReceiveStream final : public FArchive
{
public:
FTestHttpReceiveStream(TSharedRef<int64> InTotalBytesReceived)
: TotalBytesReceived(InTotalBytesReceived)
{
}
virtual void Serialize(void* V, int64 Length) override
{
*TotalBytesReceived += Length;
SetError();
}
TSharedRef<int64> TotalBytesReceived;
};
TSharedRef<FTestHttpReceiveStream> Stream = MakeShared<FTestHttpReceiveStream>(TotalBytesReceived);
CHECK(HttpRequest->SetResponseBodyReceiveStream(Stream));
HttpRequest->OnProcessRequestComplete().BindLambda([ChunkSize, TotalBytesReceived](FHttpRequestPtr HttpRequest, FHttpResponsePtr HttpResponse, bool bSucceeded) {
CHECK(!bSucceeded);
CHECK(HttpResponse != nullptr);
CHECK(*TotalBytesReceived <= ChunkSize);
});
}
SECTION("Use customized stream delegate to receive response body but failed when call")
{
DisableWarningsInThisTest();
FHttpRequestStreamDelegate Delegate;
Delegate.BindLambda([TotalBytesReceived](void* Ptr, int64 Length) {
*TotalBytesReceived += Length;
return false;
});
CHECK(HttpRequest->SetResponseBodyReceiveStreamDelegate(Delegate));
HttpRequest->OnProcessRequestComplete().BindLambda([ChunkSize, TotalBytesReceived](FHttpRequestPtr HttpRequest, FHttpResponsePtr HttpResponse, bool bSucceeded) {
CHECK(!bSucceeded);
CHECK(HttpResponse != nullptr);
CHECK(*TotalBytesReceived <= ChunkSize);
});
}
SECTION("Success with file stream to receive response body")
{
FString Filename = FString(FPlatformProcess::UserSettingsDir()) / TEXT("TestStreamDownload.dat");
FArchive* RawFile = IFileManager::Get().CreateFileWriter(*Filename);
CHECK(RawFile != nullptr);
TSharedRef<FArchive> FileToWrite = MakeShareable(RawFile);
CHECK(HttpRequest->SetResponseBodyReceiveStream(FileToWrite));
HttpRequest->OnProcessRequestComplete().BindLambda([Chunks, ChunkSize, Filename, FileToWrite](FHttpRequestPtr HttpRequest, FHttpResponsePtr HttpResponse, bool bSucceeded) {
CHECK(bSucceeded);
REQUIRE(HttpResponse != nullptr);
CHECK(HttpResponse->GetContentLength() == Chunks * ChunkSize);
CHECK(HttpResponse->GetContent().IsEmpty());
CHECK(HttpResponse->GetResponseCode() == 200);
CHECK(!HttpResponse->GetAllHeaders().IsEmpty());
FileToWrite->FlushCache();
FileToWrite->Close();
TSharedRef<FArchive> FileToRead = MakeShareable(IFileManager::Get().CreateFileReader(*Filename));
CHECK(FileToRead->TotalSize() == Chunks * ChunkSize);
FileToRead->Close();
IFileManager::Get().Delete(*Filename);
});
}
HttpRequest->ProcessRequest();
}
TEST_CASE_METHOD(FWaitUntilCompleteHttpFixture, "Can run parallel stream download requests", HTTP_TAG)
{
uint32 Chunks = 5;
uint32 ChunkSize = 1024*1024;
for (int i = 0; i < 3; ++i)
{
TSharedRef<IHttpRequest> HttpRequest = CreateRequest();
HttpRequest->SetURL(UrlStreamDownload(Chunks, ChunkSize));
HttpRequest->SetVerb(TEXT("GET"));
HttpRequest->OnProcessRequestComplete().BindLambda([Chunks, ChunkSize](FHttpRequestPtr HttpRequest, FHttpResponsePtr HttpResponse, bool bSucceeded) {
CHECK(HttpResponse->GetContentLength() == Chunks * ChunkSize);
CHECK(bSucceeded);
CHECK(HttpResponse->GetResponseCode() == 200);
});
HttpRequest->ProcessRequest();
}
}
TEST_CASE_METHOD(FWaitUntilCompleteHttpFixture, "Can download big file exceeds 32 bits", HTTP_TAG)
{
if (!bRunHeavyTests)
{
return;
}
// 5 * 1024 * 1024 * 1024 BYTES = 5368709120 BYTES = 5 GB
uint64 Chunks = 5 * 1024;
uint64 ChunkSize = 1024 * 1024;
TSharedRef<IHttpRequest> HttpRequest = CreateRequest();
HttpRequest->SetURL(UrlStreamDownload(Chunks, ChunkSize));
HttpRequest->SetVerb(TEXT("GET"));
TSharedRef<int64> TotalBytesReceived = MakeShared<int64>(0);
FHttpRequestStreamDelegate Delegate;
Delegate.BindLambda([TotalBytesReceived](void* Ptr, int64 Length) {
*TotalBytesReceived += Length;
return true;
});
HttpRequest->SetResponseBodyReceiveStreamDelegate(Delegate);
HttpRequest->OnProcessRequestComplete().BindLambda([Chunks, ChunkSize, TotalBytesReceived](FHttpRequestPtr HttpRequest, FHttpResponsePtr HttpResponse, bool bSucceeded) {
CHECK(bSucceeded);
REQUIRE(HttpResponse != nullptr);
CHECK(HttpResponse->GetContentLength() == Chunks * ChunkSize);
CHECK(HttpResponse->GetContent().IsEmpty());
CHECK(*TotalBytesReceived == Chunks * ChunkSize);
CHECK(HttpResponse->GetResponseCode() == 200);
});
HttpRequest->ProcessRequest();
}
TEST_CASE_METHOD(FWaitUntilCompleteHttpFixture, "Streaming http upload from memory", HTTP_TAG)
{
TSharedRef<IHttpRequest> HttpRequest = CreateRequest();
HttpRequest->SetURL(FString::Format(TEXT("{0}/streaming_upload_post"), { *UrlHttpTests() }));
HttpRequest->SetVerb(TEXT("POST"));
const char* BoundaryLabel = "test_http_boundary";
HttpRequest->SetHeader(TEXT("Content-Type"), FString::Format(TEXT("multipart/form-data; boundary={0}"), { BoundaryLabel }));
// Data will be sent by chunks in http request
const uint32 FileSize = 10*1024*1024;
char* FileData = (char*)FMemory::Malloc(FileSize + 1);
FMemory::Memset(FileData, 'd', FileSize);
FileData[FileSize + 1] = '\0';
TArray<uint8> ContentData;
const int32 ContentMaxSize = FileSize + 256/*max length of format string*/;
ContentData.Reserve(ContentMaxSize);
const int32 ContentLength = FCStringAnsi::Snprintf(
(char*)ContentData.GetData(),
ContentMaxSize,
"--%s\r\n"
"Content-Disposition: form-data; name=\"file\"; filename=\"bigfile.zip\"\r\n"
"Content-Type: application/octet-stream\r\n\r\n"
"%s\r\n"
"--%s--",
BoundaryLabel, FileData, BoundaryLabel);
FMemory::Free(FileData);
CHECK(ContentLength > 0);
CHECK(ContentLength < ContentMaxSize);
ContentData.SetNumUninitialized(ContentLength);
HttpRequest->SetContent(MoveTemp(ContentData));
HttpRequest->OnProcessRequestComplete().BindLambda([](FHttpRequestPtr HttpRequest, FHttpResponsePtr HttpResponse, bool bSucceeded) {
CHECK(bSucceeded);
REQUIRE(HttpResponse != nullptr);
CHECK(HttpResponse->GetResponseCode() == 200);
});
HttpRequest->ProcessRequest();
}
class FTestHttpUploadStream final : public FArchive
{
public:
FTestHttpUploadStream(uint64 InTotalSize)
: FakeTotalSize(InTotalSize)
{
}
virtual void Serialize(void* V, int64 Length) override
{
for (int64 i = 0; i < Length; ++i)
{
((char*)V)[i] = 'd';
}
CurrentPos += Length;
}
virtual int64 TotalSize() override
{
return FakeTotalSize;
}
virtual void Seek(int64 InPos)
{
CurrentPos = InPos;
}
virtual int64 Tell() override
{
return CurrentPos;
}
uint64 FakeTotalSize;
uint64 CurrentPos = 0;
};
TEST_CASE_METHOD(FWaitUntilCompleteHttpFixture, "Can upload big file exceeds 32 bits", HTTP_TAG)
{
if (!bRunHeavyTests)
{
return;
}
// TODO: Back to check later. xCurl 2206.4.0.0 doesn't work with file bigger than 32 bits
// 5 * 1024 * 1024 * 1024 BYTES = 5368709120 BYTES = 5 GB
//const uint64 TotalSize = 5368709120;
//const uint64 TotalSize = 4294967296;
//const uint64 TotalSize = 4294967295;
//const uint64 TotalSize = 2147483648;
const uint64 TotalSize = 2147483647;
TSharedRef<FTestHttpUploadStream> Stream = MakeShared<FTestHttpUploadStream>(TotalSize);
TSharedRef<IHttpRequest> HttpRequest = CreateRequest();
HttpRequest->SetURL(FString::Format(TEXT("{0}/streaming_upload_put"), { *UrlHttpTests() }));
HttpRequest->SetVerb(TEXT("PUT"));
HttpRequest->SetContentFromStream(Stream);
HttpRequest->SetHeader(TEXT("Content-Disposition"), TEXT("attachment;filename=TestStreamUpload.dat"));
HttpRequest->OnProcessRequestComplete().BindLambda([Stream, TotalSize](FHttpRequestPtr HttpRequest, FHttpResponsePtr HttpResponse, bool bSucceeded) {
CHECK(bSucceeded);
REQUIRE(HttpResponse != nullptr);
CHECK(HttpResponse->GetResponseCode() == 200);
CHECK(Stream->CurrentPos == TotalSize);
});
HttpRequest->ProcessRequest();
}
TEST_CASE_METHOD(FWaitUntilCompleteHttpFixture, "Streaming http upload from file by PUT can work well", HTTP_TAG)
{
FString Filename = FString(FPlatformProcess::UserSettingsDir()) / TEXT("TestStreamUpload.dat");
FArchive* RawFile = IFileManager::Get().CreateFileWriter(*Filename);
CHECK(RawFile != nullptr);
TSharedRef<FArchive> FileToWrite = MakeShareable(RawFile);
const uint64 FileSize = 5*1024*1024; // 5MB
char* FileData = (char*)FMemory::Malloc(FileSize);
FMemory::Memset(FileData, 'd', FileSize);
FileToWrite->Serialize(FileData, FileSize);
FileToWrite->FlushCache();
FileToWrite->Close();
FMemory::Free(FileData);
TSharedRef<IHttpRequest> HttpRequest = CreateRequest();
HttpRequest->SetURL(FString::Format(TEXT("{0}/streaming_upload_put"), { *UrlHttpTests() }));
HttpRequest->SetVerb(TEXT("PUT"));
HttpRequest->SetHeader(TEXT("Content-Disposition"), TEXT("attachment;filename=TestStreamUpload.dat"));
HttpRequest->SetContentAsStreamedFile(Filename);
HttpRequest->OnProcessRequestComplete().BindLambda([Filename](FHttpRequestPtr HttpRequest, FHttpResponsePtr HttpResponse, bool bSucceeded) {
CHECK(bSucceeded);
CHECK(HttpResponse->GetResponseCode() == 200);
IFileManager::Get().Delete(*Filename);
});
HttpRequest->ProcessRequest();
}
TEST_CASE_METHOD(FWaitUntilCompleteHttpFixture, "Redirect enabled by default and can work well", HTTP_TAG)
{
TSharedRef<IHttpRequest> HttpRequest = CreateRequest();
HttpRequest->SetURL(FString::Format(TEXT("{0}/redirect_from"), { *UrlHttpTests() }));
HttpRequest->SetVerb(TEXT("GET"));
HttpRequest->OnProcessRequestComplete().BindLambda([](FHttpRequestPtr HttpRequest, FHttpResponsePtr HttpResponse, bool bSucceeded) {
CHECK(bSucceeded);
CHECK(HttpResponse->GetResponseCode() == 200);
});
HttpRequest->ProcessRequest();
}
class FWaitUntilQuitFromTestFixture : public FWaitUntilCompleteHttpFixture
{
public:
FWaitUntilQuitFromTestFixture()
{
}
~FWaitUntilQuitFromTestFixture()
{
WaitUntilQuitFromTest();
}
void WaitUntilQuitFromTest()
{
while (!bQuitRequested)
{
HttpModule->GetHttpManager().Tick(TickFrequency);
FPlatformProcess::Sleep(TickFrequency);
}
}
bool bQuitRequested = false;
};
TEST_CASE_METHOD(FWaitUntilQuitFromTestFixture, "Http request can be reused", HTTP_TAG)
{
TSharedRef<IHttpRequest> HttpRequest = CreateRequest();
HttpRequest->SetURL(UrlToTestMethods());
HttpRequest->SetVerb(TEXT("POST"));
HttpRequest->OnProcessRequestComplete().BindLambda([this](FHttpRequestPtr HttpRequest, FHttpResponsePtr HttpResponse, bool bSucceeded) {
CHECK(bSucceeded);
CHECK(HttpResponse->GetResponseCode() == 200);
uint32 Chunks = 3;
uint32 ChunkSize = 1024;
HttpRequest->SetURL(UrlStreamDownload(Chunks, ChunkSize));
HttpRequest->SetVerb(TEXT("GET"));
HttpRequest->OnProcessRequestComplete().BindLambda([this, Chunks, ChunkSize](FHttpRequestPtr HttpRequest, FHttpResponsePtr HttpResponse, bool bSucceeded) {
CHECK(bSucceeded);
REQUIRE(HttpResponse != nullptr);
CHECK(HttpResponse->GetResponseCode() == 200);
CHECK(HttpResponse->GetContentLength() == Chunks * ChunkSize);
bQuitRequested = true;
});
HttpRequest->ProcessRequest();
});
HttpRequest->ProcessRequest();
}
// Response shared ptr should be able to be kept by user code and valid to access without http request
class FValidateResponseDependencyFixture : public FWaitUntilCompleteHttpFixture
{
public:
DECLARE_DELEGATE(FValidateResponseDependencyDelegate);
~FValidateResponseDependencyFixture()
{
WaitUntilAllHttpRequestsComplete();
ValidateResponseDependencyDelegate.ExecuteIfBound();
}
FValidateResponseDependencyDelegate ValidateResponseDependencyDelegate;
};
TEST_CASE_METHOD(FValidateResponseDependencyFixture, "Http query with parameters", HTTP_TAG)
{
struct FQueryWithParamsResponse : public FJsonSerializable
{
int32 VarInt;
FString VarStr;
BEGIN_JSON_SERIALIZER
JSON_SERIALIZE("var_int", VarInt);
JSON_SERIALIZE("var_str", VarStr);
END_JSON_SERIALIZER
};
TSharedRef<IHttpRequest> HttpRequest = HttpModule->CreateRequest();
FString UrlQueryWithParams = FString::Format(TEXT("{0}/query_with_params/?var_int=3&var_str=abc"), { *UrlHttpTests() });
HttpRequest->SetURL(UrlQueryWithParams);
HttpRequest->SetVerb(TEXT("GET"));
HttpRequest->OnProcessRequestComplete().BindLambda([this, UrlQueryWithParams](FHttpRequestPtr HttpRequest, FHttpResponsePtr HttpResponse, bool bSucceeded) {
CHECK(bSucceeded);
REQUIRE(HttpResponse != nullptr);
CHECK(HttpResponse->GetResponseCode() == 200);
CHECK(HttpRequest->GetURL() == UrlQueryWithParams);
FQueryWithParamsResponse QueryWithParamsResponse;
REQUIRE(QueryWithParamsResponse.FromJson(HttpResponse->GetContentAsString()));
CHECK(FString::FromInt(QueryWithParamsResponse.VarInt) == HttpRequest->GetURLParameter(TEXT("var_int")));
CHECK(QueryWithParamsResponse.VarStr == HttpRequest->GetURLParameter(TEXT("var_str")));
CHECK(FString::FromInt(QueryWithParamsResponse.VarInt) == HttpResponse->GetURLParameter(TEXT("var_int")));
CHECK(QueryWithParamsResponse.VarStr == HttpResponse->GetURLParameter(TEXT("var_str")));
ValidateResponseDependencyDelegate.BindLambda([HttpResponse, UrlQueryWithParams, QueryWithParamsResponse](){
// Validate all interfaces of http response can be called without accessing the destroyed http request
CHECK(HttpResponse->GetResponseCode() == 200);
CHECK(!HttpResponse->GetContent().IsEmpty());
CHECK(!HttpResponse->GetContentAsString().IsEmpty());
CHECK(HttpResponse->GetContentType() == TEXT("application/json"));
CHECK(HttpResponse->GetHeader("Content-Type") == TEXT("application/json"));
CHECK(!HttpResponse->GetAllHeaders().IsEmpty());
CHECK(HttpResponse->GetURL() == UrlQueryWithParams);
CHECK(HttpResponse->GetURLParameter(TEXT("var_int")) == FString::FromInt(QueryWithParamsResponse.VarInt));
CHECK(HttpResponse->GetURLParameter(TEXT("var_str")) == QueryWithParamsResponse.VarStr);
});
});
HttpRequest->ProcessRequest();
}
class FThreadedHttpRunnable : public FRunnable
{
public:
DECLARE_DELEGATE(FRunActualTestCodeDelegate);
FRunActualTestCodeDelegate& OnRunFromThread()
{
return ThreadCallback;
}
// FRunnable interface
virtual uint32 Run() override
{
ThreadCallback.ExecuteIfBound();
return 0;
}
void StartTestHttpThread(bool bBlockGameThread)
{
bBlockingGameThreadTick = bBlockGameThread;
RunnableThread = TSharedPtr<FRunnableThread>(FRunnableThread::Create(this, TEXT("Test Http Thread")));
while (bBlockingGameThreadTick)
{
float TickFrequency = 1.0f / 60; /*60 FPS*/;
FPlatformProcess::Sleep(TickFrequency);
}
}
void UnblockGameThread()
{
bBlockingGameThreadTick = false;
}
private:
FRunActualTestCodeDelegate ThreadCallback;
TSharedPtr<FRunnableThread> RunnableThread;
std::atomic<bool> bBlockingGameThreadTick = true;
};
class FWaitThreadedHttpFixture : public FWaitUntilCompleteHttpFixture
{
public:
~FWaitThreadedHttpFixture()
{
WaitUntilAllHttpRequestsComplete();
}
FThreadedHttpRunnable ThreadedHttpRunnable;
};
TEST_CASE_METHOD(FWaitThreadedHttpFixture, "Http streaming download request can work in non game thread", HTTP_TAG)
{
ThreadedHttpRunnable.OnRunFromThread().BindLambda([this]() {
TSharedRef<IHttpRequest> HttpRequest = CreateRequest();
HttpRequest->SetURL(UrlStreamDownload(3/*Chunks*/, 1024/*ChunkSize*/));
HttpRequest->SetVerb(TEXT("GET"));
HttpRequest->SetDelegateThreadPolicy(EHttpRequestDelegateThreadPolicy::CompleteOnHttpThread);
class FTestHttpReceiveStream final : public FArchive
{
public:
virtual void Serialize(void* V, int64 Length) override
{
// No matter what's the thread policy, Serialize always get called in http thread.
CHECK(!IsInGameThread());
}
};
CHECK(HttpRequest->SetResponseBodyReceiveStream(MakeShared<FTestHttpReceiveStream>()));
HttpRequest->OnProcessRequestComplete().BindLambda([this](FHttpRequestPtr HttpRequest, FHttpResponsePtr HttpResponse, bool bSucceeded) {
// EHttpRequestDelegateThreadPolicy::CompleteOnHttpThread was used, so not in game thread here
CHECK(!IsInGameThread());
CHECK(bSucceeded);
REQUIRE(HttpResponse != nullptr);
CHECK(HttpResponse->GetResponseCode() == 200);
CHECK(!HttpResponse->GetAllHeaders().IsEmpty());
ThreadedHttpRunnable.UnblockGameThread();
});
HttpRequest->ProcessRequest();
});
ThreadedHttpRunnable.StartTestHttpThread(true/*bBlockGameThread*/);
}
TEST_CASE_METHOD(FWaitThreadedHttpFixture, "Http download request progress callback can be received in http thread", HTTP_TAG)
{
std::atomic<bool> bRequestProgressTriggered = false;
ThreadedHttpRunnable.OnRunFromThread().BindLambda([this, &bRequestProgressTriggered]() {
TSharedRef<IHttpRequest> HttpRequest = CreateRequest();
HttpRequest->SetURL(UrlStreamDownload(10/*Chunks*/, 1024*1024/*ChunkSize*/));
HttpRequest->SetVerb(TEXT("GET"));
HttpRequest->SetDelegateThreadPolicy(EHttpRequestDelegateThreadPolicy::CompleteOnHttpThread);
HttpRequest->OnRequestProgress64().BindLambda([this, &bRequestProgressTriggered](FHttpRequestPtr Request, uint64 /*BytesSent*/, uint64 BytesReceived) {
if (!bRequestProgressTriggered)
{
// Only do these checks once, because when http request complete, this callback also get triggered
CHECK(BytesReceived > 0);
CHECK(BytesReceived < 10/*Chunks*/ * 1024*1024/*ChunkSize*/);
CHECK(!IsInGameThread());
CHECK(Request->GetStatus() == EHttpRequestStatus::Processing);
bRequestProgressTriggered = true;
}
});
HttpRequest->OnProcessRequestComplete().BindLambda([this](FHttpRequestPtr /*HttpRequest*/, FHttpResponsePtr /*HttpResponse */, bool bSucceeded) {
CHECK(bSucceeded);
ThreadedHttpRunnable.UnblockGameThread();
});
HttpRequest->ProcessRequest();
});
ThreadedHttpRunnable.StartTestHttpThread(true/*bBlockGameThread*/);
CHECK(bRequestProgressTriggered);
}
TEST_CASE_METHOD(FWaitUntilCompleteHttpFixture, "Http request pre check will fail", HTTP_TAG)
{
DisableWarningsInThisTest();
TSharedRef<IHttpRequest> HttpRequest = HttpModule->CreateRequest();
SECTION("when verb was set to empty")
{
HttpRequest->SetURL(UrlToTestMethods());
HttpRequest->SetVerb(TEXT(""));
}
SECTION("when url protocol is not valid")
{
HttpRequest->SetURL("http_abc://www.epicgames.com");
HttpRequest->SetVerb(TEXT("GET"));
}
SECTION("when url was not set")
{
HttpRequest->SetVerb(TEXT("GET"));
}
HttpRequest->OnProcessRequestComplete().BindLambda([](FHttpRequestPtr HttpRequest, FHttpResponsePtr HttpResponse, bool bSucceeded) {
CHECK(!bSucceeded);
});
HttpRequest->ProcessRequest();
}
namespace UE
{
namespace TestHttp
{
void SetupURLRequestFilter(FHttpModule* HttpModule)
{
// Pre check will fail when domain is not allowed
UE::Core::FURLRequestFilter::FRequestMap SchemeMap;
SchemeMap.Add(TEXT("http"), TArray<FString>{TEXT("epicgames.com")});
UE::Core::FURLRequestFilter Filter{SchemeMap};
HttpModule->GetHttpManager().SetURLRequestFilter(Filter);
}
}
}
// Pre-check failed requests won't be added into http manager, so it can't rely on the requested added/completed callback in FWaitUntilCompleteHttpFixture
TEST_CASE_METHOD(FWaitUntilQuitFromTestFixture, "Http request pre check will fail by thread policy", HTTP_TAG)
{
DisableWarningsInThisTest();
// Pre check will fail when domain is not allowed
UE::TestHttp::SetupURLRequestFilter(HttpModule);
TSharedRef<IHttpRequest> HttpRequest = CreateRequest();
HttpRequest->SetVerb(TEXT("GET"));
HttpRequest->SetURL(UrlToTestMethods());
SECTION("on game thread")
{
HttpRequest->OnProcessRequestComplete().BindLambda([this](FHttpRequestPtr HttpRequest, FHttpResponsePtr HttpResponse, bool bSucceeded) {
CHECK(IsInGameThread());
CHECK(!bSucceeded);
bQuitRequested = true;
});
}
SECTION("on http thread")
{
HttpRequest->SetDelegateThreadPolicy(EHttpRequestDelegateThreadPolicy::CompleteOnHttpThread);
HttpRequest->OnProcessRequestComplete().BindLambda([this](FHttpRequestPtr HttpRequest, FHttpResponsePtr HttpResponse, bool bSucceeded) {
CHECK(!IsInGameThread());
CHECK(!bSucceeded);
bQuitRequested = true;
});
}
HttpRequest->ProcessRequest();
}
class FWaitUnitilQuitFromTestThreadedFixture : public FWaitUntilQuitFromTestFixture
{
public:
~FWaitUnitilQuitFromTestThreadedFixture()
{
WaitUntilQuitFromTest();
}
FThreadedHttpRunnable ThreadedHttpRunnable;
};
// Pre-check failed requests won't be added into http manager, so it can't rely on the requested added/completed callback in FWaitUntilCompleteHttpFixture
TEST_CASE_METHOD(FWaitUnitilQuitFromTestThreadedFixture, "Threaded http request pre check will fail by thread policy", HTTP_TAG)
{
DisableWarningsInThisTest();
ThreadedHttpRunnable.OnRunFromThread().BindLambda([this]() {
// Pre check will fail when domain is not allowed
UE::TestHttp::SetupURLRequestFilter(HttpModule);
TSharedRef<IHttpRequest> HttpRequest = CreateRequest();
HttpRequest->SetVerb(TEXT("GET"));
HttpRequest->SetURL(UrlToTestMethods());
SECTION("on game thread")
{
HttpRequest->OnProcessRequestComplete().BindLambda([this](FHttpRequestPtr HttpRequest, FHttpResponsePtr HttpResponse, bool bSucceeded) {
CHECK(IsInGameThread());
CHECK(!bSucceeded);
bQuitRequested = true;
});
}
SECTION("on http thread")
{
HttpRequest->SetDelegateThreadPolicy(EHttpRequestDelegateThreadPolicy::CompleteOnHttpThread);
HttpRequest->OnProcessRequestComplete().BindLambda([this](FHttpRequestPtr HttpRequest, FHttpResponsePtr HttpResponse, bool bSucceeded) {
CHECK(!IsInGameThread());
CHECK(!bSucceeded);
bQuitRequested = true;
});
}
HttpRequest->ProcessRequest();
});
ThreadedHttpRunnable.StartTestHttpThread(false/*bBlockGameThread*/);
}
TEST_CASE_METHOD(FWaitUntilCompleteHttpFixture, "Cancel http request connect before timeout", HTTP_TAG)
{
DisableWarningsInThisTest();
TSharedRef<IHttpRequest, ESPMode::ThreadSafe> HttpRequest = CreateRequest();
HttpRequest->SetURL(UrlWithInvalidPortToTestConnectTimeout());
HttpRequest->SetVerb(TEXT("GET"));
HttpRequest->SetTimeout(7);
FDateTime StartTime = FDateTime::Now();
HttpRequest->OnProcessRequestComplete().BindLambda([StartTime](FHttpRequestPtr HttpRequest, FHttpResponsePtr HttpResponse, bool bSucceeded) {
CHECK(!bSucceeded);
FTimespan Timespan = FDateTime::Now() - StartTime;
float DurationInSeconds = Timespan.GetTotalSeconds();
CHECK(DurationInSeconds < 2);
});
HttpRequest->ProcessRequest();
FPlatformProcess::Sleep(0.5);
HttpRequest->CancelRequest();
}
class FThreadedBatchRequestsFixture : public FWaitThreadedHttpFixture
{
public:
void LaunchBatchRequests(uint32 BatchSize)
{
for (uint32 i = 0; i < BatchSize; ++i)
{
TSharedRef<IHttpRequest> HttpRequest = CreateRequest();
HttpRequest->SetURL(UrlStreamDownload(3, 1024*1024));
HttpRequest->SetVerb(TEXT("GET"));
HttpRequest->ProcessRequest();
}
}
void BlockUntilFlushed()
{
if (bRetryEnabled)
{
HttpRetryManager->BlockUntilFlushed(5.0);
}
else
{
HttpModule->GetHttpManager().Flush(EHttpFlushReason::Default);
}
}
};
TEST_CASE_METHOD(FThreadedBatchRequestsFixture, "Retry manager and http manager is thread safe", HTTP_TAG)
{
DisableWarningsInThisTest();
ThreadedHttpRunnable.OnRunFromThread().BindLambda([this]() {
LaunchBatchRequests(10);
BlockUntilFlushed();
});
ThreadedHttpRunnable.StartTestHttpThread(false/*bBlockGameThread*/);
LaunchBatchRequests(10);
BlockUntilFlushed();
}
// TODO: Add cancel test, with multiple cancel calls
// TODO: Add test case to validate header received callback can be received in http/game thread, and can be received before the request complete