// Copyright 1998-2019 Epic Games, Inc. All Rights Reserved. #include "AmdAmfVideoEncoder.h" #if PLATFORM_WINDOWS #include "GameplayMediaEncoderSample.h" #include "ScreenRendering.h" #include "ShaderCore.h" #include "RendererInterface.h" #include "RHIStaticStates.h" #include "PipelineStateCache.h" #include "Modules/ModuleManager.h" #include "CommonRenderResources.h" GAMEPLAYMEDIAENCODER_START DEFINE_LOG_CATEGORY(AmdAmf); FAmdAmfVideoEncoder::FAmdAmfVideoEncoder(const FOutputSampleCallback& OutputCallback) : FBaseVideoEncoder(OutputCallback) {} FAmdAmfVideoEncoder::~FAmdAmfVideoEncoder() { // BufferedFrames keep references to AMFData, we need to release them before destroying AMF for (FFrame& Frame : BufferedFrames) { Frame.EncodedData = nullptr; } // Cleanup in this order if (AmfEncoder) { AmfEncoder->Terminate(); AmfEncoder = nullptr; } if (AmfContext) { AmfContext->Terminate(); AmfContext = nullptr; } AmfFactory = nullptr; if (DllHandle) { FPlatformProcess::FreeDllHandle(DllHandle); DllHandle = nullptr; } } bool FAmdAmfVideoEncoder::Initialize(const FVideoEncoderConfig& InConfig) { if (bInitialized) { UE_LOG(AmdAmf, Error, TEXT("Encoder already initialized. Re-initialization is not supported. Instead recreate the instance.")); return false; } UE_LOG(AmdAmf, Log, TEXT("VideoEncoder config: %dx%d, %d FPS, %.2f Mbps"), InConfig.Width, InConfig.Height, InConfig.Framerate, InConfig.Bitrate / 1000000.0f); if (!FBaseVideoEncoder::Initialize(InConfig)) { return false; } DllHandle = FPlatformProcess::GetDllHandle(AMF_DLL_NAME); if (DllHandle == nullptr) { return false; } AMFInit_Fn AmfInitFn = (AMFInit_Fn)FPlatformProcess::GetDllExport((HMODULE)DllHandle, TEXT(AMF_INIT_FUNCTION_NAME)); if (AmfInitFn == nullptr) { return false; } CHECK_AMF_RET(AmfInitFn(AMF_FULL_VERSION, &AmfFactory)); AMFQueryVersion_Fn AmfVersionFun = (AMFQueryVersion_Fn)FPlatformProcess::GetDllExport((HMODULE)DllHandle, TEXT(AMF_QUERY_VERSION_FUNCTION_NAME)); if (AmfVersionFun == nullptr) { return false; } uint64 AmfVersion = 0; AmfVersionFun(&AmfVersion); CHECK_AMF_RET(AmfFactory->CreateContext(&AmfContext)); CHECK_AMF_RET(AmfContext->InitDX11(GetUE4DxDevice())); CHECK_AMF_RET(AmfFactory->CreateComponent(AmfContext, AMFVideoEncoderVCE_AVC, &AmfEncoder)); CHECK_AMF_RET(AmfEncoder->SetProperty(AMF_VIDEO_ENCODER_USAGE, /*AMF_VIDEO_ENCODER_USAGE_ULTRA_LOW_LATENCY*/AMF_VIDEO_ENCODER_USAGE_TRANSCONDING)); CHECK_AMF_RET(AmfEncoder->SetProperty(AMF_VIDEO_ENCODER_PROFILE, AMF_VIDEO_ENCODER_PROFILE_MAIN)); //CHECK_AMF_RET(AmfEncoder->SetProperty(AMF_VIDEO_ENCODER_B_PIC_PATTERN, 0)); CHECK_AMF_RET(AmfEncoder->SetProperty(AMF_VIDEO_ENCODER_QUALITY_PRESET, AMF_VIDEO_ENCODER_QUALITY_PRESET_QUALITY)); CHECK_AMF_RET(AmfEncoder->SetProperty(AMF_VIDEO_ENCODER_TARGET_BITRATE, InConfig.Bitrate)); CHECK_AMF_RET(AmfEncoder->SetProperty(AMF_VIDEO_ENCODER_FRAMESIZE, ::AMFConstructSize(InConfig.Width, InConfig.Height))); CHECK_AMF_RET(AmfEncoder->SetProperty(AMF_VIDEO_ENCODER_FRAMERATE, ::AMFConstructRate(InConfig.Framerate, 1))); // generate key-frames every second, useful for seeking in resulting .mp4 and keeping recording ring buffer // of second-precise duration uint64 IdrPeriod = InConfig.Framerate; CHECK_AMF_RET(AmfEncoder->SetProperty(AMF_VIDEO_ENCODER_IDR_PERIOD, IdrPeriod)); // insert SPS/PPS before every key-frame. .mp4 file video stream must start from SPS/PPS. Their size is // negligible so having them before every key-frame is not an issue but they presence simplifies // implementation significantly. Otherwise we would extract SPS/PPS from the first key-frame and store // them manually at the beginning of resulting .mp4 file CHECK_AMF_RET(AmfEncoder->SetProperty(AMF_VIDEO_ENCODER_HEADER_INSERTION_SPACING, IdrPeriod)); CHECK_AMF_RET(AmfEncoder->Init(amf::AMF_SURFACE_RGBA, InConfig.Width, InConfig.Height)); //LogAmfPropertyStorage(AmfEncoder); for (uint32 i = 0; i < NumBufferedFrames; ++i) { FFrame& Frame = BufferedFrames[i]; ResetResolvedBackBuffer(Frame); } UE_LOG(AmdAmf, Log, TEXT("AMF H.264 encoder initialised, v.0x%X"), AmfVersion); bInitialized = true; return true; } bool FAmdAmfVideoEncoder::Start() { return true; } void FAmdAmfVideoEncoder::Stop() { } bool FAmdAmfVideoEncoder::SetBitrate(uint32 Bitrate) { // update `Config` and `OutputType` if (!FBaseVideoEncoder::SetBitrate(Bitrate)) { return false; } // reconfigure AMF CHECK_AMF_RET(AmfEncoder->SetProperty(AMF_VIDEO_ENCODER_TARGET_BITRATE, Bitrate)); return true; } bool FAmdAmfVideoEncoder::SetFramerate(uint32 Framerate) { // update `Config` and `OutputType` if (!FBaseVideoEncoder::SetFramerate(Framerate)) { return false; } // reconfigure AMF CHECK_AMF_RET(AmfEncoder->SetProperty(AMF_VIDEO_ENCODER_FRAMERATE, ::AMFConstructRate(Framerate, 1))); return true; } DECLARE_CYCLE_STAT(TEXT("Process"), STAT_FAmdAmfVideoEncoder_Process, STATGROUP_AmdAmfVideoEncoder); bool FAmdAmfVideoEncoder::Process(const FTexture2DRHIRef& Texture, FTimespan Timestamp, FTimespan Duration) { SCOPE_CYCLE_COUNTER(STAT_FAmdAmfVideoEncoder_Process); check(IsInRenderingThread()); // first process output to free reused instances of input frames return ProcessOutput() && ProcessInput(Texture, Timestamp, Duration); } DECLARE_CYCLE_STAT(TEXT("ProcessInput"), STAT_FAmdAmfVideoEncoder_ProcessInput, STATGROUP_AmdAmfVideoEncoder); bool FAmdAmfVideoEncoder::ProcessInput(const FTexture2DRHIRef& Texture, FTimespan Timestamp, FTimespan Duration) // prepare frame and submit to encoder { SCOPE_CYCLE_COUNTER(STAT_FAmdAmfVideoEncoder_ProcessInput); UE_LOG(AmdAmf, VeryVerbose, TEXT("frame #%d input"), InputFrameCount); uint32 BufferIndex = InputFrameCount % NumBufferedFrames; FFrame& Frame = BufferedFrames[BufferIndex]; if (Frame.bEncoding) { UE_LOG(AmdAmf, Verbose, TEXT("Dropped frame because encoder is lagging")); return true; } Frame.bEncoding = true; ResolveBackBuffer(Texture, Frame.ResolvedBackBuffer); Frame.FrameIdx = InputFrameCount; Frame.Timestamp = Timestamp; Frame.Duration = Duration; // ResolveBackBuffer can be asynchronous (executed by RHI Command List, so we need to do the same // for submitting frame to encoder ExecuteRHICommand([this, &Frame]() { SubmitFrameToEncoder(Frame); }); ++InputFrameCount; return true; } DECLARE_CYCLE_STAT(TEXT("SubmitFrameToEncoder"), STAT_FAmdAmfVideoEncoder_SubmitFrameToEncoder, STATGROUP_AmdAmfVideoEncoder); DECLARE_CYCLE_STAT(TEXT("AmfEncoder->SubmitInput"), STAT_FAmdAmfVideoEncoder_AmfEncoder_SubmitInput, STATGROUP_AmdAmfVideoEncoder); bool FAmdAmfVideoEncoder::SubmitFrameToEncoder(FFrame& Frame) { SCOPE_CYCLE_COUNTER(STAT_FAmdAmfVideoEncoder_SubmitFrameToEncoder); amf::AMFSurfacePtr AmfSurfaceIn; ID3D11Texture2D* ResolvedBackBufferDX11 = static_cast(GetD3D11TextureFromRHITexture(Frame.ResolvedBackBuffer)->GetResource()); CHECK_AMF_RET(AmfContext->CreateSurfaceFromDX11Native(ResolvedBackBufferDX11, &AmfSurfaceIn, nullptr)); { // if `-d3ddebug` is enabled `SubmitInput` crashes with DX11 error, see output window // we believe it's an internal AMF shader problem so we disable those errors explicitly, otherwise // DX Debug Layer can't be used at all FScopeDisabledDxDebugErrors Errors({ D3D11_MESSAGE_ID_DEVICE_UNORDEREDACCESSVIEW_RETURN_TYPE_MISMATCH, D3D11_MESSAGE_ID_DEVICE_CSSETUNORDEREDACCESSVIEWS_TOOMANYVIEWS, }); { SCOPE_CYCLE_COUNTER(STAT_FAmdAmfVideoEncoder_AmfEncoder_SubmitInput); CHECK_AMF_RET(AmfEncoder->SubmitInput(AmfSurfaceIn)); } } return true; } DECLARE_CYCLE_STAT(TEXT("QueryEncoderOutput"), STAT_FAmdAmfVideoEncoder_QueryEncoderOutput, STATGROUP_AmdAmfVideoEncoder); DECLARE_CYCLE_STAT(TEXT("AmfEncoder->QueryOutput"), STAT_FAmdAmfVideoEncoder_AmfEncoder_QueryOutput, STATGROUP_AmdAmfVideoEncoder); bool FAmdAmfVideoEncoder::ProcessOutput() // check if output is ready and handle it { SCOPE_CYCLE_COUNTER(STAT_FAmdAmfVideoEncoder_QueryEncoderOutput); check(IsInRenderingThread()); // more than one output frame can be ready while (BufferedFrames[OutputFrameCount % NumBufferedFrames].bEncoding) { amf::AMFDataPtr EncodedData; AMF_RESULT Ret; { SCOPE_CYCLE_COUNTER(STAT_FAmdAmfVideoEncoder_AmfEncoder_QueryOutput); Ret = AmfEncoder->QueryOutput(&EncodedData); } if (Ret == AMF_OK && EncodedData != nullptr) { UE_LOG(AmdAmf, VeryVerbose, TEXT("frame #%d encoded"), OutputFrameCount); FFrame& Frame = BufferedFrames[OutputFrameCount % NumBufferedFrames]; check(Frame.bEncoding); checkf(OutputFrameCount == Frame.FrameIdx, TEXT("%d - %d"), OutputFrameCount, Frame.FrameIdx); Frame.EncodedData = EncodedData; if (!HandleEncodedFrame(Frame)) { return false; } ++OutputFrameCount; } else if (Ret == AMF_REPEAT) { break; // not ready yet } else { UE_LOG(AmdAmf, Error, TEXT("Failed to query AMF H.264 Encoder output: %d, %p"), Ret, EncodedData.GetPtr()); return false; } } return true; } DECLARE_CYCLE_STAT(TEXT("ProcessEncodedFrame"), STAT_FAmdAmfVideoEncoder_ProcessEncodedFrame, STATGROUP_AmdAmfVideoEncoder); bool FAmdAmfVideoEncoder::HandleEncodedFrame(FFrame& Frame) { SCOPE_CYCLE_COUNTER(STAT_FAmdAmfVideoEncoder_ProcessEncodedFrame); checkf(Frame.bEncoding && Frame.EncodedData != nullptr, TEXT("Internal error: %d, %p"), static_cast(Frame.bEncoding), Frame.EncodedData.GetPtr()); if (!Frame.bEncoding || Frame.EncodedData == nullptr) { return false; } // Query for buffer interface amf::AMFBufferPtr EncodedBuffer(Frame.EncodedData); void* EncodedBufferPtr = EncodedBuffer->GetNative(); size_t EncodedBufferSize = EncodedBuffer->GetSize(); // Retrieve encoded frame from AMFBuffer and copy to IMFMediaBuffer TRefCountPtr MediaBuffer; CHECK_HR(MFCreateMemoryBuffer(EncodedBufferSize, MediaBuffer.GetInitReference())); CHECK_HR(MediaBuffer->SetCurrentLength(EncodedBufferSize)); BYTE* MediaBufferData = nullptr; DWORD MediaBufferLength = 0; MediaBuffer->Lock(&MediaBufferData, nullptr, &MediaBufferLength); FMemory::Memcpy(MediaBufferData, EncodedBufferPtr, EncodedBufferSize); MediaBuffer->Unlock(); FGameplayMediaEncoderSample OutputSample{ EMediaType::Video }; if (!OutputSample.CreateSample()) { return false; } CHECK_HR(OutputSample.GetSample()->AddBuffer(MediaBuffer)); OutputSample.SetTime(Frame.Timestamp); OutputSample.SetDuration(Frame.Duration); // mark as a key-frame (if it is) uint64 OutputFrameType; CHECK_AMF_RET(EncodedBuffer->GetProperty(AMF_VIDEO_ENCODER_OUTPUT_DATA_TYPE, &OutputFrameType)); CHECK_HR(OutputSample.GetSample()->SetUINT32(MFSampleExtension_CleanPoint, OutputFrameType == AMF_VIDEO_ENCODER_OUTPUT_DATA_TYPE_IDR ? 1 : 0)); UE_LOG(AmdAmf, Verbose, TEXT("encoded frame #%d: time %.3f, duration %.3f, size %d, type %d"), Frame.FrameIdx, OutputSample.GetTime().GetTotalSeconds(), OutputSample.GetDuration().GetTotalSeconds(), EncodedBufferSize, OutputFrameType); // only now when we're done dealing with encoded data we can "release" this frame // to be reused for encoding input Frame.EncodedData = nullptr; Frame.bEncoding = false; OutputCallback(OutputSample); return true; } void FAmdAmfVideoEncoder::ResetResolvedBackBuffer(FFrame& Frame) { Frame.ResolvedBackBuffer.SafeRelease(); // Make sure format used here is compatible with AMF_SURFACE_FORMAT specified in encoder Init() function. FRHIResourceCreateInfo CreateInfo; Frame.ResolvedBackBuffer = RHICreateTexture2D(Config.Width, Config.Height, EPixelFormat::PF_R8G8B8A8, 1, 1, TexCreate_RenderTargetable, CreateInfo); } DECLARE_CYCLE_STAT(TEXT("ResolveBackBuffer"), STAT_FAmdAmfVideoEncoder_ResolveBackBuffer, STATGROUP_AmdAmfVideoEncoder); void FAmdAmfVideoEncoder::ResolveBackBuffer(const FTexture2DRHIRef& BackBuffer, const FTexture2DRHIRef& ResolvedBackBuffer) { SCOPE_CYCLE_COUNTER(STAT_FAmdAmfVideoEncoder_ResolveBackBuffer); IRendererModule* RendererModule = &FModuleManager::GetModuleChecked("Renderer"); FRHICommandListImmediate& RHICmdList = FRHICommandListExecutor::GetImmediateCommandList(); if (BackBuffer->GetFormat() == ResolvedBackBuffer->GetFormat() && BackBuffer->GetSizeXY() == ResolvedBackBuffer->GetSizeXY()) { RHICmdList.CopyToResolveTarget(BackBuffer, ResolvedBackBuffer, FResolveParams()); } else // Texture format mismatch, use a shader to do the copy. { PRAGMA_DISABLE_DEPRECATION_WARNINGS SetRenderTarget(RHICmdList, ResolvedBackBuffer, FTextureRHIRef()); PRAGMA_ENABLE_DEPRECATION_WARNINGS RHICmdList.SetViewport(0, 0, 0.0f, ResolvedBackBuffer->GetSizeX(), ResolvedBackBuffer->GetSizeY(), 1.0f); FGraphicsPipelineStateInitializer GraphicsPSOInit; RHICmdList.ApplyCachedRenderTargets(GraphicsPSOInit); GraphicsPSOInit.BlendState = TStaticBlendState<>::GetRHI(); GraphicsPSOInit.RasterizerState = TStaticRasterizerState<>::GetRHI(); GraphicsPSOInit.DepthStencilState = TStaticDepthStencilState::GetRHI(); TShaderMap* ShaderMap = GetGlobalShaderMap(GMaxRHIFeatureLevel); TShaderMapRef VertexShader(ShaderMap); TShaderMapRef PixelShader(ShaderMap); GraphicsPSOInit.BoundShaderState.VertexDeclarationRHI = GFilterVertexDeclaration.VertexDeclarationRHI; GraphicsPSOInit.BoundShaderState.VertexShaderRHI = GETSAFERHISHADER_VERTEX(*VertexShader); GraphicsPSOInit.BoundShaderState.PixelShaderRHI = GETSAFERHISHADER_PIXEL(*PixelShader); GraphicsPSOInit.PrimitiveType = PT_TriangleList; SetGraphicsPipelineState(RHICmdList, GraphicsPSOInit); if (ResolvedBackBuffer->GetSizeX() != BackBuffer->GetSizeX() || ResolvedBackBuffer->GetSizeY() != BackBuffer->GetSizeY()) PixelShader->SetParameters(RHICmdList, TStaticSamplerState::GetRHI(), BackBuffer); else PixelShader->SetParameters(RHICmdList, TStaticSamplerState::GetRHI(), BackBuffer); RendererModule->DrawRectangle( RHICmdList, 0, 0, // Dest X, Y ResolvedBackBuffer->GetSizeX(), // Dest Width ResolvedBackBuffer->GetSizeY(), // Dest Height 0, 0, // Source U, V 1, 1, // Source USize, VSize ResolvedBackBuffer->GetSizeXY(), // Target buffer size FIntPoint(1, 1), // Source texture size *VertexShader, EDRF_Default); UnbindRenderTargets(RHICmdList); } } GAMEPLAYMEDIAENCODER_END #endif