izzy/UnrealEngineUWP
0
0
Fork 0
mirror of https://github.com/izzy2lost/UnrealEngineUWP.git synced 2026-03-26 18:15:20 -07:00
Code Issues Packages Projects Releases Wiki Activity

Fix: Stuttering and latency issues on Mac VCam Pixel Streaming.

Browse Source 
- VTCodecs configure low latency mode, infinite GOP, no bframes etc, all of which is ideal for WebRTC streaming.
- VTCodecs enable temporal encoding (e.g. delta frames) this will yield better compression.
- VTCodecs add support for setting Min and Max QP values which we use to bound encoding quality.
- PixelStreamingVCam set keyframe interval on Mac to a very large number to effectively disable sending keyframes, keyframes add additional latency and are not needed beyond the first frame in LAN streaming.

#rb Luke.Bermingham
[FYI] William.Belcher, Thomas.Kilkenny

[CL 31169798 by luke bermingham in ue5-main branch]
This commit is contained in:
luke bermingham
2024-02-05 07:21:12 -05:00
parent e201946e11
commit 2f070878d2
2 changed files with 38 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Engine/Plugins/Experimental/AVCodecs/VTCodecs/Source/VTCodecs/Public/Video/Encoders/VideoEncoderVT.hpp
+31 -17
View File
@@ -119,12 +119,31 @@ FAVResult TVideoEncoderVT<TResource>::ApplyConfig()
CONDITIONAL_RELEASE(IOSurfaceValue);
CONDITIONAL_RELEASE(PixelFormat);
// Encoder specifications
const size_t EncoderSpecsSize = 2;
CFTypeRef EncoderKeys[AttributesSize] =
{
// We want HW acceleration for best latency, if we can't get it then fail creating the session
kVTVideoEncoderSpecification_RequireHardwareAcceleratedVideoEncoder,
// Low latency mode is ideal for WebRTC, infinite GOP, no bframes, temporal layer structure etc
kVTVideoEncoderSpecification_EnableLowLatencyRateControl
};
CFTypeRef EncoderValues[EncoderSpecsSize] =
{
kCFBooleanTrue,
kCFBooleanTrue
};
CFDictionaryRef EncoderSpec = CFDictionaryCreate(kCFAllocatorDefault, EncoderKeys, EncoderValues, EncoderSpecsSize, &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks);
OSStatus Result = VTCompressionSessionCreate(kCFAllocatorDefault,
PendingConfig.Width,
PendingConfig.Height,
PendingConfig.Codec,
NULL,
EncoderSpec,
SourceAttributes,
NULL /* Default Compressed Data Allocator */,
Internal::VTCompressionOutputCallback,
@@ -132,25 +151,13 @@ FAVResult TVideoEncoderVT<TResource>::ApplyConfig()
&Encoder);
CONDITIONAL_RELEASE(SourceAttributes);
CONDITIONAL_RELEASE(EncoderSpec);
if(Result != 0)
{
return FAVResult(EAVResult::ErrorCreating, TEXT("Failed to create VTCompressionSession"), TEXT("VT"), Result);
}
CFBooleanRef bIsUsingHardwareEncoder;
Result = VTSessionCopyProperty(Encoder, kVTCompressionPropertyKey_UsingHardwareAcceleratedVideoEncoder, NULL, &bIsUsingHardwareEncoder);
if(Result != 0)
{
FAVResult::Log(EAVResult::Warning, TEXT("VTSessionCopyProperty(kVTCompressionPropertyKey_UsingHardwareAcceleratedVideoEncoder) failed"), TEXT("VT"), Result);
}
else
{
FAVResult::Log(EAVResult::Warning, FString::Printf(TEXT("Created compression session. UsingHardwareEncoder: %s"), (CFBooleanGetValue(bIsUsingHardwareEncoder) ? TEXT("TRUE") : TEXT("FALSE"))), TEXT("VT"));
}
CONDITIONAL_RELEASE(bIsUsingHardwareEncoder);
ConfigureCompressionSession(PendingConfig);
}
}
@@ -170,15 +177,21 @@ FAVResult TVideoEncoderVT<TResource>::ConfigureCompressionSession(FVideoEncoderC
SetEncoderBitrate(Config);
VTSessionHelpers::SetVTSessionProperty(Encoder, kVTCompressionPropertyKey_AllowTemporalCompression, false);
// Enable temporal compression, e.g. creating delta frames
VTSessionHelpers::SetVTSessionProperty(Encoder, kVTCompressionPropertyKey_AllowTemporalCompression, true);
// Frame reordering is unhelpful in WebRTC, real-time streaming usecases
VTSessionHelpers::SetVTSessionProperty(Encoder, kVTCompressionPropertyKey_AllowFrameReordering, false);
VTSessionHelpers::SetVTSessionProperty(Encoder, kVTCompressionPropertyKey_MaxKeyFrameInterval, (int32_t)Config.KeyframeInterval);
VTSessionHelpers::SetVTSessionProperty(Encoder, kVTCompressionPropertyKey_MaxKeyFrameIntervalDuration, (Config.KeyframeInterval > 0 ? (int32_t)(Config.KeyframeInterval / Config.FrameRate) : 0));
VTSessionHelpers::SetVTSessionProperty(Encoder, kVTCompressionPropertyKey_MaxKeyFrameInterval, (int32_t)Config.KeyframeInterval);
VTSessionHelpers::SetVTSessionProperty(Encoder, kVTCompressionPropertyKey_MaximizePowerEfficiency, false);
VTSessionHelpers::SetVTSessionProperty(Encoder, kVTCompressionPropertyKey_MinAllowedFrameQP, (int32_t)Config.MinQP);
VTSessionHelpers::SetVTSessionProperty(Encoder, kVTCompressionPropertyKey_MaxAllowedFrameQP, (int32_t)Config.MaxQP);
if(Config.Codec == kCMVideoCodecType_H264)
{
@@ -212,7 +225,8 @@ FAVResult TVideoEncoderVT<TResource>::SendFrame(TSharedPtr<FVideoResourceMetal>
if(Resource.IsValid())
{
CMTime PresentationTime = CMTimeMake(Timestamp, 1000000);
float TimestampSecs = Timestamp / 1000000.0f;
CMTime PresentationTime = CMTimeMakeWithSeconds(TimestampSecs, NSEC_PER_SEC);
CFDictionaryRef FrameProperties = nullptr;
if (bForceKeyframe)
{
Engine/Plugins/VirtualProduction/VirtualCameraCore/Source/PixelStreamingVCam/Private/PixelStreamingVCamModule.cpp
+7 -2
View File
@@ -81,13 +81,18 @@ namespace UE::PixelStreamingVCam::Private
{
CaptureUseFence->Set(false);
}
// Disable keyframes interval
if (IConsoleVariable* KeyframeInterval = IConsoleManager::Get().FindConsoleVariable(TEXT("PixelStreaming.Encoder.KeyframeInterval")))
{
#if PLATFORM_MAC
// On Mac the VideoToolbox encoder has no concept of no key frames, so we set it a large value.
KeyframeInterval->Set(100000000);
#else
KeyframeInterval->Set(0);
#endif
}
// Set a fixed target bitrate (this way quality and network transmission should be bounded).
// We want this network transmission to be as consistent as possible so that the jitter buffer on the recv side
// stays well bounded and shrinks to its optimal value and stays there.