Files
UnrealEngineUWP/Engine/Source/Developer/ShaderCompilerCommon/Public/ShaderCompilerCommon.h
christopher waters ae1bd451f9 Bindless Resources
- Adding correct handling for bindless uniform buffer parameters.
- Reworking RHITextureReference creation to allow the RHI to create them differently, especially with bindless views.
- Adding RHITextureReference bindless view updates to D3D12RHI.

#jira UE-162014
#rb mihnea.balta, jeannoe.morissette
#preflight 642dba03c6769c6082f592f6

[CL 24936647 by christopher waters in ue5-main branch]
2023-04-05 17:03:17 -04:00

549 lines
19 KiB
C++

// Copyright Epic Games, Inc. All Rights Reserved.
#pragma once
#include "CoreMinimal.h"
#include "ShaderCore.h"
#include "ShaderCompilerCore.h"
#include "CrossCompilerDefinitions.h"
#include "ShaderConductorContext.h"
#if UE_ENABLE_INCLUDE_ORDER_DEPRECATED_IN_5_1
#include "ShaderParameterParser.h"
#endif
#include "Templates/Function.h"
class FShaderParameterParser;
namespace UE::ShaderCompilerCommon
{
static constexpr const TCHAR* kUniformBufferConstantBufferPrefix = TEXT("UniformBufferConstants_");
static constexpr const TCHAR* kBindlessResourcePrefix = TEXT("BindlessResource_");
static constexpr const TCHAR* kBindlessSamplerPrefix = TEXT("BindlessSampler_");
}
/**
* This function looks for resources specified in ResourceTableMap in the
* parameter map, adds them to the resource table, and removes them from the
* parameter map. If a resource is used from a currently unmapped uniform
* buffer we allocate a slot for it from UsedUniformBufferSlots.
* Returns false if there's any internal error.
*/
extern SHADERCOMPILERCOMMON_API bool BuildResourceTableMapping(
const TMap<FString,FResourceTableEntry>& ResourceTableMap,
const TMap<FString,FUniformBufferEntry>& UniformBufferMap,
TBitArray<>& UsedUniformBufferSlots,
FShaderParameterMap& ParameterMap,
FShaderCompilerResourceTable& OutSRT
);
/** Culls global uniform buffer entries from the parameter map. */
extern SHADERCOMPILERCOMMON_API void CullGlobalUniformBuffers(const TMap<FString, FUniformBufferEntry>& UniformBufferMap, FShaderParameterMap& ParameterMap);
/**
* Builds a token stream out of the resource map. The resource map is one
* of the arrays generated by BuildResourceTableMapping. The token stream
* is used at runtime to gather resources from tables and bind them to the
* appropriate slots.
*/
extern SHADERCOMPILERCOMMON_API void BuildResourceTableTokenStream(
const TArray<uint32>& InResourceMap,
int32 MaxBoundResourceTable,
TArray<uint32>& OutTokenStream,
bool bGenerateEmptyTokenStreamIfNoResources = false
);
// Finds the number of used uniform buffers in a resource map
extern SHADERCOMPILERCOMMON_API int16 GetNumUniformBuffersUsed(const FShaderCompilerResourceTable& InSRT);
namespace UE::ShaderCompilerCommon
{
extern SHADERCOMPILERCOMMON_API bool ShouldUseStableConstantBuffer(const FShaderCompilerInput& Input);
extern SHADERCOMPILERCOMMON_API EShaderParameterType ParseParameterType(FStringView InType, TArrayView<const TCHAR* const> InExtraSRVTypes, TArrayView<const TCHAR* const> InExtraUAVTypes);
extern SHADERCOMPILERCOMMON_API FStringView RemoveConstantBufferPrefix(FStringView InName);
extern SHADERCOMPILERCOMMON_API FString RemoveConstantBufferPrefix(const FString& InName);
extern SHADERCOMPILERCOMMON_API EShaderParameterType ParseAndRemoveBindlessParameterPrefix(FStringView& InName);
extern SHADERCOMPILERCOMMON_API EShaderParameterType ParseAndRemoveBindlessParameterPrefix(FString& InName);
extern SHADERCOMPILERCOMMON_API bool RemoveBindlessParameterPrefix(FString& InName);
extern SHADERCOMPILERCOMMON_API bool ValidatePackedResourceCounts(FShaderCompilerOutput& Output, const FShaderCodePackedResourceCounts& PackedResourceCounts);
/*
* Parses ray tracing shader entry point specification string in one of the following formats:
* 1) Verbatim single entry point name, e.g. "MainRGS"
* 2) Complex entry point for ray tracing hit group shaders:
* a) "closesthit=MainCHS"
* b) "closesthit=MainCHS anyhit=MainAHS"
* c) "closesthit=MainCHS anyhit=MainAHS intersection=MainIS"
* d) "closesthit=MainCHS intersection=MainIS"
* NOTE: closesthit attribute must always be provided for complex hit group entry points
*/
extern SHADERCOMPILERCOMMON_API void ParseRayTracingEntryPoint(const FString& Input, FString& OutMain, FString& OutAnyHit, FString& OutIntersection);
/**
* Rewrites a fully preprocessed shader source code, removing any functions or structs that are not reachable from a given entry point or list of symbols.
* This is a high-level wrapper for UE::ShaderMinifier that should be used in all shader compiler back-ends for consistency.
* Any errors encountered during parsing or minification are added to OutErrors.
* InOutPreprocessedShaderSource is replaced with the rewritten code on success and is kept intact on failure.
*/
extern SHADERCOMPILERCOMMON_API bool RemoveDeadCode(FString& InOutPreprocessedShaderSource, const FString& EntryPoint, TArray<FShaderCompilerError>& OutErrors);
extern SHADERCOMPILERCOMMON_API bool RemoveDeadCode(FString& InOutPreprocessedShaderSource, TConstArrayView<FStringView> RequiredSymbols, TArray<FShaderCompilerError>& OutErrors);
struct FDebugShaderDataOptions
{
uint32 HlslCCFlags = 0;
const TCHAR* OverrideBaseFilename = nullptr;
const TCHAR* FilenamePrefix = nullptr;
TFunction<FString()> AppendPreSource{};
TFunction<FString()> AppendPostSource{};
bool bSkipDirectCompileTxt = false;
};
/*
* Dumps common debug information (preprocessed .usf as constructed by GetDebugShaderContents, and a directcompile.txt file
* containing the commandline for launching ShaderCompileWorker manually) for the given shader compile input
* and preprocessed source according to the options provided.
* @param Input The input of the compilation job
* @param PreprocessedSource The unmodified preprocessed source (used as input to the compilation)
* @param Options Options which can change behaviour of the debug dump; see above.
*/
extern SHADERCOMPILERCOMMON_API void DumpDebugShaderData(const FShaderCompilerInput& Input, const FString& PreprocessedSource, const FDebugShaderDataOptions& Options = FDebugShaderDataOptions());
/*
* Constructs the modified preprocessed source that would be dumped to a .usf file via DumpDebugShaderData, including the following additions:
* - resource table debug information
* - commandlines for running ShaderCompileWorker in DirectCompile mode (one which re-runs the preprocessor and one which bypasses it)
* - a comment containing the FShaderCompilerInput DebugDescription
* - any additions performed by the AppendPreSource/AppendPostSource TFunctions
* @param Input The input of the compilation job
* @param PreprocessedSource The unmodified preprocessed source (used as input to the compilation)
* @param Options Options which can change behaviour of the debug dump; see above.
*/
extern SHADERCOMPILERCOMMON_API FString GetDebugShaderContents(const FShaderCompilerInput& Input, const FString& PreprocessedSource, const FDebugShaderDataOptions& Options = FDebugShaderDataOptions());
}
extern SHADERCOMPILERCOMMON_API void HandleReflectedGlobalConstantBufferMember(
const FString& MemberName,
uint32 ConstantBufferIndex,
int32 ReflectionOffset,
int32 ReflectionSize,
FShaderCompilerOutput& CompilerOutput
);
extern SHADERCOMPILERCOMMON_API void HandleReflectedUniformBufferConstantBufferMember(
int32 UniformBufferSlot,
const FString& MemberName,
int32 ReflectionOffset,
int32 ReflectionSize,
FShaderCompilerOutput& CompilerOutput
);
extern SHADERCOMPILERCOMMON_API void HandleReflectedRootConstantBufferMember(
const FShaderCompilerInput& Input,
const FShaderParameterParser& ShaderParameterParser,
const FString& MemberName,
int32 ReflectionOffset,
int32 ReflectionSize,
FShaderCompilerOutput& CompilerOutput
);
extern SHADERCOMPILERCOMMON_API void HandleReflectedRootConstantBuffer(
int32 ConstantBufferSize,
FShaderCompilerOutput& CompilerOutput
);
extern SHADERCOMPILERCOMMON_API void HandleReflectedUniformBuffer(
const FString& UniformBufferName,
int32 ReflectionSlot,
int32 BaseIndex,
int32 BufferSize,
FShaderCompilerOutput& CompilerOutput
);
inline void HandleReflectedUniformBuffer(const FString& UniformBufferName, int32 ReflectionSlot, int32 BufferSize, FShaderCompilerOutput& CompilerOutput)
{
HandleReflectedUniformBuffer(UniformBufferName, ReflectionSlot, 0, BufferSize, CompilerOutput);
}
inline void HandleReflectedUniformBuffer(const FString& UniformBufferName, int32 ReflectionSlot, FShaderCompilerOutput& CompilerOutput)
{
HandleReflectedUniformBuffer(UniformBufferName, ReflectionSlot, 0, CompilerOutput);
}
extern SHADERCOMPILERCOMMON_API void HandleReflectedShaderResource(
const FString& ResourceName,
int32 BindOffset,
int32 ReflectionSlot,
int32 BindCount,
FShaderCompilerOutput& CompilerOutput
);
extern SHADERCOMPILERCOMMON_API void UpdateStructuredBufferStride(
const FShaderCompilerInput& Input,
const FString& ResourceName,
uint16 BindPoint,
uint16 Stride,
FShaderCompilerOutput& CompilerOutput
);
inline void HandleReflectedShaderResource(const FString& ResourceName, int32 ReflectionSlot, int32 BindCount, FShaderCompilerOutput& CompilerOutput)
{
HandleReflectedShaderResource(ResourceName, 0, ReflectionSlot, BindCount, CompilerOutput);
}
inline void HandleReflectedShaderResource(const FString& ResourceName, int32 ReflectionSlot, FShaderCompilerOutput& CompilerOutput)
{
HandleReflectedShaderResource(ResourceName, ReflectionSlot, 1, CompilerOutput);
}
extern SHADERCOMPILERCOMMON_API void HandleReflectedShaderUAV(
const FString& UAVName,
int32 BindOffset,
int32 ReflectionSlot,
int32 BindCount,
FShaderCompilerOutput& CompilerOutput
);
inline void HandleReflectedShaderUAV(const FString& UAVName, int32 ReflectionSlot, int32 BindCount, FShaderCompilerOutput& CompilerOutput)
{
HandleReflectedShaderUAV(UAVName, 0, ReflectionSlot, BindCount, CompilerOutput);
}
inline void HandleReflectedShaderUAV(const FString& UAVName, int32 ReflectionSlot, FShaderCompilerOutput& CompilerOutput)
{
HandleReflectedShaderUAV(UAVName, ReflectionSlot, 1, CompilerOutput);
}
extern SHADERCOMPILERCOMMON_API void HandleReflectedShaderSampler(
const FString& SamplerName,
int32 BindOffset,
int32 ReflectionSlot,
int32 BindCount,
FShaderCompilerOutput& CompilerOutput
);
inline void HandleReflectedShaderSampler(const FString& SamplerName, int32 ReflectionSlot, int32 BindCount, FShaderCompilerOutput& CompilerOutput)
{
HandleReflectedShaderSampler(SamplerName, 0, ReflectionSlot, BindCount, CompilerOutput);
}
inline void HandleReflectedShaderSampler(const FString& SamplerName, int32 ReflectionSlot, FShaderCompilerOutput& CompilerOutput)
{
HandleReflectedShaderSampler(SamplerName, ReflectionSlot, 1, CompilerOutput);
}
/** Adds a note to CompilerOutput.Error about where the shader parameter structure is on C++ side. */
extern SHADERCOMPILERCOMMON_API void AddNoteToDisplayShaderParameterStructureOnCppSide(
const FShaderParametersMetadata* ParametersStructure,
FShaderCompilerOutput& CompilerOutput);
/** Adds an error to CompilerOutput.Error about a shader parameters that could not be bound. */
extern SHADERCOMPILERCOMMON_API void AddUnboundShaderParameterError(
const FShaderCompilerInput& CompilerInput,
const FShaderParameterParser& ShaderParameterParser,
const FString& ParameterBindingName,
FShaderCompilerOutput& CompilerOutput);
// The cross compiler doesn't yet support struct initializers needed to construct static structs for uniform buffers
// Replace all uniform buffer struct member references (View.WorldToClip) with a flattened name that removes the struct dependency (View_WorldToClip)
extern SHADERCOMPILERCOMMON_API void RemoveUniformBuffersFromSource(const FShaderCompilerEnvironment& Environment, FString& PreprocessedShaderSource);
// Processes TEXT macros
extern SHADERCOMPILERCOMMON_API void TransformStringIntoCharacterArray(FString& PreprocessedShaderSource);
extern SHADERCOMPILERCOMMON_API bool RemoveUnusedOutputs(FString& InOutSourceCode, const TArray<FString>& InUsedOutputs, const TArray<FString>& InExceptions, FString& InOutEntryPoint, TArray<FString>& OutErrors);
extern SHADERCOMPILERCOMMON_API bool RemoveUnusedInputs(FString& InOutSourceCode, const TArray<FString>& InUsedInputs, FString& InOutEntryPoint, TArray<FString>& OutErrors);
extern SHADERCOMPILERCOMMON_API bool ConvertFromFP32ToFP16(FString& InOutSourceCode, TArray<FString>& OutErrors);
enum class EShaderConductorTarget
{
Dxil,
Spirv,
};
extern SHADERCOMPILERCOMMON_API void WriteShaderConductorCommandLine(const FShaderCompilerInput& Input, const FString& SourceFilename, EShaderConductorTarget Target);
extern SHADERCOMPILERCOMMON_API void DumpDebugShaderText(const FShaderCompilerInput& Input, const FString& InSource, const FString& FileExtension);
extern SHADERCOMPILERCOMMON_API void DumpDebugShaderText(const FShaderCompilerInput& Input, ANSICHAR* InSource, int32 InSourceLength, const FString& FileExtension);
extern SHADERCOMPILERCOMMON_API void DumpDebugShaderText(const FShaderCompilerInput& Input, ANSICHAR* InSource, int32 InSourceLength, const FString& FileName, const FString& FileExtension);
extern SHADERCOMPILERCOMMON_API void DumpDebugShaderBinary(const FShaderCompilerInput& Input, void* InData, int32 InDataByteSize, const FString& FileExtension);
extern SHADERCOMPILERCOMMON_API void DumpDebugShaderBinary(const FShaderCompilerInput& Input, void* InData, int32 InDataByteSize, const FString& FileName, const FString& FileExtension);
extern SHADERCOMPILERCOMMON_API void DumpDebugShaderDisassembledSpirv(const FShaderCompilerInput& Input, void* InData, int32 InDataByteSize, const FString& FileExtension);
extern SHADERCOMPILERCOMMON_API void DumpDebugShaderDisassembledDxil(const FShaderCompilerInput& Input, void* InData, int32 InDataByteSize, const FString& FileExtension);
// calls 'Mali Offline Compiler' to compile the glsl source code and extract the generated instruction count
extern SHADERCOMPILERCOMMON_API void CompileOfflineMali(const FShaderCompilerInput &Input, FShaderCompilerOutput& ShaderOutput, const ANSICHAR* ShaderSource, const int32 SourceSize, bool bVulkanSpirV, const ANSICHAR* VulkanSpirVEntryPoint = nullptr);
// Cross compiler support/common functionality
namespace CrossCompiler
{
extern SHADERCOMPILERCOMMON_API FString CreateResourceTableFromEnvironment(const FShaderCompilerEnvironment& Environment);
extern SHADERCOMPILERCOMMON_API void CreateEnvironmentFromResourceTable(const FString& String, FShaderCompilerEnvironment& OutEnvironment);
extern SHADERCOMPILERCOMMON_API void ParseHlslccError(TArray<FShaderCompilerError>& OutErrors, const FString& InLine, bool bUseAbsolutePaths = false);
struct SHADERCOMPILERCOMMON_API FHlslccHeader
{
FHlslccHeader();
virtual ~FHlslccHeader() { }
bool Read(const ANSICHAR*& ShaderSource, int32 SourceLen);
// After the standard header, different backends can output their own info
virtual bool ParseCustomHeaderEntries(const ANSICHAR*& ShaderSource)
{
return true;
}
struct FInOut
{
FString Type;
int32 Index;
int32 ArrayCount;
FString Name;
};
struct FAttribute
{
int32 Index;
FString Name;
};
struct FPackedGlobal
{
ANSICHAR PackedType;
FString Name;
int32 Offset;
int32 Count;
};
//struct FUniform
//{
//};
struct FPackedUB
{
FAttribute Attribute;
struct FMember
{
FString Name;
int32 Offset;
int32 Count;
};
TArray<FMember> Members;
};
struct FPackedUBCopy
{
int32 SourceUB;
int32 SourceOffset;
int32 DestUB;
ANSICHAR DestPackedType;
int32 DestOffset;
int32 Count;
};
struct FSampler
{
FString Name;
int32 Offset;
int32 Count;
TArray<FString> SamplerStates;
};
struct FUAV
{
FString Name;
int32 Offset;
int32 Count;
};
struct FAccelerationStructure
{
FString Name;
int32 Offset = 0;
};
FString Name;
TArray<FInOut> Inputs;
TArray<FInOut> Outputs;
TArray<FAttribute> UniformBlocks;
//TArray<FUniform> Uniforms;
TArray<FPackedGlobal> PackedGlobals;
TArray<FPackedUB> PackedUBs;
TArray<FPackedUBCopy> PackedUBCopies;
TArray<FPackedUBCopy> PackedUBGlobalCopies;
TArray<FSampler> Samplers;
TArray<FUAV> UAVs;
TArray<FAttribute> SamplerStates;
TArray<FAccelerationStructure> AccelerationStructures;
uint32 NumThreads[3];
static bool ReadInOut(const ANSICHAR*& ShaderSource, TArray<FInOut>& OutAttributes);
static bool ReadCopies(const ANSICHAR*& ShaderSource, bool bGlobals, TArray<FPackedUBCopy>& OutCopies);
};
extern SHADERCOMPILERCOMMON_API const TCHAR* GetFrequencyName(EShaderFrequency Frequency);
inline bool ParseIdentifier(const ANSICHAR*& Str, FString& OutStr)
{
OutStr = TEXT("");
FString Result;
while ((*Str >= 'A' && *Str <= 'Z')
|| (*Str >= 'a' && *Str <= 'z')
|| (*Str >= '0' && *Str <= '9')
|| *Str == '_')
{
OutStr += (TCHAR)*Str;
++Str;
}
return OutStr.Len() > 0;
}
inline bool ParseIdentifier(const TCHAR*& Str, FString& OutStr)
{
OutStr = TEXT("");
FString Result;
while ((*Str >= 'A' && *Str <= 'Z')
|| (*Str >= 'a' && *Str <= 'z')
|| (*Str >= '0' && *Str <= '9')
|| *Str == '_')
{
OutStr += (TCHAR)*Str;
++Str;
}
return OutStr.Len() > 0;
}
inline bool ParseString(const ANSICHAR*& Str, FString& OutStr)
{
OutStr = TEXT("");
FString Result;
while (*Str != ' ' && *Str != '\n')
{
OutStr += (TCHAR)*Str;
++Str;
}
return OutStr.Len() > 0;
}
inline bool ParseString(const TCHAR*& Str, FString& OutStr)
{
OutStr = TEXT("");
FString Result;
while (*Str != ' ' && *Str != '\n')
{
OutStr += (TCHAR)*Str;
++Str;
}
return OutStr.Len() > 0;
}
FORCEINLINE bool Match(const ANSICHAR*& Str, ANSICHAR Char)
{
if (*Str == Char)
{
++Str;
return true;
}
return false;
}
FORCEINLINE bool Match(const TCHAR*& Str, ANSICHAR Char)
{
if (*Str == Char)
{
++Str;
return true;
}
return false;
}
FORCEINLINE bool Match(const ANSICHAR*& Str, const ANSICHAR* Sub)
{
int32 SubLen = FCStringAnsi::Strlen(Sub);
if (FCStringAnsi::Strncmp(Str, Sub, SubLen) == 0)
{
Str += SubLen;
return true;
}
return false;
}
FORCEINLINE bool Match(const TCHAR*& Str, const TCHAR* Sub)
{
int32 SubLen = FCString::Strlen(Sub);
if (FCString::Strncmp(Str, Sub, SubLen) == 0)
{
Str += SubLen;
return true;
}
return false;
}
template <typename T>
inline bool ParseIntegerNumber(const ANSICHAR*& Str, T& OutNum)
{
auto* OriginalStr = Str;
OutNum = 0;
while (*Str >= '0' && *Str <= '9')
{
OutNum = OutNum * 10 + *Str++ - '0';
}
return Str != OriginalStr;
}
template <typename T>
inline bool ParseIntegerNumber(const TCHAR*& Str, T& OutNum)
{
auto* OriginalStr = Str;
OutNum = 0;
while (*Str >= '0' && *Str <= '9')
{
OutNum = OutNum * 10 + *Str++ - '0';
}
return Str != OriginalStr;
}
inline bool ParseSignedNumber(const ANSICHAR*& Str, int32& OutNum)
{
int32 Sign = Match(Str, '-') ? -1 : 1;
uint32 Num = 0;
if (ParseIntegerNumber(Str, Num))
{
OutNum = Sign * (int32)Num;
return true;
}
return false;
}
inline bool ParseSignedNumber(const TCHAR*& Str, int32& OutNum)
{
int32 Sign = Match(Str, '-') ? -1 : 1;
uint32 Num = 0;
if (ParseIntegerNumber(Str, Num))
{
OutNum = Sign * (int32)Num;
return true;
}
return false;
}
}