izzy/UnrealEngineUWP
0
0
Fork 0
mirror of https://github.com/izzy2lost/UnrealEngineUWP.git synced 2026-03-26 18:15:20 -07:00
Code Issues Packages Projects Releases Wiki Activity
Files
ecd2f824a1f2381d907593c853794de82d8bd580
UnrealEngineUWP/Engine/Source/Developer/ShaderCompilerCommon/Private/ShaderCompilerCommon.cpp
jason hoerner 6f67025ccc Shader Compiler: Optimized version of CleanupUniformBufferCode, called during preprocessing. New version is 3.8x faster, producing a 19% overall speedup in ConditionalPreprocessShader. Processes the shader in a single pass, scanning for potential compound identifiers (symbol tokens connected by dots and optional whitespace), then checking if they match known uniform buffer members. A few optimizations improve the speed of checking if an identifier matches, including an early out based on identifier length and start character, followed by organizing uniform buffers and their members by length, requiring only a small subset of members to be tested. Parsing uniform buffer definitions was also optimized to avoid memory allocation. 
Doesn't produce bit identical preprocessed output, as it compacts whitespace from compound identifiers before testing them, whereas the original implementation compacts whitespace after matching.  Compacting first makes testing more efficient and simpler, since it doesn't need to take into account whitespace.  For A/B validation, I did add temporary debug code that stored and reverted un-matched identifiers, mimicking the old behavior, and it was identical across the board.

#rnx
#rb dan.elksnitis chris.waters jason.nadro

[CL 28313553 by jason hoerner in ue5-main branch]
2023-09-28 07:08:47 -04:00

3398 lines
99 KiB
C++
Raw Blame History

// Copyright Epic Games, Inc. All Rights Reserved.
#include "ShaderCompilerCommon.h"
#include "ShaderParameterParser.h"
#include "Misc/FileHelper.h"
#include "Misc/Paths.h"
#include "Misc/PathViews.h"
#include "Modules/ModuleManager.h"
#include "HlslccDefinitions.h"
#include "HAL/FileManager.h"
#include "String/RemoveFrom.h"
#include "ShaderPreprocessTypes.h"
#include "ShaderSymbolExport.h"
#include "ShaderMinifier.h"
#include "Algo/Sort.h"
static TAutoConsoleVariable<bool> CVarShaderCompilerCleanupUniformBufferCodeNew(
TEXT("r.ShaderCompiler.CleanupUniformBufferCodeNew"),
true,
TEXT("Run new optimized version of CleanupUniformBufferCode. Temporary chicken switch in case there are issues, before we remove old version."),
ECVF_Default
);
IMPLEMENT_MODULE(FDefaultModuleImpl, ShaderCompilerCommon);
int16 GetNumUniformBuffersUsed(const FShaderCompilerResourceTable& InSRT)
{
auto CountLambda = [&](const TArray<uint32>& In)
{
int16 LastIndex = -1;
for (int32 i = 0; i < In.Num(); ++i)
{
auto BufferIndex = FRHIResourceTableEntry::GetUniformBufferIndex(In[i]);
if (BufferIndex != static_cast<uint16>(FRHIResourceTableEntry::GetEndOfStreamToken()) )
{
LastIndex = FMath::Max(LastIndex, (int16)BufferIndex);
}
}
return LastIndex + 1;
};
int16 Num = CountLambda(InSRT.SamplerMap);
Num = FMath::Max(Num, (int16)CountLambda(InSRT.ShaderResourceViewMap));
Num = FMath::Max(Num, (int16)CountLambda(InSRT.TextureMap));
Num = FMath::Max(Num, (int16)CountLambda(InSRT.UnorderedAccessViewMap));
return Num;
}
void BuildResourceTableTokenStream(const TArray<uint32>& InResourceMap, int32 MaxBoundResourceTable, TArray<uint32>& OutTokenStream, bool bGenerateEmptyTokenStreamIfNoResources)
{
if (bGenerateEmptyTokenStreamIfNoResources)
{
if (InResourceMap.Num() == 0)
{
return;
}
}
// First we sort the resource map.
TArray<uint32> SortedResourceMap = InResourceMap;
SortedResourceMap.Sort();
// The token stream begins with a table that contains offsets per bound uniform buffer.
// This offset provides the start of the token stream.
OutTokenStream.AddZeroed(MaxBoundResourceTable+1);
auto LastBufferIndex = FRHIResourceTableEntry::GetEndOfStreamToken();
for (int32 i = 0; i < SortedResourceMap.Num(); ++i)
{
auto BufferIndex = FRHIResourceTableEntry::GetUniformBufferIndex(SortedResourceMap[i]);
if (BufferIndex != LastBufferIndex)
{
// Store the offset for resources from this buffer.
OutTokenStream[BufferIndex] = OutTokenStream.Num();
LastBufferIndex = BufferIndex;
}
OutTokenStream.Add(SortedResourceMap[i]);
}
// Add a token to mark the end of the stream. Not needed if there are no bound resources.
if (OutTokenStream.Num())
{
OutTokenStream.Add(FRHIResourceTableEntry::GetEndOfStreamToken());
}
}
bool BuildResourceTableMapping(
const FShaderResourceTableMap& ResourceTableMap,
const TMap<FString, FUniformBufferEntry>& UniformBufferMap,
TBitArray<>& UsedUniformBufferSlots,
FShaderParameterMap& ParameterMap,
FShaderCompilerResourceTable& OutSRT)
{
check(OutSRT.ResourceTableBits == 0);
check(OutSRT.ResourceTableLayoutHashes.Num() == 0);
// Build resource table mapping
int32 MaxBoundResourceTable = -1;
// Go through ALL the members of ALL the UB resources
for (const FUniformResourceEntry& Entry : ResourceTableMap.Resources)
{
const FString& Name = Entry.UniformBufferMemberName;
// If the shaders uses this member (eg View_PerlinNoise3DTexture)...
if (TOptional<FParameterAllocation> Allocation = ParameterMap.FindParameterAllocation(Name))
{
const EShaderParameterType ParameterType = Allocation->Type;
const bool bBindlessParameter = (ParameterType == EShaderParameterType::BindlessResourceIndex || ParameterType == EShaderParameterType::BindlessSamplerIndex);
// Force bindless "indices" to zero since they're not needed in SetResourcesFromTables
const uint16 BaseIndex = bBindlessParameter ? 0 : Allocation->BaseIndex;
ParameterMap.RemoveParameterAllocation(*Name);
uint16 UniformBufferIndex = INDEX_NONE;
uint16 UBBaseIndex, UBSize;
// Add the UB itself as a parameter if not there
FString UniformBufferName(Entry.GetUniformBufferName());
if (!ParameterMap.FindParameterAllocation(*UniformBufferName, UniformBufferIndex, UBBaseIndex, UBSize))
{
UniformBufferIndex = UsedUniformBufferSlots.FindAndSetFirstZeroBit();
ParameterMap.AddParameterAllocation(*UniformBufferName, UniformBufferIndex,0,0,EShaderParameterType::UniformBuffer);
}
// Mark used UB index
if (UniformBufferIndex >= sizeof(OutSRT.ResourceTableBits) * 8)
{
return false;
}
OutSRT.ResourceTableBits |= (1 << UniformBufferIndex);
// How many resource tables max we'll use, and fill it with zeroes
MaxBoundResourceTable = FMath::Max<int32>(MaxBoundResourceTable, (int32)UniformBufferIndex);
auto ResourceMap = FRHIResourceTableEntry::Create(UniformBufferIndex, Entry.ResourceIndex, BaseIndex);
switch( Entry.Type )
{
case UBMT_TEXTURE:
case UBMT_RDG_TEXTURE:
OutSRT.TextureMap.Add(ResourceMap);
break;
case UBMT_SAMPLER:
OutSRT.SamplerMap.Add(ResourceMap);
break;
case UBMT_SRV:
case UBMT_RDG_TEXTURE_SRV:
case UBMT_RDG_BUFFER_SRV:
OutSRT.ShaderResourceViewMap.Add(ResourceMap);
break;
case UBMT_UAV:
case UBMT_RDG_TEXTURE_UAV:
case UBMT_RDG_BUFFER_UAV:
OutSRT.UnorderedAccessViewMap.Add(ResourceMap);
break;
default:
return false;
}
}
}
// Emit hashes for all uniform buffers in the parameter map. We need to include the ones without resources as well
// (i.e. just constants), since the global uniform buffer bindings rely on valid hashes.
for (const TPair<FString, FParameterAllocation>& KeyValue : ParameterMap.GetParameterMap())
{
const FString& UniformBufferName = KeyValue.Key;
const FParameterAllocation& UniformBufferParameter = KeyValue.Value;
if (UniformBufferParameter.Type == EShaderParameterType::UniformBuffer)
{
if (OutSRT.ResourceTableLayoutHashes.Num() <= UniformBufferParameter.BufferIndex)
{
OutSRT.ResourceTableLayoutHashes.SetNumZeroed(UniformBufferParameter.BufferIndex + 1);
}
// Data-driven uniform buffers will not have registered this information.
if (const FUniformBufferEntry* UniformBufferEntry = UniformBufferMap.Find(UniformBufferName))
{
OutSRT.ResourceTableLayoutHashes[UniformBufferParameter.BufferIndex] = UniformBufferEntry->LayoutHash;
}
}
}
OutSRT.MaxBoundResourceTable = MaxBoundResourceTable;
return true;
}
PRAGMA_DISABLE_DEPRECATION_WARNINGS
// Deprecated version of function
bool BuildResourceTableMapping(
const TMap<FString, FResourceTableEntry>& ResourceTableMap,
const TMap<FString, FUniformBufferEntry>& UniformBufferMap,
TBitArray<>& UsedUniformBufferSlots,
FShaderParameterMap& ParameterMap,
FShaderCompilerResourceTable& OutSRT)
{
UE_LOG(LogShaders, Error, TEXT("Using unimplemented deprecated version of BuildResourceTableMapping -- use version that accepts FShaderResourceTableMap instead."));
return false;
}
PRAGMA_ENABLE_DEPRECATION_WARNINGS
void CullGlobalUniformBuffers(const TMap<FString, FUniformBufferEntry>& UniformBufferMap, FShaderParameterMap& ParameterMap)
{
TArray<FString> ParameterNames;
ParameterMap.GetAllParameterNames(ParameterNames);
for (const FString& Name : ParameterNames)
{
if (const FUniformBufferEntry* UniformBufferEntry = UniformBufferMap.Find(*Name))
{
// A uniform buffer that is bound per-shader keeps its allocation in the map.
if (EnumHasAnyFlags(UniformBufferEntry->BindingFlags, EUniformBufferBindingFlags::Shader))
{
continue;
}
ParameterMap.RemoveParameterAllocation(*Name);
}
}
}
static bool IsSpaceOrTabOrEOL(TCHAR Char)
{
return Char == ' ' || Char == '\t' || Char == '\n' || Char == '\r';
}
static const TCHAR* FindNextChar(const TCHAR* ReadStart, TCHAR SearchChar)
{
const TCHAR* SearchPtr = ReadStart;
while (*SearchPtr && *SearchPtr != SearchChar)
{
SearchPtr++;
}
return SearchPtr;
}
const TCHAR* FindNextWhitespace(const TCHAR* StringPtr)
{
while (*StringPtr && !IsSpaceOrTabOrEOL(*StringPtr))
{
StringPtr++;
}
if (*StringPtr && IsSpaceOrTabOrEOL(*StringPtr))
{
return StringPtr;
}
else
{
return nullptr;
}
}
const TCHAR* FindNextNonWhitespace(const TCHAR* StringPtr)
{
while (*StringPtr && IsSpaceOrTabOrEOL(*StringPtr))
{
StringPtr++;
}
if (*StringPtr && !IsSpaceOrTabOrEOL(*StringPtr))
{
return StringPtr;
}
return nullptr;
}
const TCHAR* FindPreviousNonWhitespace(const TCHAR* StringPtr)
{
do
{
StringPtr--;
} while (*StringPtr && IsSpaceOrTabOrEOL(*StringPtr));
if (*StringPtr && !IsSpaceOrTabOrEOL(*StringPtr))
{
return StringPtr;
}
return nullptr;
}
const TCHAR* FindMatchingBlock(const TCHAR* OpeningCharPtr, char OpenChar, char CloseChar)
{
const TCHAR* SearchPtr = OpeningCharPtr;
int32 Depth = 0;
while (*SearchPtr)
{
if (*SearchPtr == OpenChar)
{
Depth++;
}
else if (*SearchPtr == CloseChar)
{
if (Depth == 0)
{
return SearchPtr;
}
Depth--;
}
SearchPtr++;
}
return nullptr;
}
const TCHAR* FindMatchingClosingBrace(const TCHAR* OpeningCharPtr) { return FindMatchingBlock(OpeningCharPtr, '{', '}'); };
const TCHAR* FindMatchingClosingParenthesis(const TCHAR* OpeningCharPtr) { return FindMatchingBlock(OpeningCharPtr, '(', ')'); };
// See MSDN HLSL 'Symbol Name Restrictions' doc
inline bool IsValidHLSLIdentifierCharacter(TCHAR Char)
{
return (Char >= 'a' && Char <= 'z') ||
(Char >= 'A' && Char <= 'Z') ||
(Char >= '0' && Char <= '9') ||
Char == '_';
}
void ParseHLSLTypeName(const TCHAR* SearchString, const TCHAR*& TypeNameStartPtr, const TCHAR*& TypeNameEndPtr)
{
TypeNameStartPtr = FindNextNonWhitespace(SearchString);
check(TypeNameStartPtr);
TypeNameEndPtr = TypeNameStartPtr;
int32 Depth = 0;
const TCHAR* NextWhitespace = FindNextWhitespace(TypeNameStartPtr);
const TCHAR* PotentialExtraTypeInfoPtr = NextWhitespace ? FindNextNonWhitespace(NextWhitespace) : nullptr;
// Find terminating whitespace, but skip over trailing ' < float4 >'
while (*TypeNameEndPtr)
{
if (*TypeNameEndPtr == '<')
{
Depth++;
}
else if (*TypeNameEndPtr == '>')
{
Depth--;
}
else if (Depth == 0
&& IsSpaceOrTabOrEOL(*TypeNameEndPtr)
// If we found a '<', we must not accept any whitespace before it
&& (!PotentialExtraTypeInfoPtr || *PotentialExtraTypeInfoPtr != '<' || TypeNameEndPtr > PotentialExtraTypeInfoPtr))
{
break;
}
TypeNameEndPtr++;
}
check(TypeNameEndPtr);
}
FStringView ParseHLSLSymbolName(const TCHAR* SearchString)
{
const TCHAR* SymbolNameStartPtr = FindNextNonWhitespace(SearchString);
check(SymbolNameStartPtr);
const TCHAR* SymbolNameEndPtr = SymbolNameStartPtr;
while (*SymbolNameEndPtr && IsValidHLSLIdentifierCharacter(*SymbolNameEndPtr))
{
SymbolNameEndPtr++;
}
return FStringView(SymbolNameStartPtr, SymbolNameEndPtr - SymbolNameStartPtr);
}
const TCHAR* ParseHLSLSymbolName(const TCHAR* SearchString, FString& SymbolName)
{
FStringView Result = ParseHLSLSymbolName(SearchString);
SymbolName = FString(Result);
return Result.GetData() + Result.Len();
}
const TCHAR* MatchStructMemberName(const FString& SymbolName, const TCHAR* SearchPtr)
{
for (int32 i = 0; i < SymbolName.Len(); i++)
{
if (*SearchPtr != SymbolName[i])
{
return nullptr;
}
SearchPtr++;
if (i < SymbolName.Len() - 1)
{
// Skip whitespace within the struct member reference before the end
// eg 'View. ViewToClip'
while (IsSpaceOrTabOrEOL(*SearchPtr))
{
SearchPtr++;
}
}
}
// Only match whole symbol
if (IsValidHLSLIdentifierCharacter(*SearchPtr))
{
return nullptr;
}
return SearchPtr;
}
EShaderParameterType UE::ShaderCompilerCommon::ParseParameterType(
FStringView InType,
TArrayView<const TCHAR* const> InExtraSRVTypes,
TArrayView<const TCHAR* const> InExtraUAVTypes)
{
return FShaderParameterParser::ParseParameterType(InType, InExtraSRVTypes, InExtraUAVTypes);
}
FStringView UE::ShaderCompilerCommon::RemoveConstantBufferPrefix(FStringView InName)
{
return UE::String::RemoveFromStart(InName, FStringView(UE::ShaderCompilerCommon::kUniformBufferConstantBufferPrefix));
}
FString UE::ShaderCompilerCommon::RemoveConstantBufferPrefix(const FString& InName)
{
return FString(RemoveConstantBufferPrefix(FStringView(InName)));
}
EShaderParameterType UE::ShaderCompilerCommon::ParseAndRemoveBindlessParameterPrefix(FStringView& InName)
{
return FShaderParameterParser::ParseAndRemoveBindlessParameterPrefix(InName);
}
EShaderParameterType UE::ShaderCompilerCommon::ParseAndRemoveBindlessParameterPrefix(FString& InName)
{
return FShaderParameterParser::ParseAndRemoveBindlessParameterPrefix(InName);
}
bool UE::ShaderCompilerCommon::RemoveBindlessParameterPrefix(FString& InName)
{
return FShaderParameterParser::RemoveBindlessParameterPrefix(InName);
}
bool UE::ShaderCompilerCommon::ValidatePackedResourceCounts(FShaderCompilerOutput& Output, const FShaderCodePackedResourceCounts& PackedResourceCounts)
{
if (Output.bSucceeded)
{
auto GetAllResourcesOfType = [&](EShaderParameterType InType)
{
const TArray<FString> AllNames = Output.ParameterMap.GetAllParameterNamesOfType(InType);
if (AllNames.IsEmpty())
{
return FString();
}
return FString::Join(AllNames, TEXT(", "));
};
if (EnumHasAnyFlags(PackedResourceCounts.UsageFlags, EShaderResourceUsageFlags::BindlessResources) && PackedResourceCounts.NumSRVs > 0)
{
const FString Names = GetAllResourcesOfType(EShaderParameterType::SRV);
Output.Errors.Add(FString::Printf(TEXT("Shader is mixing bindless resources with non-bindless resources. %d SRV slots were detected: %s"), PackedResourceCounts.NumSRVs, *Names));
Output.bSucceeded = false;
}
if (EnumHasAnyFlags(PackedResourceCounts.UsageFlags, EShaderResourceUsageFlags::BindlessResources) && PackedResourceCounts.NumUAVs > 0)
{
const FString Names = GetAllResourcesOfType(EShaderParameterType::UAV);
Output.Errors.Add(FString::Printf(TEXT("Shader is mixing bindless resources with non-bindless resources. %d UAV slots were detected: %s"), PackedResourceCounts.NumUAVs, *Names));
Output.bSucceeded = false;
}
if (EnumHasAnyFlags(PackedResourceCounts.UsageFlags, EShaderResourceUsageFlags::BindlessSamplers) && PackedResourceCounts.NumSamplers > 0)
{
const FString Names = GetAllResourcesOfType(EShaderParameterType::Sampler);
Output.Errors.Add(FString::Printf(TEXT("Shader is mixing bindless samplers with non-bindless samplers. %d sampler slots were detected: %s"), PackedResourceCounts.NumSamplers, *Names));
Output.bSucceeded = false;
}
}
return Output.bSucceeded;
}
void UE::ShaderCompilerCommon::ParseRayTracingEntryPoint(const FString& Input, FString& OutMain, FString& OutAnyHit, FString& OutIntersection)
{
auto ParseEntry = [&Input](const TCHAR* Marker)
{
FString Result;
const int32 BeginIndex = Input.Find(Marker, ESearchCase::IgnoreCase, ESearchDir::FromStart);
if (BeginIndex != INDEX_NONE)
{
int32 EndIndex = Input.Find(TEXT(" "), ESearchCase::IgnoreCase, ESearchDir::FromStart, BeginIndex);
if (EndIndex == INDEX_NONE)
{
EndIndex = Input.Len() + 1;
}
const int32 MarkerLen = FCString::Strlen(Marker);
const int32 Count = EndIndex - BeginIndex;
Result = Input.Mid(BeginIndex + MarkerLen, Count - MarkerLen);
}
return Result;
};
OutMain = ParseEntry(TEXT("closesthit="));
OutAnyHit = ParseEntry(TEXT("anyhit="));
OutIntersection = ParseEntry(TEXT("intersection="));
// If complex hit group entry is not specified, assume a single verbatim entry point
if (OutMain.IsEmpty() && OutAnyHit.IsEmpty() && OutIntersection.IsEmpty())
{
OutMain = Input;
}
}
bool UE::ShaderCompilerCommon::RemoveDeadCode(FString& InOutPreprocessedShaderSource, TConstArrayView<FStringView> RequiredSymbols, TArray<FShaderCompilerError>& OutErrors)
{
UE::ShaderMinifier::EMinifyShaderFlags ExtraFlags = UE::ShaderMinifier::EMinifyShaderFlags::None;
#if 0 // Extra features that may be useful during development / debugging
ExtraFlags |= UE::ShaderMinifier::EMinifyShaderFlags::OutputReasons // Output a comment every struct/function describing why it was included (i.e. which code block uses it)
| UE::ShaderMinifier::EMinifyShaderFlags::OutputStats; // Output a comment detailing how many blocks of each type (functions/structs/etc.) were emitted
#endif
UE::ShaderMinifier::FMinifiedShader Minified = UE::ShaderMinifier::Minify(InOutPreprocessedShaderSource, RequiredSymbols,
UE::ShaderMinifier::EMinifyShaderFlags::OutputCommentLines // Preserve comments that were left after preprocessing
| UE::ShaderMinifier::EMinifyShaderFlags::OutputLines // Emit #line directives
| ExtraFlags);
if (Minified.Success())
{
Swap(InOutPreprocessedShaderSource, Minified.Code);
return true;
}
else
{
OutErrors.Add(TEXT("warning: Shader minification failed."));
return false;
}
}
bool UE::ShaderCompilerCommon::RemoveDeadCode(FString& InOutPreprocessedShaderSource, const FString& EntryPoint, TArray<FShaderCompilerError>& OutErrors)
{
TArray<FStringView> RequiredSymbols;
FString EntryMain;
FString EntryAnyHit;
FString EntryIntersection;
UE::ShaderCompilerCommon::ParseRayTracingEntryPoint(EntryPoint, EntryMain, EntryAnyHit, EntryIntersection);
RequiredSymbols.Add(EntryMain);
if (!EntryAnyHit.IsEmpty())
{
RequiredSymbols.Add(EntryAnyHit);
}
if (!EntryIntersection.IsEmpty())
{
RequiredSymbols.Add(EntryIntersection);
}
return UE::ShaderCompilerCommon::RemoveDeadCode(InOutPreprocessedShaderSource, RequiredSymbols, OutErrors);
}
void HandleReflectedGlobalConstantBufferMember(
const FString& InMemberName,
uint32 ConstantBufferIndex,
int32 ReflectionOffset,
int32 ReflectionSize,
FShaderCompilerOutput& Output
)
{
FString MemberName = InMemberName;
const EShaderParameterType ParameterType = FShaderParameterParser::ParseAndRemoveBindlessParameterPrefix(MemberName);
Output.ParameterMap.AddParameterAllocation(
*MemberName,
ConstantBufferIndex,
ReflectionOffset,
ReflectionSize,
ParameterType);
}
void HandleReflectedUniformBufferConstantBufferMember(
int32 UniformBufferSlot,
const FString& InMemberName,
int32 ReflectionOffset,
int32 ReflectionSize,
FShaderCompilerOutput& Output
)
{
FString MemberName = InMemberName;
const EShaderParameterType ParameterType = FShaderParameterParser::ParseAndRemoveBindlessParameterPrefix(MemberName);
if (ParameterType != EShaderParameterType::LooseData)
{
Output.ParameterMap.AddParameterAllocation(
*MemberName,
UniformBufferSlot,
ReflectionOffset,
1,
ParameterType
);
}
}
void HandleReflectedRootConstantBufferMember(
const FShaderCompilerInput& Input,
const FShaderParameterParser& ShaderParameterParser,
const FString& MemberName,
int32 ReflectionOffset,
int32 ReflectionSize,
FShaderCompilerOutput& Output
)
{
ShaderParameterParser.ValidateShaderParameterType(Input, MemberName, ReflectionOffset, ReflectionSize, Output);
HandleReflectedUniformBufferConstantBufferMember(
FShaderParametersMetadata::kRootCBufferBindingIndex,
MemberName,
ReflectionOffset,
ReflectionSize,
Output
);
}
void HandleReflectedRootConstantBuffer(
int32 ConstantBufferSize,
FShaderCompilerOutput& CompilerOutput
)
{
CompilerOutput.ParameterMap.AddParameterAllocation(
FShaderParametersMetadata::kRootUniformBufferBindingName,
FShaderParametersMetadata::kRootCBufferBindingIndex,
0,
static_cast<uint16>(ConstantBufferSize),
EShaderParameterType::LooseData);
}
void HandleReflectedUniformBuffer(
const FString& UniformBufferName,
int32 ReflectionSlot,
int32 BaseIndex,
int32 BufferSize,
FShaderCompilerOutput& CompilerOutput
)
{
FString AdjustedUniformBufferName(UE::ShaderCompilerCommon::RemoveConstantBufferPrefix(UniformBufferName));
CompilerOutput.ParameterMap.AddParameterAllocation(
*AdjustedUniformBufferName,
ReflectionSlot,
BaseIndex,
BufferSize,
EShaderParameterType::UniformBuffer
);
}
void HandleReflectedShaderResource(
const FString& ResourceName,
int32 BindOffset,
int32 ReflectionSlot,
int32 BindCount,
FShaderCompilerOutput& CompilerOutput
)
{
CompilerOutput.ParameterMap.AddParameterAllocation(
*ResourceName,
BindOffset,
ReflectionSlot,
BindCount,
EShaderParameterType::SRV
);
}
void UpdateStructuredBufferStride(
const FShaderCompilerInput& Input,
const FString& ResourceName,
uint16 BindPoint,
uint16 Stride,
FShaderCompilerOutput& CompilerOutput
)
{
if (BindPoint <= UINT16_MAX && Stride <= UINT16_MAX)
{
CompilerOutput.ParametersStrideToValidate.Add(FShaderCodeValidationStride{ BindPoint, Stride });
}
else
{
FString ErrorMessage = FString::Printf(TEXT("%s: Failed to set stride on parameter %s: Bind point %d, Stride %d"), *Input.GenerateShaderName(), *ResourceName, BindPoint, Stride);
CompilerOutput.Errors.Add(FShaderCompilerError(*ErrorMessage));
}
}
void AddShaderValidationSRVType(uint16 BindPoint,
EShaderCodeResourceBindingType TypeDecl,
FShaderCompilerOutput& CompilerOutput)
{
if (BindPoint <= UINT16_MAX)
{
CompilerOutput.ParametersSRVTypeToValidate.Add(FShaderCodeValidationType{ BindPoint, TypeDecl });
}
}
void AddShaderValidationUAVType(uint16 BindPoint,
EShaderCodeResourceBindingType TypeDecl,
FShaderCompilerOutput& CompilerOutput)
{
if (BindPoint <= UINT16_MAX)
{
CompilerOutput.ParametersUAVTypeToValidate.Add(FShaderCodeValidationType{ BindPoint, TypeDecl });
}
}
void AddShaderValidationUBSize(uint16 BindPoint,
uint32_t Size,
FShaderCompilerOutput& CompilerOutput)
{
if (BindPoint <= UINT16_MAX)
{
CompilerOutput.ParametersUBSizeToValidate.Add(FShaderCodeValidationUBSize{ BindPoint, Size });
}
}
void HandleReflectedShaderUAV(
const FString& UAVName,
int32 BindOffset,
int32 ReflectionSlot,
int32 BindCount,
FShaderCompilerOutput& CompilerOutput
)
{
CompilerOutput.ParameterMap.AddParameterAllocation(
*UAVName,
BindOffset,
ReflectionSlot,
BindCount,
EShaderParameterType::UAV
);
}
void HandleReflectedShaderSampler(
const FString& SamplerName,
int32 BindOffset,
int32 ReflectionSlot,
int32 BindCount,
FShaderCompilerOutput& CompilerOutput
)
{
CompilerOutput.ParameterMap.AddParameterAllocation(
*SamplerName,
BindOffset,
ReflectionSlot,
BindCount,
EShaderParameterType::Sampler
);
}
void AddNoteToDisplayShaderParameterStructureOnCppSide(
const FShaderParametersMetadata* ParametersStructure,
FShaderCompilerOutput& CompilerOutput)
{
FShaderCompilerError Error;
Error.StrippedErrorMessage = FString::Printf(
TEXT("Note: Definition of structure %s"),
ParametersStructure->GetStructTypeName());
Error.ErrorVirtualFilePath = ANSI_TO_TCHAR(ParametersStructure->GetFileName());
Error.ErrorLineString = FString::FromInt(ParametersStructure->GetFileLine());
CompilerOutput.Errors.Add(Error);
}
void AddUnboundShaderParameterError(
const FShaderCompilerInput& CompilerInput,
const FShaderParameterParser& ShaderParameterParser,
const FString& ParameterBindingName,
FShaderCompilerOutput& CompilerOutput)
{
check(CompilerInput.RootParametersStructure);
const FShaderParameterParser::FParsedShaderParameter& Member = ShaderParameterParser.FindParameterInfos(ParameterBindingName);
check(!Member.bIsBindable);
FShaderCompilerError Error(FString::Printf(
TEXT("Error: Shader parameter %s could not be bound to %s's shader parameter structure %s."),
*ParameterBindingName,
*CompilerInput.ShaderName,
CompilerInput.RootParametersStructure->GetStructTypeName()));
ShaderParameterParser.GetParameterFileAndLine(Member, Error.ErrorVirtualFilePath, Error.ErrorLineString);
CompilerOutput.Errors.Add(Error);
CompilerOutput.bSucceeded = false;
AddNoteToDisplayShaderParameterStructureOnCppSide(CompilerInput.RootParametersStructure, CompilerOutput);
}
void RemoveUniformBuffersFromSource(const FShaderCompilerEnvironment& Environment, FString& PreprocessedShaderSource)
{
CleanupUniformBufferCode(Environment, PreprocessedShaderSource);
}
struct FUniformBufferMemberInfo
{
// eg View.WorldToClip
FString NameAsStructMember;
// eg View_WorldToClip
FString GlobalName;
};
struct FUniformBufferInfo
{
int32 DefinitionEndOffset;
TArray<FUniformBufferMemberInfo> Members;
};
struct FUniformBufferMemberInfoNew
{
// eg View.WorldToClip
FStringView NameAsStructMember;
// eg View_WorldToClip
FStringView GlobalName;
bool operator<(const FUniformBufferMemberInfoNew& Other)
{
if (NameAsStructMember.Len() != Other.NameAsStructMember.Len())
{
return NameAsStructMember.Len() < Other.NameAsStructMember.Len();
}
else
{
return NameAsStructMember.Compare(Other.NameAsStructMember, ESearchCase::CaseSensitive) < 0;
}
}
};
// Index and count of subset of members
struct FUniformBufferMemberView
{
int32 MemberOffset;
int32 MemberCount;
};
struct FUniformBufferInfoNew
{
FStringView Name;
int32 NextWithSameLength; // Linked list of uniform buffer infos with same name length
TArray<FUniformBufferMemberInfoNew> Members; // Members sorted by length
TArray<FUniformBufferMemberView> MembersByLength; // Offset and count of members of a given length
};
static void FillUniformBufferDefinition(TStringBuilder<128>& Builder, const TCHAR* Start, const TCHAR* End)
{
Builder.Reset();
const TCHAR* Cursor = Start;
while (Cursor != End)
{
if (!IsSpaceOrTabOrEOL(*Cursor))
{
Builder.AppendChar(*Cursor);
}
Cursor++;
}
}
const TCHAR* ParseUniformBufferDefinition(const TCHAR* SourceStart, const TCHAR* ReadStart, TMap<FStringView, FUniformBufferInfo>& UniformBufferInfos)
{
const FStringView UniformBufferName(ParseHLSLSymbolName(ReadStart));
const TCHAR* OpeningBrace = FindNextChar(ReadStart, '{');
const TCHAR* ClosingBrace = FindMatchingClosingBrace(OpeningBrace + 1);
const TCHAR* CurrentParseStart = OpeningBrace + 1;
const TCHAR* NextSemicolon = FindNextChar(CurrentParseStart, ';');
TStringBuilder<128> StructName;
TStringBuilder<128> GlobalName;
while (NextSemicolon < ClosingBrace)
{
const TCHAR* NextSeparator = FindNextChar(CurrentParseStart, '=');
if (NextSeparator < NextSemicolon)
{
const TCHAR* StructStart = CurrentParseStart;
const TCHAR* StructEnd = NextSeparator - 1;
const TCHAR* GlobalStart = NextSeparator + 1;
const TCHAR* GlobalEnd = NextSemicolon - 1;
FillUniformBufferDefinition(StructName, StructStart, StructEnd);
FillUniformBufferDefinition(GlobalName, GlobalStart, GlobalEnd);
// Avoid unnecessary conversions
if (StructName.Len() == GlobalName.Len() && FCString::Strncmp(*StructName, *GlobalName, StructName.Len()) != 0)
{
FUniformBufferMemberInfo NewMemberInfo;
NewMemberInfo.NameAsStructMember = StructName;
NewMemberInfo.GlobalName = GlobalName;
// Add this member to the map
if (FUniformBufferInfo* Info = UniformBufferInfos.Find(UniformBufferName))
{
Info->Members.Add(MoveTemp(NewMemberInfo));
}
else
{
FUniformBufferInfo NewInfo;
NewInfo.DefinitionEndOffset = ClosingBrace - SourceStart;
NewInfo.Members.Add(MoveTemp(NewMemberInfo));
UniformBufferInfos.Emplace(UniformBufferName, MoveTemp(NewInfo));
}
}
}
CurrentParseStart = NextSemicolon + 1;
NextSemicolon = FindNextChar(CurrentParseStart, ';');
}
const TCHAR* EndPtr = ClosingBrace;
// Skip to the end of the UniformBuffer
while (*EndPtr && *EndPtr != ';')
{
EndPtr++;
}
return EndPtr;
}
inline bool IsValidSymbolStart(const TCHAR* SearchPtr, int32 UniformBufferNameLen)
{
// Make sure this isn't the end of another symbol or a sub-member of a different struct
// ex: "OpaqueBasePass.Substrate.MaxBytesPerPixel" when matching "Substrate"
if (IsValidHLSLIdentifierCharacter(*(SearchPtr - 1)) || *(SearchPtr - 1) == '.')
{
return false;
}
// Avoid lines like '#line 1 "/Engine/Generated/UniformBuffers/View.ush"'
if (FCString::Strncmp(SearchPtr + UniformBufferNameLen, TEXT("ush"), 3) == 0)
{
return false;
}
return true;
}
// Compacts spaces out of a compound identifier. Returns the new end pointer of the compacted identifier.
// End and result pointers are exclusive (length of the string is End - Start).
static TCHAR* CompactCompoundIdentifier(TCHAR* Start, TCHAR* End)
{
// Find first whitespace in the identifier, if present
TCHAR* ReadChar;
for (ReadChar = Start; ReadChar < End; ++ReadChar)
{
if (IsSpaceOrTabOrEOL(*ReadChar))
{
break;
}
}
if (ReadChar == End)
{
// No whitespace, we're done!
return End;
}
// Found some whitespace, so we need to compact the non-whitespace, swapping the whitespace to the end of the range
// WriteChar here will be the first whitespace character that we need to compact into.
TCHAR* WriteChar = ReadChar;
for (++ReadChar; ReadChar < End; ++ReadChar)
{
// If the current read character is non-whitespace, compact it down
if (!IsSpaceOrTabOrEOL(*ReadChar))
{
Swap(*ReadChar, *WriteChar);
WriteChar++;
}
}
return WriteChar;
}
const TCHAR* ParseUniformBufferDefinitionNew(const TCHAR* ReadStart, TArray<FUniformBufferInfoNew>& UniformBufferInfos, uint64 UniformBufferFilter[64], int32 UniformBuffersByLength[64])
{
// TODO: should we check for an existing item? In my testing, there's only one uniform buffer declaration with a given name,
// but the original code used a map, theoretically allowing for multiple.
int32 InfoIndex = UniformBufferInfos.AddDefaulted();
FUniformBufferInfoNew& Info = UniformBufferInfos[InfoIndex];
Info.Name = ParseHLSLSymbolName(ReadStart);
check(Info.Name.Len() < 64);
const TCHAR* OpeningBrace = FindNextChar(ReadStart, '{');
const TCHAR* ClosingBrace = FindMatchingClosingBrace(OpeningBrace + 1);
const TCHAR* CurrentParseStart = OpeningBrace + 1;
const TCHAR* NextSemicolon = FindNextChar(CurrentParseStart, ';');
while (NextSemicolon < ClosingBrace)
{
const TCHAR* NextSeparator = FindNextChar(CurrentParseStart, '=');
if (NextSeparator < NextSemicolon)
{
const TCHAR* StructStart = CurrentParseStart;
const TCHAR* StructEnd = NextSeparator - 1;
const TCHAR* GlobalStart = NextSeparator + 1;
const TCHAR* GlobalEnd = NextSemicolon - 1;
while (IsSpaceOrTabOrEOL(*StructStart))
{
StructStart++;
}
while (IsSpaceOrTabOrEOL(*GlobalStart))
{
GlobalStart++;
}
StructEnd = CompactCompoundIdentifier(const_cast<TCHAR*>(StructStart), const_cast<TCHAR*>(StructEnd));
GlobalEnd = CompactCompoundIdentifier(const_cast<TCHAR*>(GlobalStart), const_cast<TCHAR*>(GlobalEnd));
FStringView StructName(StructStart, StructEnd - StructStart);
FStringView GlobalName(GlobalStart, GlobalEnd - GlobalStart);
// Avoid unnecessary conversions
if (StructName.Len() == GlobalName.Len() && FCString::Strncmp(StructName.GetData(), GlobalName.GetData(), StructName.Len()) != 0)
{
FUniformBufferMemberInfoNew NewMemberInfo;
NewMemberInfo.NameAsStructMember = StructName;
NewMemberInfo.GlobalName = GlobalName;
// Need to be able to replace strings in place, so make sure GlobalName will fit in space of NameAsStructMember
check(NewMemberInfo.NameAsStructMember.Len() >= NewMemberInfo.GlobalName.Len());
Info.Members.Add(NewMemberInfo);
}
}
CurrentParseStart = NextSemicolon + 1;
NextSemicolon = FindNextChar(CurrentParseStart, ';');
}
const TCHAR* EndPtr = ClosingBrace;
// Skip to the end of the UniformBuffer
while (*EndPtr && *EndPtr != ';')
{
EndPtr++;
}
if (Info.Members.Num())
{
// We have members. Sort them. Note that the sort is by length first, not alphabetical, so the last item will be the longest.
Algo::Sort(Info.Members);
int32 MaxLen = Info.Members.Last().NameAsStructMember.Len();
// Initialize table with offset of first member with a given length, and the count of members of that length (going backwards so the
// index of the first element of a given size is the last one written to "MemberOffset").
Info.MembersByLength.SetNumZeroed(MaxLen + 1);
for (int32 MemberIndex = Info.Members.Num() - 1; MemberIndex >= 0; MemberIndex--)
{
int32 CurrentMemberLen = Info.Members[MemberIndex].NameAsStructMember.Len();
Info.MembersByLength[CurrentMemberLen].MemberOffset = MemberIndex;
Info.MembersByLength[CurrentMemberLen].MemberCount++;
}
// Initialize the uniform buffer name filter. The filter is a mask based on the first character of the name (minus 64 so valid token
// starting characters which are in ASCII range 64..127 fit in 64 bits). We can quickly check if a token of the given length and start
// character might be one we care about.
UniformBufferFilter[Info.Name.Len()] |= 1ull << (Info.Name[0] - 64);
// Add to linked list of uniform buffers by name length
Info.NextWithSameLength = UniformBuffersByLength[Info.Name.Len()];
UniformBuffersByLength[Info.Name.Len()] = InfoIndex;
}
else
{
// If no members, we don't care about it
UniformBufferInfos.RemoveAt(UniformBufferInfos.Num() - 1);
}
return EndPtr;
}
enum class AsciiFlags
{
TerminatorOrSlash = (1 << 0), // Null terminator OR slash (latter we care about for skipping commented out uniform blocks)
Whitespace = (1 << 1), // Includes other special characters below 32 (in addition to tab / newline)
Other = (1 << 2), // Anything else not one of the other types
SymbolStart = (1<<3), // Letters plus underscore (anything that can start a symbol)
Digit = (1 << 4),
Dot = (1 << 5),
Quote = (1 << 6),
Hash = (1 << 7),
};
static uint8 AsciiFlagTable[256] =
{
1,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, // Treat all special characters as whitespace
2,4,64,128,4,4,4,4, // 34 == Quote 35 == Hash
4,4,4,4,4,4,32,1, // 46 == Dot 47 == Slash
16,16,16,16,16,16,16,16, // Digits 0-7
16,16,4,4,4,4,4,4, // Digits 8-9
4,8,8,8,8,8,8,8, 8,8,8,8,8,8,8,8, 8,8,8,8,8,8,8,8, 8,8,8,4,4,4,4,8, // Upper case letters, 95 == Underscore
4,8,8,8,8,8,8,8, 8,8,8,8,8,8,8,8, 8,8,8,8,8,8,8,8, 8,8,8,4,4,4,4,4, // Lower case letters
4,4,4,4,4,4,4,4, 4,4,4,4,4,4,4,4, 4,4,4,4,4,4,4,4, 4,4,4,4,4,4,4,4, // Treat all non-ASCII characters as Other
4,4,4,4,4,4,4,4, 4,4,4,4,4,4,4,4, 4,4,4,4,4,4,4,4, 4,4,4,4,4,4,4,4,
4,4,4,4,4,4,4,4, 4,4,4,4,4,4,4,4, 4,4,4,4,4,4,4,4, 4,4,4,4,4,4,4,4,
4,4,4,4,4,4,4,4, 4,4,4,4,4,4,4,4, 4,4,4,4,4,4,4,4, 4,4,4,4,4,4,4,4,
};
struct FCompoundIdentifierResult
{
const TCHAR* Identifier; // Start of identifier
const TCHAR* IdentifierEnd; // End of entire identifier
const TCHAR* IdentifierRootEnd; // End of root token of identifier
};
// Searches for a "compound identifier" (series of symbol tokens separated by dots) that also passes the "RootIdentifierFilter".
// The filter is a mask table of valid identifier start characters indexed by identifier length. Since identifier characters start
// with letters or underscore, we can store a 64-bit mask representing ASCII characters 64..127, as all valid start characters are
// in that range. As an example, if "View" is a valid root identifier, RootIdentifierFilter[4] will have the bit ('V' - 64) set,
// and any other 4 character identifier that doesn't start with that letter can be skipped, saving overhead in the caller.
bool FindNextCompoundIdentifier(const TCHAR*& Search, const uint64 RootIdentifierFilter[64], FCompoundIdentifierResult& OutResult)
{
const TCHAR* SearchChar = Search;
uint8 SearchCharFlag = AsciiFlagTable[(uint8)*SearchChar];
// Scanning loop
while (1)
{
static constexpr uint8 AsciiFlagsEchoVerbatim = (uint8)AsciiFlags::Whitespace | (uint8)AsciiFlags::Other;
static constexpr uint8 AsciiFlagsSymbol = (uint8)AsciiFlags::SymbolStart | (uint8)AsciiFlags::Digit;
static constexpr uint8 AsciiFlagsStartNumberOrDirective = (uint8)AsciiFlags::Digit | (uint8)AsciiFlags::Dot | (uint8)AsciiFlags::Hash;
static constexpr uint8 AsciiFlagsEndNumberOrDirective = (uint8)AsciiFlags::Whitespace | (uint8)AsciiFlags::Other | (uint8)AsciiFlags::Quote | (uint8)AsciiFlags::TerminatorOrSlash;
// Conditions here are organized in expected order of frequency
if (SearchCharFlag & AsciiFlagsEchoVerbatim)
{
SearchChar++;
SearchCharFlag = AsciiFlagTable[(uint8)*SearchChar];
}
else if (SearchCharFlag & (uint8)AsciiFlags::SymbolStart)
{
OutResult.Identifier = SearchChar;
SearchChar++;
while ((SearchCharFlag = AsciiFlagTable[(uint8)*SearchChar]) & AsciiFlagsSymbol)
{
SearchChar++;
}
// Track end of our root identifier
OutResult.IdentifierRootEnd = SearchChar;
// Skip any whitespace before a potential dot
while (SearchCharFlag & ((uint8)AsciiFlags::Whitespace))
{
SearchChar++;
SearchCharFlag = AsciiFlagTable[(uint8)*SearchChar];
}
// If we didn't find a dot, go back to initial scanning state
if (!(SearchCharFlag & ((uint8)AsciiFlags::Dot)))
{
continue;
}
SearchChar++;
SearchCharFlag = AsciiFlagTable[(uint8)*SearchChar];
// Determine if this root identifier passes the filter. If so, we'll continue to parse the rest of the identifier,
// but then go back to scanning. The mask in RootIdentifierFilter starts with ASCII character 64, as token start
// characters are in the range [64..127].
ptrdiff_t IdentifierRootLen = OutResult.IdentifierRootEnd - OutResult.Identifier;
if (IdentifierRootLen >= 64 || !(RootIdentifierFilter[IdentifierRootLen] & (1ull << (*OutResult.Identifier - 64))))
{
// Clear this, marking that we didn't find a candidate root identifier
OutResult.IdentifierRootEnd = nullptr;
}
// Skip any whitespace after dot
while (SearchCharFlag & ((uint8)AsciiFlags::Whitespace))
{
SearchChar++;
SearchCharFlag = AsciiFlagTable[(uint8)*SearchChar];
}
// Check for the start of another symbol after the dot -- if it's not a symbol, switch back to scanning -- some kind of incorrect code
if (!(SearchCharFlag & (uint8)AsciiFlags::SymbolStart))
{
continue;
}
// Repeatedly scan for additional parts of the identifier separated by dots
while (1)
{
SearchChar++;
while ((SearchCharFlag = AsciiFlagTable[(uint8)*SearchChar]) & AsciiFlagsSymbol)
{
SearchChar++;
}
// Track that this may be the end of the identifier (if there's not more dot separated tokens)
OutResult.IdentifierEnd = SearchChar;
// Skip any whitespace before a potential dot
while (SearchCharFlag & ((uint8)AsciiFlags::Whitespace))
{
SearchChar++;
SearchCharFlag = AsciiFlagTable[(uint8)*SearchChar];
}
// If we found something other than a dot, we're done!
if (!(SearchCharFlag & ((uint8)AsciiFlags::Dot)))
{
// Is the root token for this identifier a candidate based on the filter?
if (OutResult.IdentifierRootEnd)
{
Search = SearchChar;
return true;
}
else
{
// If not, go back to initial scanning state
break;
}
}
// Skip the dot
SearchChar++;
SearchCharFlag = AsciiFlagTable[(uint8)*SearchChar];
// Skip any whitespace after dot
while (SearchCharFlag & ((uint8)AsciiFlags::Whitespace))
{
SearchChar++;
SearchCharFlag = AsciiFlagTable[(uint8)*SearchChar];
}
// Did we find the start of another symbol after the dot? If not, break out, some kind of invalid code...
if (!(SearchCharFlag & (uint8)AsciiFlags::SymbolStart))
{
break;
}
}
}
else if (SearchCharFlag & AsciiFlagsStartNumberOrDirective)
{
// Number or directive, skip to Whitespace, Other, or Quote (numbers may contain letters or #, i.e. "1.#INF" for infinity, or "e" for an exponent)
SearchChar++;
while (!((SearchCharFlag = AsciiFlagTable[(uint8)*SearchChar]) & AsciiFlagsEndNumberOrDirective))
{
SearchChar++;
}
}
else if (SearchCharFlag & (uint8)AsciiFlags::Quote)
{
// Quote, skip to next Quote (or maybe end of string if text is malformed)
SearchChar++;
while (*SearchChar && *SearchChar != TEXT('\"'))
{
SearchChar++;
}
// Could be end of string or the quote -- skip over the quote if not the null terminator
if (*SearchChar)
{
SearchChar++;
}
SearchCharFlag = AsciiFlagTable[(uint8)*SearchChar];
}
// Must be null terminator or slash at this point -- we've tested all other possibilities
else if (*SearchChar == TEXT('/'))
{
// Check if this is a commented out block (typically a commented out uniform declaration) and skip over it.
// If the text is bad, there could be a /* right at the end of the string, so we need to check there is at least
// one more character.
if (SearchChar[1] == TEXT('*') && SearchChar[2] != 0)
{
// Search for slash (or end of string), starting at SearchChar + 3. If we find a slash, we'll check the previous
// character to see if it's the end of the comment. Starting at +3 is necessary to avoid matching a slash as the
// first character of the comment, i.e. "/*/".
SearchChar += 3;
while (1)
{
while ((SearchCharFlag = AsciiFlagTable[(uint8)*SearchChar]) != (uint8)AsciiFlags::TerminatorOrSlash)
{
SearchChar++;
}
// Is this the end of the comment?
if (*(SearchChar - 1) == TEXT('*'))
{
if (*SearchChar)
{
SearchChar++;
SearchCharFlag = AsciiFlagTable[(uint8)*SearchChar];
break;
}
}
else
{
// More characters, continue the comment scanning loop, or if somehow at end of string, return false...
if (*SearchChar)
{
SearchChar++;
}
else
{
return false;
}
}
}
}
else
{
// Just a slash, not part of a block comment
SearchChar++;
SearchCharFlag = AsciiFlagTable[(uint8)*SearchChar];
}
}
else
{
// End of string
Search = SearchChar;
return false;
}
}
}
TCHAR* FindNextUniformBufferDefinition(TCHAR* SearchPtr, TCHAR* SourceStart, const TCHAR* UniformBufferStructIdentifier, int32 StructIdentifierLen)
{
while (SearchPtr)
{
SearchPtr = FCString::Strstr(SearchPtr, UniformBufferStructIdentifier);
if (SearchPtr)
{
if (SearchPtr > SourceStart && IsSpaceOrTabOrEOL(*(SearchPtr - 1)) && IsSpaceOrTabOrEOL(*(SearchPtr + StructIdentifierLen)))
{
break;
}
else
{
SearchPtr = SearchPtr + 1;
}
}
}
return SearchPtr;
}
const TCHAR* FindPreviousDot(const TCHAR* SearchPtr, const TCHAR* SearchMin)
{
while ((SearchPtr > SearchMin) && (*SearchPtr != TEXT('.')))
{
SearchPtr--;
}
return SearchPtr;
}
// Unlike the original version, this parses uniform buffers and replaces symbols in a single pass.
void CleanupUniformBufferCodeNew(const FShaderCompilerEnvironment& Environment, FString& PreprocessedShaderSource)
{
TArray<FUniformBufferInfoNew> UniformBufferInfos;
uint64 UniformBufferFilter[64] = { 0 }; // A bit set for valid start characters for uniform buffer name of given length
int32 UniformBuffersByLength[64]; // Linked list head index into UniformBufferInfos by length (connected by "NextWithSameLength")
UniformBufferInfos.Reserve(Environment.UniformBufferMap.Num());
memset(UniformBuffersByLength, 0xff, sizeof(UniformBuffersByLength));
const TCHAR* UniformBufferStructIdentifier = TEXT("UniformBuffer");
const int32 StructIdentifierLen = FCString::Strlen(UniformBufferStructIdentifier);
TCHAR* SourceStart = &PreprocessedShaderSource[0];
TCHAR* SearchPtr = SourceStart;
TCHAR* EndOfPreviousUniformBuffer = SourceStart;
bool bUniformBufferFound;
do
{
// Find the next uniform buffer definition
SearchPtr = FindNextUniformBufferDefinition(SearchPtr, SourceStart, UniformBufferStructIdentifier, StructIdentifierLen);
if (SearchPtr)
{
// Track that we found a uniform buffer, and temporarily null terminate the string so we can parse to this point
bUniformBufferFound = true;
*SearchPtr = 0;
}
else
{
bUniformBufferFound = false;
}
// Parse the source between the last uniform buffer and the current uniform buffer (or potentially the end of the source if no more
// were found). If there are no uniform buffers yet, we don't need to parse anything.
if (UniformBufferInfos.Num())
{
const TCHAR* ParsePtr = EndOfPreviousUniformBuffer;
FCompoundIdentifierResult Result;
while (FindNextCompoundIdentifier(ParsePtr, UniformBufferFilter, Result))
{
// Check if the identifier corresponds to a uniform buffer
FStringView IdentifierRoot(Result.Identifier, Result.IdentifierRootEnd - Result.Identifier);
for (int32 UniformInfoIndex = UniformBuffersByLength[IdentifierRoot.Len()]; UniformInfoIndex != INDEX_NONE; UniformInfoIndex = UniformBufferInfos[UniformInfoIndex].NextWithSameLength)
{
FUniformBufferInfoNew& Info = UniformBufferInfos[UniformInfoIndex];
if (IdentifierRoot.Equals(Info.Name, ESearchCase::CaseSensitive))
{
// Found the uniform buffer, clean up potential whitespace
Result.IdentifierEnd = CompactCompoundIdentifier(const_cast<TCHAR*>(Result.Identifier), const_cast<TCHAR*>(Result.IdentifierEnd));
// Now try to find a matching member. We need to check subsets of the full "identifier", to strip away function calls, components, or child structures.
bool bMatchFound = false;
for (; Result.IdentifierEnd > Result.IdentifierRootEnd; Result.IdentifierEnd = FindPreviousDot(Result.IdentifierEnd - 1, Result.IdentifierRootEnd))
{
FStringView Identifier(Result.Identifier, Result.IdentifierEnd - Result.Identifier);
if (Identifier.Len() < Info.MembersByLength.Num())
{
const FUniformBufferMemberView& MemberView = Info.MembersByLength[Identifier.Len()];
for (int32 MemberIndex = MemberView.MemberOffset; MemberIndex < MemberView.MemberOffset + MemberView.MemberCount; MemberIndex++)
{
if (Info.Members[MemberIndex].NameAsStructMember.Equals(Identifier, ESearchCase::CaseSensitive))
{
bMatchFound = true;
const int32 OriginalTextLen = Info.Members[MemberIndex].NameAsStructMember.Len();
const int32 ReplacementTextLen = Info.Members[MemberIndex].GlobalName.Len();
const TCHAR* GlobalNameStart = GetData(Info.Members[MemberIndex].GlobalName);
TCHAR* IdentifierStart = const_cast<TCHAR*>(Result.Identifier);
int32 Index = 0;
for (; Index < ReplacementTextLen; Index++)
{
IdentifierStart[Index] = GlobalNameStart[Index];
}
for (; Index < OriginalTextLen; Index++)
{
IdentifierStart[Index] = ' ';
}
break;
}
}
if (bMatchFound)
{
break;
}
}
}
break;
}
}
}
}
// Parse the current uniform buffer.
if (bUniformBufferFound)
{
// Unterminate the string (put the first character of the struct identifier back in place) and parse it
*SearchPtr = UniformBufferStructIdentifier[0];
const TCHAR* ConstStructEndPtr = ParseUniformBufferDefinitionNew(SearchPtr + StructIdentifierLen, UniformBufferInfos, UniformBufferFilter, UniformBuffersByLength);
TCHAR* StructEndPtr = &PreprocessedShaderSource[ConstStructEndPtr - &PreprocessedShaderSource[0]];
// Comment out the uniform buffer struct and initializer
*SearchPtr = '/';
*(SearchPtr + 1) = '*';
*(StructEndPtr - 1) = '*';
*StructEndPtr = '/';
EndOfPreviousUniformBuffer = StructEndPtr + 1;
SearchPtr = StructEndPtr + 1;
}
} while (bUniformBufferFound);
}
// The cross compiler doesn't yet support struct initializers needed to construct static structs for uniform buffers
// Replace all uniform buffer struct member references (View.WorldToClip) with a flattened name that removes the struct dependency (View_WorldToClip)
void CleanupUniformBufferCode(const FShaderCompilerEnvironment& Environment, FString& PreprocessedShaderSource)
{
TRACE_CPUPROFILER_EVENT_SCOPE(CleanupUniformBufferCode);
// Putting new code on a chicken switch in case there are issues
static const bool bRunCleanupUniformBufferCodeNew = CVarShaderCompilerCleanupUniformBufferCodeNew.GetValueOnAnyThread();
if (bRunCleanupUniformBufferCodeNew)
{
CleanupUniformBufferCodeNew(Environment, PreprocessedShaderSource);
return;
}
TMap<FStringView, FUniformBufferInfo> UniformBufferInfos;
UniformBufferInfos.Reserve(Environment.UniformBufferMap.Num());
// Build a mapping from uniform buffer name to its members
{
const TCHAR* UniformBufferStructIdentifier = TEXT("UniformBuffer");
const int32 StructIdentifierLen = FCString::Strlen(UniformBufferStructIdentifier);
const TCHAR* SourceStart = &PreprocessedShaderSource[0];
TCHAR* SearchPtr = FCString::Strstr(&PreprocessedShaderSource[0], UniformBufferStructIdentifier);
while (SearchPtr)
{
const ptrdiff_t Offset = SearchPtr - SourceStart;
if (Offset > 0 && IsSpaceOrTabOrEOL(*(SearchPtr - 1)) && IsSpaceOrTabOrEOL(*(SearchPtr + StructIdentifierLen)))
{
const TCHAR* ConstStructEndPtr = ParseUniformBufferDefinition(&PreprocessedShaderSource[0], SearchPtr + StructIdentifierLen, UniformBufferInfos);
TCHAR* StructEndPtr = &PreprocessedShaderSource[ConstStructEndPtr - &PreprocessedShaderSource[0]];
// Comment out the uniform buffer struct and initializer
*SearchPtr = '/';
*(SearchPtr + 1) = '*';
*(StructEndPtr - 1) = '*';
*StructEndPtr = '/';
SearchPtr = FCString::Strstr(StructEndPtr, UniformBufferStructIdentifier);
}
else
{
SearchPtr = FCString::Strstr(SearchPtr + 1, UniformBufferStructIdentifier);
}
}
}
const TCHAR* const SearchSuffixes[] = { TEXT("."), TEXT(" .") };
TStringBuilder<64> UniformBufferName;
// Replace all uniform buffer struct member references (View.WorldToClip) with a flattened name that removes the struct dependency (View_WorldToClip)
for (TMap<FStringView, FUniformBufferInfo>::TConstIterator It(UniformBufferInfos); It; ++It)
{
const FUniformBufferInfo& UniformBufferInfo = It.Value();
// Start after the definition of this uniform buffer
TCHAR* const SearchStart = &PreprocessedShaderSource[UniformBufferInfo.DefinitionEndOffset];
// Trying "." and " ." separately can be faster than checking for each case as we iterate
for (const TCHAR* SearchSuffix : SearchSuffixes)
{
UniformBufferName.Reset();
UniformBufferName.Append(It.Key());
UniformBufferName.Append(SearchSuffix);
// Search for the uniform buffer name first, as an optimization (instead of searching the entire source for every member)
TCHAR* SearchPtr = FCString::Strstr(SearchStart, *UniformBufferName);
while (SearchPtr)
{
// Only match whole symbol
if (IsValidSymbolStart(SearchPtr, UniformBufferName.Len()))
{
// Find the matching member we are replacing
for (const FUniformBufferMemberInfo& MemberInfo : UniformBufferInfo.Members)
{
if (const TCHAR* SearchEnd = MatchStructMemberName(MemberInfo.NameAsStructMember, SearchPtr))
{
const int32 FoundTextLen = SearchEnd - SearchPtr;
const int32 ReplacementTextLen = MemberInfo.GlobalName.Len();
check(FoundTextLen >= ReplacementTextLen);
const TCHAR* SourceStart = GetData(MemberInfo.GlobalName);
TCHAR* const DestinationStart = SearchPtr;
int32 Index = 0;
for (; Index < ReplacementTextLen; Index++)
{
DestinationStart[Index] = SourceStart[Index];
}
// The shader preprocessor inserts spaces after defines
// #define ReflectionStruct OpaqueBasePass.Shared.Reflection
// 'ReflectionStruct.SkyLightCubemapBrightness' becomes 'OpaqueBasePass.Shared.Reflection .SkyLightCubemapBrightness' after MCPP
// In order to convert this struct member reference into a globally unique variable we move the spaces to the end
// 'OpaqueBasePass.Shared.Reflection .SkyLightCubemapBrightness' -> 'OpaqueBasePass_Shared_Reflection_SkyLightCubemapBrightness '
for (; Index < FoundTextLen; Index++)
{
// If we passed MatchStructMemberName, it should not be possible to overwrite the null terminator
check(DestinationStart[Index] != '\0');
DestinationStart[Index] = ' ';
}
break;
}
}
}
SearchPtr = FCString::Strstr(SearchPtr + UniformBufferName.Len(), *UniformBufferName);
}
}
}
}
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// Process TEXT() macro to convert them into GPU ASCII characters
static FString ParseText(const TCHAR* StartPtr, const TCHAR*& EndPtr)
{
const TCHAR* OpeningBracePtr = FindNextChar(StartPtr, '(');
check(OpeningBracePtr);
const TCHAR* ClosingBracePtr = FindMatchingClosingParenthesis(OpeningBracePtr + 1);
check(ClosingBracePtr);
FString Out;
if (OpeningBracePtr && ClosingBracePtr)
{
const TCHAR* CurrPtr = OpeningBracePtr;
do
{
Out += *CurrPtr;
CurrPtr++;
} while (CurrPtr != ClosingBracePtr+1);
}
EndPtr = ClosingBracePtr;
return Out;
}
static void ConvertTextToAsciiCharacter(const FString& InText, FString& OutText, FString& OutEncodedText)
{
const uint32 CharCount = InText.Len();
OutEncodedText.Reserve(CharCount * 3); // ~2 digits per character + a comma
OutText = InText;
for (uint32 CharIt = 0; CharIt < CharCount; ++CharIt)
{
const char C = InText[CharIt];
OutEncodedText.AppendInt(uint8(C));
if (CharIt + 1 != CharCount)
{
OutEncodedText += ',';
}
}
}
struct FAssertParsingStack
{
bool IsAssert() const { return BeginPtr != nullptr && EndPtr != nullptr; }
const TCHAR* BeginPtr = nullptr;
const TCHAR* EndPtr = nullptr;
};
static bool SearchText(const TCHAR*& InOut, FAssertParsingStack& OutAssertStack)
{
const TCHAR* TextIdentifier = TEXT("TEXT(");
const TCHAR* AssertIdentifier = TEXT("UEReportAssertWithPayload(");
// Find the next TEXT() or UEReportAssertWithPayload()
const TCHAR* PrintfPtr = FCString::Strstr(InOut, TextIdentifier);
const TCHAR* AssertPtr = FCString::Strstr(InOut, AssertIdentifier);
// 1. Default is current TEXT parsing
InOut = PrintfPtr;
// 2. If current pointer is beyond the current assert context, reset the context
if (InOut >= OutAssertStack.EndPtr)
{
OutAssertStack.BeginPtr = nullptr;
OutAssertStack.EndPtr = nullptr;
}
// 3. If we are within an assert context, continue to part TEXT within that context
if (OutAssertStack.IsAssert())
{
// Nothing to do. InOut will be initialized by PrintfPtr, which is the next TEXT block.
}
// 4. If a new assert is detected, start a new assert context.
else if (AssertPtr && AssertPtr < PrintfPtr)
{
// Sanity check
check(!OutAssertStack.IsAssert());
// Check if the current assert is valid, i.e., containt a TEXT() argument
const TCHAR* EndPtr = nullptr;
const FString Tmp = ParseText(AssertPtr, EndPtr);
const bool bIsValid = FCString::Strstr(&Tmp[0], TextIdentifier) != nullptr;
if (bIsValid)
{
OutAssertStack.BeginPtr = AssertPtr;
OutAssertStack.EndPtr = EndPtr;
InOut = FCString::Strstr(AssertPtr, TextIdentifier);
}
else
{
const uint32 LenAssertIdentifier = FString(AssertIdentifier).Len();
AssertPtr += LenAssertIdentifier;
return SearchText(AssertPtr, OutAssertStack);
}
}
return InOut != nullptr;
}
// Simple token matching and expansion to replace TEXT macro into supported character string
void TransformStringIntoCharacterArray(FString& PreprocessedShaderSource, TArray<FShaderDiagnosticData>* OutDiagnosticDatas)
{
// Early out if input is empty; '&PreprocessedShaderSource[0]' below does not return a valid pointer for empty FString
if (PreprocessedShaderSource.IsEmpty())
{
return;
}
struct FTextEntry
{
uint32 Index;
uint32 Hash;
uint32 Offset;
bool bIsAssert;
FString SourceText;
FString ConvertedText;
FString EncodedText;
};
TArray<FTextEntry> Entries;
// 1. Find all TEXT strings
// 2. Add a text entry
// 3. Replace TEXT by its entry number
uint32 GlobalCount = 0;
uint32 AssertCount = 0;
uint32 PrintfCount = 0;
{
const FString InitHashBegin(TEXT("InitShaderPrintText("));
const FString InitHashEnd(TEXT(")"));
FAssertParsingStack AssertParsingStack;
const TCHAR* SearchPtr = &PreprocessedShaderSource[0];
while (SearchText(SearchPtr, AssertParsingStack))
{
const TCHAR* EndPtr = nullptr;
FString Text = ParseText(SearchPtr, EndPtr);
if (EndPtr)
{
// Trim enclosing
Text.RemoveFromEnd("\")");
Text.RemoveFromStart("(\"");
// Register entry and convert text
const uint32 EntryIndex = Entries.Num();
uint32 ValidCharCount = 0;
FTextEntry& Entry = Entries.AddDefaulted_GetRef();
Entry.Index = EntryIndex;
Entry.Offset = GlobalCount;
Entry.SourceText = Text;
ConvertTextToAsciiCharacter(Entry.SourceText, Entry.ConvertedText, Entry.EncodedText);
Entry.Hash = CityHash32((const char*)Entry.SourceText.GetCharArray().GetData(), sizeof(FString::ElementType) * Entry.SourceText.Len());
Entry.bIsAssert = AssertParsingStack.IsAssert();
if (Entry.bIsAssert)
{
++AssertCount;
}
else
{
++PrintfCount;
}
// Sanity check
uint32 HCheck = CityHash32((const char*)Entry.SourceText.GetCharArray().GetData(), sizeof(FString::ElementType) * Entry.SourceText.Len());
check(HCheck == Entry.Hash);
GlobalCount += Entry.ConvertedText.Len();
// Replace string
const TCHAR* StartPtr = &PreprocessedShaderSource[0];
const uint32 StartIndex = SearchPtr - StartPtr;
const uint32 CharCount = (EndPtr - SearchPtr) + 1;
PreprocessedShaderSource.RemoveAt(StartIndex, CharCount);
// * If this is an assert, replace the text with a simple hash
// * If this is a regular print, replace the text with IniShaderPrintText for runtime lookup
if (Entry.bIsAssert)
{
const FString HashString = FString::Printf(TEXT("%u"), Entry.Hash);
PreprocessedShaderSource.InsertAt(StartIndex, HashString);
}
else
{
const FString HashText = InitHashBegin + FString::FromInt(EntryIndex) + InitHashEnd;
PreprocessedShaderSource.InsertAt(StartIndex, HashText);
}
// Update SearchPtr, as PreprocessedShaderSource has been modified, and its memory could have been reallocated, causing SearchPtr to be invalid.
SearchPtr = &PreprocessedShaderSource[0] + StartIndex;
}
}
}
// 4. Write a global struct containing all the entries
// 5. Write the function for fetching character for a given entry index
const uint32 EntryCount = Entries.Num();
FString TextChars;
if (PrintfCount>0 && EntryCount>0 && GlobalCount>0)
{
// 1. Encoded character for each text entry within a single global char array
TextChars = FString::Printf(TEXT("static const uint TEXT_CHARS[%d] = {\n"), GlobalCount);
for (FTextEntry& Entry : Entries)
{
TextChars += FString::Printf(TEXT("\t%s%s // %d: \"%s\"\n"), *Entry.EncodedText, Entry.Index < EntryCount - 1 ? TEXT(",") : TEXT(""), Entry.Index, * Entry.SourceText);
}
TextChars += TEXT("};\n\n");
// 2. Offset within the global array
TextChars += FString::Printf(TEXT("static const uint TEXT_OFFSETS[%d] = {\n"), EntryCount+1);
for (FTextEntry& Entry : Entries)
{
TextChars += FString::Printf(TEXT("\t%d, // %d: \"%s\"\n"), Entry.Offset, Entry.Index, *Entry.SourceText);
}
TextChars += FString::Printf(TEXT("\t%d // end\n"), GlobalCount);
TextChars += TEXT("};\n\n");
// 3. Entry hashes
TextChars += TEXT("// Hashes are computed using the CityHash32 function\n");
TextChars += FString::Printf(TEXT("static const uint TEXT_HASHES[%d] = {\n"), EntryCount);
for (FTextEntry& Entry : Entries)
{
TextChars += FString::Printf(TEXT("\t0x%x%s // %d: \"%s\"\n"), Entry.Hash, Entry.Index < EntryCount - 1 ? TEXT(",") : TEXT(""), Entry.Index, * Entry.SourceText);
}
TextChars += TEXT("};\n\n");
TextChars += TEXT("uint ShaderPrintGetChar(uint InIndex) { return TEXT_CHARS[InIndex]; }\n");
TextChars += TEXT("uint ShaderPrintGetOffset(FShaderPrintText InText) { return TEXT_OFFSETS[InText.Index]; }\n");
TextChars += TEXT("uint ShaderPrintGetHash(FShaderPrintText InText) { return TEXT_HASHES[InText.Index]; }\n");
}
else
{
TextChars += TEXT("uint ShaderPrintGetChar(uint Index) { return 0; }\n");
TextChars += TEXT("uint ShaderPrintGetOffset(FShaderPrintText InText) { return 0; }\n");
TextChars += TEXT("uint ShaderPrintGetHash(FShaderPrintText InText) { return 0; }\n");
}
// 6. Insert global struct data + print function
{
const TCHAR* InsertToken = TEXT("GENERATED_SHADER_PRINT");
const TCHAR* SearchPtr = FCString::Strstr(&PreprocessedShaderSource[0], InsertToken);
if (SearchPtr)
{
// Replace string
const TCHAR* StartPtr = &PreprocessedShaderSource[0];
const uint32 StartIndex = SearchPtr - StartPtr;
const uint32 CharCount = FCString::Strlen(InsertToken);
PreprocessedShaderSource.RemoveAt(StartIndex, CharCount);
PreprocessedShaderSource.InsertAt(StartIndex, TextChars);
}
}
// 7. Insert assert data into shader compilation output for runtime CPU lookup
if (OutDiagnosticDatas && AssertCount > 0)
{
OutDiagnosticDatas->Reserve(OutDiagnosticDatas->Num() + AssertCount);
for (const FTextEntry& E : Entries)
{
if (E.bIsAssert)
{
FShaderDiagnosticData& Data = OutDiagnosticDatas->AddDefaulted_GetRef();
Data.Hash = E.Hash;
Data.Message = E.SourceText;
}
}
}
}
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
FString CreateShaderCompilerWorkerDirectCommandLine(const FShaderCompilerInput& Input)
{
FString Text(TEXT("-directcompile -format="));
Text += Input.ShaderFormat.GetPlainNameString();
Text += TEXT(" -entry=\"");
Text += Input.EntryPointName;
Text += TEXT("\" -shaderPlatformName=");
Text += Input.ShaderPlatformName.GetPlainNameString();
switch (Input.Target.Frequency)
{
case SF_Vertex: Text += TEXT(" -vs"); break;
case SF_Mesh: Text += TEXT(" -ms"); break;
case SF_Amplification: Text += TEXT(" -as"); break;
case SF_Geometry: Text += TEXT(" -gs"); break;
case SF_Pixel: Text += TEXT(" -ps"); break;
case SF_Compute: Text += TEXT(" -cs"); break;
#if RHI_RAYTRACING
case SF_RayGen: Text += TEXT(" -rgs"); break;
case SF_RayMiss: Text += TEXT(" -rms"); break;
case SF_RayHitGroup: Text += TEXT(" -rhs"); break;
case SF_RayCallable: Text += TEXT(" -rcs"); break;
#endif // RHI_RAYTRACING
default: break;
}
if (Input.bCompilingForShaderPipeline)
{
Text += TEXT(" -pipeline");
}
if (Input.bIncludeUsedOutputs)
{
Text += TEXT(" -usedoutputs=");
for (int32 Index = 0; Index < Input.UsedOutputs.Num(); ++Index)
{
if (Index != 0)
{
Text += TEXT("+");
}
Text += Input.UsedOutputs[Index];
}
}
Text += TEXT(" ");
Text += Input.DumpDebugInfoPath / Input.GetSourceFilename();
Text += TEXT(" -cflags=");
Text += FString::Printf(TEXT("%llu"), Input.Environment.CompilerFlags.GetData());
// When we're running in directcompile mode, we don't to spam the crash reporter
Text += TEXT(" -nocrashreports");
return Text;
}
static FString CreateShaderConductorCommandLine(const FShaderCompilerInput& Input, const FString& SourceFilename, EShaderConductorTarget SCTarget)
{
const TCHAR* Stage = nullptr;
switch (Input.Target.GetFrequency())
{
case SF_Vertex: Stage = TEXT("vs"); break;
case SF_Pixel: Stage = TEXT("ps"); break;
case SF_Geometry: Stage = TEXT("gs"); break;
case SF_Compute: Stage = TEXT("cs"); break;
default: return FString();
}
const TCHAR* Target = nullptr;
switch (SCTarget)
{
case EShaderConductorTarget::Dxil: Target = TEXT("dxil"); break;
case EShaderConductorTarget::Spirv: Target = TEXT("spirv"); break;
default: return FString();
}
FString CmdLine = TEXT("-E ") + Input.EntryPointName;
//CmdLine += TEXT("-O ") + *(CompilerInfo.Input.D);
CmdLine += TEXT(" -S ") + FString(Stage);
CmdLine += TEXT(" -T ");
CmdLine += Target;
CmdLine += TEXT(" -I ") + (Input.DumpDebugInfoPath / SourceFilename);
return CmdLine;
}
SHADERCOMPILERCOMMON_API void WriteShaderConductorCommandLine(const FShaderCompilerInput& Input, const FString& SourceFilename, EShaderConductorTarget Target)
{
FArchive* FileWriter = IFileManager::Get().CreateFileWriter(*(Input.DumpDebugInfoPath / TEXT("ShaderConductorCmdLine.txt")));
if (FileWriter)
{
FString CmdLine = CreateShaderConductorCommandLine(Input, SourceFilename, Target);
FileWriter->Serialize(TCHAR_TO_ANSI(*CmdLine), CmdLine.Len());
FileWriter->Close();
delete FileWriter;
}
}
static uint32 Mali_ExtractNumberInstructions(const FString &MaliOutput)
{
uint32 ReturnedNum = 0;
// Parse the instruction count
int32 InstructionStringLength = FPlatformString::Strlen(TEXT("Instructions Emitted:"));
int32 InstructionsIndex = MaliOutput.Find(TEXT("Instructions Emitted:"));
// new version of mali offline compiler uses a different string in its output
if (InstructionsIndex == INDEX_NONE)
{
InstructionStringLength = FPlatformString::Strlen(TEXT("Total instruction cycles:"));
InstructionsIndex = MaliOutput.Find(TEXT("Total instruction cycles:"));
}
if (InstructionsIndex != INDEX_NONE && InstructionsIndex + InstructionStringLength < MaliOutput.Len())
{
const int32 EndIndex = MaliOutput.Find(TEXT("\n"), ESearchCase::IgnoreCase, ESearchDir::FromStart, InstructionsIndex + InstructionStringLength);
if (EndIndex != INDEX_NONE)
{
int32 StartIndex = InstructionsIndex + InstructionStringLength;
bool bFoundNrStart = false;
int32 NumberIndex = 0;
while (StartIndex < EndIndex)
{
if (FChar::IsDigit(MaliOutput[StartIndex]) && !bFoundNrStart)
{
// found number's beginning
bFoundNrStart = true;
NumberIndex = StartIndex;
}
else if (FChar::IsWhitespace(MaliOutput[StartIndex]) && bFoundNrStart)
{
// found number's end
bFoundNrStart = false;
const FString NumberString = MaliOutput.Mid(NumberIndex, StartIndex - NumberIndex);
const float fNrInstructions = FCString::Atof(*NumberString);
ReturnedNum += (uint32)FMath::Max(0.0, ceil(fNrInstructions));
}
++StartIndex;
}
}
}
return ReturnedNum;
}
static FString Mali_ExtractErrors(const FString &MaliOutput)
{
FString ReturnedErrors;
const int32 GlobalErrorIndex = MaliOutput.Find(TEXT("Compilation failed."));
// find each 'line' that begins with token "ERROR:" and copy it to the returned string
if (GlobalErrorIndex != INDEX_NONE)
{
int32 CompilationErrorIndex = MaliOutput.Find(TEXT("ERROR:"));
while (CompilationErrorIndex != INDEX_NONE)
{
int32 EndLineIndex = MaliOutput.Find(TEXT("\n"), ESearchCase::CaseSensitive, ESearchDir::FromStart, CompilationErrorIndex + 1);
EndLineIndex = EndLineIndex == INDEX_NONE ? MaliOutput.Len() - 1 : EndLineIndex;
ReturnedErrors += MaliOutput.Mid(CompilationErrorIndex, EndLineIndex - CompilationErrorIndex + 1);
CompilationErrorIndex = MaliOutput.Find(TEXT("ERROR:"), ESearchCase::CaseSensitive, ESearchDir::FromStart, EndLineIndex);
}
}
return ReturnedErrors;
}
void CompileOfflineMali(const FShaderCompilerInput& Input, FShaderCompilerOutput& ShaderOutput, const ANSICHAR* ShaderSource, const int32 SourceSize, bool bVulkanSpirV, const ANSICHAR* VulkanSpirVEntryPoint)
{
const bool bCompilerExecutableExists = FPaths::FileExists(Input.ExtraSettings.OfflineCompilerPath);
if (bCompilerExecutableExists)
{
const auto Frequency = (EShaderFrequency)Input.Target.Frequency;
const FString WorkingDir(FPlatformProcess::ShaderDir());
FString CompilerPath = Input.ExtraSettings.OfflineCompilerPath;
FString CompilerCommand = "";
// add process and thread ids to the file name to avoid collision between workers
auto ProcID = FPlatformProcess::GetCurrentProcessId();
auto ThreadID = FPlatformTLS::GetCurrentThreadId();
FString GLSLSourceFile = WorkingDir / TEXT("GLSLSource#") + FString::FromInt(ProcID) + TEXT("#") + FString::FromInt(ThreadID);
// setup compilation arguments
TCHAR *FileExt = nullptr;
switch (Frequency)
{
case SF_Vertex:
GLSLSourceFile += bVulkanSpirV ? TEXT(".spv") : TEXT(".vert");
CompilerCommand += TEXT(" -v");
break;
case SF_Pixel:
GLSLSourceFile += bVulkanSpirV ? TEXT(".spv") : TEXT(".frag");
CompilerCommand += TEXT(" -f");
break;
case SF_Geometry:
GLSLSourceFile += bVulkanSpirV ? TEXT(".spv") : TEXT(".geom");
CompilerCommand += TEXT(" -g");
break;
case SF_Compute:
GLSLSourceFile += bVulkanSpirV ? TEXT(".spv") : TEXT(".comp");
CompilerCommand += TEXT(" -C");
break;
default:
GLSLSourceFile += TEXT(".shd");
break;
}
if (bVulkanSpirV)
{
CompilerCommand += FString::Printf(TEXT(" -y %s -p"), ANSI_TO_TCHAR(VulkanSpirVEntryPoint));
}
else
{
CompilerCommand += TEXT(" -s");
}
FArchive* Ar = IFileManager::Get().CreateFileWriter(*GLSLSourceFile, FILEWRITE_EvenIfReadOnly);
if (Ar == nullptr)
{
return;
}
// write out the shader source to a file and use it below as input for the compiler
Ar->Serialize((void*)ShaderSource, SourceSize);
delete Ar;
FString StdOut;
FString StdErr;
int32 ReturnCode = 0;
// Since v6.2.0, Mali compiler needs to be started in the executable folder or it won't find "external/glslangValidator" for Vulkan
FString CompilerWorkingDirectory = FPaths::GetPath(CompilerPath);
if (!CompilerWorkingDirectory.IsEmpty() && FPaths::DirectoryExists(CompilerWorkingDirectory))
{
// compiler command line contains flags and the GLSL source file name
CompilerCommand += " " + FPaths::ConvertRelativePathToFull(GLSLSourceFile);
// Run Mali shader compiler and wait for completion
FPlatformProcess::ExecProcess(*CompilerPath, *CompilerCommand, &ReturnCode, &StdOut, &StdErr, *CompilerWorkingDirectory);
}
else
{
StdErr = "Couldn't find Mali offline compiler at " + CompilerPath;
}
// parse Mali's output and extract instruction count or eventual errors
ShaderOutput.bSucceeded = (ReturnCode >= 0);
if (ShaderOutput.bSucceeded)
{
// check for errors
if (StdErr.Len())
{
ShaderOutput.bSucceeded = false;
FShaderCompilerError& NewError = ShaderOutput.Errors.AddDefaulted_GetRef();
NewError.StrippedErrorMessage = TEXT("[Mali Offline Complier]\n") + StdErr;
}
else
{
FString Errors = Mali_ExtractErrors(StdOut);
if (Errors.Len())
{
FShaderCompilerError& NewError = ShaderOutput.Errors.AddDefaulted_GetRef();
NewError.StrippedErrorMessage = TEXT("[Mali Offline Complier]\n") + Errors;
ShaderOutput.bSucceeded = false;
}
}
// extract instruction count
if (ShaderOutput.bSucceeded)
{
ShaderOutput.NumInstructions = Mali_ExtractNumberInstructions(StdOut);
}
}
// we're done so delete the shader file
IFileManager::Get().Delete(*GLSLSourceFile, true, true);
}
}
// sensible default path size; TStringBuilder will allocate if it needs to
const FString GetDebugFileName(
const FShaderCompilerInput& Input,
const UE::ShaderCompilerCommon::FDebugShaderDataOptions& Options,
const TCHAR* BaseFilename)
{
TStringBuilder<512> PathBuilder;
const TCHAR* Prefix = (Options.FilenamePrefix && *Options.FilenamePrefix) ? Options.FilenamePrefix : TEXT("");
FStringView Filename = (BaseFilename && *BaseFilename) ? BaseFilename : Input.GetSourceFilenameView();
FPathViews::Append(PathBuilder, Input.DumpDebugInfoPath, Prefix);
PathBuilder << Filename;
return PathBuilder.ToString();
}
namespace UE::ShaderCompilerCommon
{
FString FDebugShaderDataOptions::GetDebugShaderPath(const FShaderCompilerInput& Input) const
{
return GetDebugFileName(Input, *this, OverrideBaseFilename);
}
void FBaseShaderFormat::OutputDebugData(
const FShaderCompilerInput& Input,
const FShaderPreprocessOutput& PreprocessOutput,
const FShaderCompilerOutput& Output) const
{
DumpExtendedDebugShaderData(Input, PreprocessOutput, Output);
}
void DumpDebugShaderData(const FShaderCompilerInput& Input, const FString& PreprocessedSource, const FDebugShaderDataOptions& Options)
{
if (!Input.DumpDebugInfoEnabled())
{
return;
}
FString Contents = UE::ShaderCompilerCommon::GetDebugShaderContents(Input, PreprocessedSource, Options);
FFileHelper::SaveStringToFile(Contents, *Options.GetDebugShaderPath(Input));
if (EnumHasAnyFlags(Input.DebugInfoFlags, EShaderDebugInfoFlags::DirectCompileCommandLine) && !Options.bSourceOnly)
{
FFileHelper::SaveStringToFile(CreateShaderCompilerWorkerDirectCommandLine(Input), *GetDebugFileName(Input, Options, TEXT("DirectCompile.txt")));
}
}
void DumpExtendedDebugShaderData(
const FShaderCompilerInput& Input,
const FShaderPreprocessOutput& PreprocessOutput,
const FShaderCompilerOutput& Output,
const FDebugShaderDataOptions& Options)
{
if (Input.bCachePreprocessed && EnumHasAnyFlags(Input.DebugInfoFlags, EShaderDebugInfoFlags::DetailedSource))
{
FDebugShaderDataOptions PrefixedOptions(Options);
uint32 SlackLen = Options.FilenamePrefix ? FCString::Strlen(Options.FilenamePrefix) : 0;
FString StrippedPrefix(TEXT("Stripped_"), SlackLen);
FString PreprocessedPrefix(TEXT("Preprocessed_"), SlackLen);
if (Options.FilenamePrefix)
{
StrippedPrefix += Options.FilenamePrefix;
PreprocessedPrefix += Options.FilenamePrefix;
}
PrefixedOptions.FilenamePrefix = *StrippedPrefix;
FFileHelper::SaveStringToFile(PreprocessOutput.GetSource(), *PrefixedOptions.GetDebugShaderPath(Input));
PrefixedOptions.FilenamePrefix = *PreprocessedPrefix;
FFileHelper::SaveStringToFile(PreprocessOutput.GetUnstrippedSource(), *PrefixedOptions.GetDebugShaderPath(Input));
}
const FString& SourceToDump = Output.ModifiedShaderSource.IsEmpty() ? PreprocessOutput.GetSource() : Output.ModifiedShaderSource;
DumpDebugShaderData(Input, SourceToDump, Options);
FFileHelper::SaveStringToFile(Output.OutputHash.ToString(), *GetDebugFileName(Input, Options, TEXT("OutputHash.txt")), FFileHelper::EEncodingOptions::ForceAnsi);
if (EnumHasAnyFlags(Input.DebugInfoFlags, EShaderDebugInfoFlags::Diagnostics))
{
FString Merged;
for (const FShaderCompilerError& Diag : Output.Errors)
{
Merged += Diag.GetErrorStringWithLineMarker() + "\n";
}
if (!Merged.IsEmpty())
{
FFileHelper::SaveStringToFile(Merged, *GetDebugFileName(Input, Options, TEXT("Diagnostics.txt")), FFileHelper::EEncodingOptions::ForceAnsi);
}
}
if (EnumHasAnyFlags(Input.DebugInfoFlags, EShaderDebugInfoFlags::InputHash))
{
FFileHelper::SaveStringToFile(LexToString(Input.Hash), *GetDebugFileName(Input, Options, TEXT("InputHash.txt")), FFileHelper::EEncodingOptions::ForceAnsi);
}
if (EnumHasAnyFlags(Input.DebugInfoFlags, EShaderDebugInfoFlags::ShaderCodeBinary))
{
FString ShaderCodeFileName = *GetDebugFileName(Input, Options, TEXT("ShaderCode.bin"));
if (Output.ShaderCode.IsCompressed())
{
// always output decompressed code as it's slightly more useful for A/B comparisons
TArray<uint8> DecompressedCode;
DecompressedCode.SetNum(Output.ShaderCode.GetUncompressedSize());
bool bSucceed = FCompression::UncompressMemory(NAME_Oodle, DecompressedCode.GetData(), DecompressedCode.Num(), Output.ShaderCode.GetReadAccess().GetData(), Output.ShaderCode.GetShaderCodeSize());
FFileHelper::SaveArrayToFile(DecompressedCode, *ShaderCodeFileName);
}
else
{
FFileHelper::SaveArrayToFile(Output.ShaderCode.GetReadAccess(), *ShaderCodeFileName);
}
}
for (const FDebugShaderDataOptions::FAdditionalOutput& AdditionalOutput : Options.AdditionalOutputs)
{
FFileHelper::SaveStringToFile(AdditionalOutput.Data, *GetDebugFileName(Input, Options, AdditionalOutput.BaseFileName), FFileHelper::EEncodingOptions::ForceAnsi);
}
}
FString GetDebugShaderContents(const FShaderCompilerInput& Input, const FString& PreprocessedSource, const FDebugShaderDataOptions& Options)
{
FString Contents = Options.AppendPreSource ? Options.AppendPreSource() : FString();
Contents += PreprocessedSource;
if (Options.AppendPostSource)
{
Contents += Options.AppendPostSource();
}
Contents += TEXT("\n");
Contents += CrossCompiler::CreateResourceTableFromEnvironment(Input.Environment);
Contents += TEXT("#if 0 /*DIRECT COMPILE*/\n");
Contents += CreateShaderCompilerWorkerDirectCommandLine(Input);
Contents += TEXT("\n#endif /*DIRECT COMPILE*/\n");
if (!Input.DebugDescription.IsEmpty())
{
Contents += TEXT("//");
Contents += Input.DebugDescription;
Contents += TEXT("\n");
}
return Contents;
}
}
void DumpDebugShaderText(const FShaderCompilerInput& Input, const FString& InSource, const FString& FileExtension)
{
FTCHARToUTF8 StringConverter(InSource.GetCharArray().GetData(), InSource.Len());
// Provide mutable container to pass string to FArchive inside inner function
TArray<ANSICHAR> SourceAnsi;
SourceAnsi.SetNum(InSource.Len() + 1);
FCStringAnsi::Strncpy(SourceAnsi.GetData(), (ANSICHAR*)StringConverter.Get(), SourceAnsi.Num());
// Forward temporary container to primary function
DumpDebugShaderText(Input, SourceAnsi.GetData(), InSource.Len(), FileExtension);
}
void DumpDebugShaderText(const FShaderCompilerInput& Input, ANSICHAR* InSource, int32 InSourceLength, const FString& FileExtension)
{
DumpDebugShaderBinary(Input, InSource, InSourceLength * sizeof(ANSICHAR), FileExtension);
}
void DumpDebugShaderText(const FShaderCompilerInput& Input, ANSICHAR* InSource, int32 InSourceLength, const FString& FileName, const FString& FileExtension)
{
DumpDebugShaderBinary(Input, InSource, InSourceLength * sizeof(ANSICHAR), FileName, FileExtension);
}
void DumpDebugShaderBinary(const FShaderCompilerInput& Input, void* InData, int32 InDataByteSize, const FString& FileExtension)
{
if (InData != nullptr && InDataByteSize > 0 && !FileExtension.IsEmpty())
{
const FString Filename = Input.DumpDebugInfoPath / FPaths::GetBaseFilename(Input.GetSourceFilename()) + TEXT(".") + FileExtension;
if (TUniquePtr<FArchive> FileWriter = TUniquePtr<FArchive>(IFileManager::Get().CreateFileWriter(*Filename)))
{
FileWriter->Serialize(InData, InDataByteSize);
FileWriter->Close();
}
}
}
void DumpDebugShaderBinary(const FShaderCompilerInput& Input, void* InData, int32 InDataByteSize, const FString& FileName, const FString& FileExtension)
{
if (InData != nullptr && InDataByteSize > 0 && !FileExtension.IsEmpty())
{
const FString Filename = Input.DumpDebugInfoPath / FileName + TEXT(".") + FileExtension;
if (TUniquePtr<FArchive> FileWriter = TUniquePtr<FArchive>(IFileManager::Get().CreateFileWriter(*Filename)))
{
FileWriter->Serialize(InData, InDataByteSize);
FileWriter->Close();
}
}
}
static void DumpDebugShaderDisassembled(const FShaderCompilerInput& Input, CrossCompiler::EShaderConductorIR Language, void* InData, int32 InDataByteSize, const FString& FileExtension)
{
if (InData != nullptr && InDataByteSize > 0 && !FileExtension.IsEmpty())
{
TArray<ANSICHAR> AssemblyText;
if (CrossCompiler::FShaderConductorContext::Disassemble(Language, InData, InDataByteSize, AssemblyText))
{
// Assembly text contains NUL terminator, so text lenght is |array|-1
DumpDebugShaderText(Input, AssemblyText.GetData(), AssemblyText.Num() - 1, FileExtension);
}
}
}
void DumpDebugShaderDisassembledSpirv(const FShaderCompilerInput& Input, void* InData, int32 InDataByteSize, const FString& FileExtension)
{
DumpDebugShaderDisassembled(Input, CrossCompiler::EShaderConductorIR::Spirv, InData, InDataByteSize, FileExtension);
}
void DumpDebugShaderDisassembledDxil(const FShaderCompilerInput& Input, void* InData, int32 InDataByteSize, const FString& FileExtension)
{
DumpDebugShaderDisassembled(Input, CrossCompiler::EShaderConductorIR::Dxil, InData, InDataByteSize, FileExtension);
}
namespace CrossCompiler
{
FString CreateResourceTableFromEnvironment(const FShaderCompilerEnvironment& Environment)
{
FString Line = TEXT("\n#if 0 /*BEGIN_RESOURCE_TABLES*/\n");
for (auto Pair : Environment.UniformBufferMap)
{
Line += FString::Printf(TEXT("%s, %d\n"), *Pair.Key, Pair.Value.LayoutHash);
}
Line += TEXT("NULL, 0\n");
for (const FUniformResourceEntry& Entry : Environment.ResourceTableMap.Resources)
{
Line += FString::Printf(TEXT("%s, %s, %d, %d\n"), Entry.UniformBufferMemberName, *FString(Entry.GetUniformBufferName()), Entry.Type, Entry.ResourceIndex);
}
Line += TEXT("NULL, NULL, 0, 0\n");
Line += TEXT("#endif /*END_RESOURCE_TABLES*/\n");
return Line;
}
void CreateEnvironmentFromResourceTable(const FString& String, FShaderCompilerEnvironment& OutEnvironment)
{
FString Prolog = TEXT("#if 0 /*BEGIN_RESOURCE_TABLES*/");
int32 FoundBegin = String.Find(Prolog, ESearchCase::CaseSensitive);
if (FoundBegin == INDEX_NONE)
{
return;
}
int32 FoundEnd = String.Find(TEXT("#endif /*END_RESOURCE_TABLES*/"), ESearchCase::CaseSensitive, ESearchDir::FromStart, FoundBegin);
if (FoundEnd == INDEX_NONE)
{
return;
}
// +1 for EOL
const TCHAR* Ptr = &String[FoundBegin + 1 + Prolog.Len()];
while (*Ptr == '\r' || *Ptr == '\n')
{
++Ptr;
}
const TCHAR* PtrEnd = &String[FoundEnd];
while (Ptr < PtrEnd)
{
FString UB;
if (!CrossCompiler::ParseIdentifier(Ptr, UB))
{
return;
}
if (!CrossCompiler::Match(Ptr, TEXT(", ")))
{
return;
}
int32 Hash;
if (!CrossCompiler::ParseSignedNumber(Ptr, Hash))
{
return;
}
// Optional \r
CrossCompiler::Match(Ptr, '\r');
if (!CrossCompiler::Match(Ptr, '\n'))
{
return;
}
if (UB == TEXT("NULL") && Hash == 0)
{
break;
}
FUniformBufferEntry& UniformBufferEntry = OutEnvironment.UniformBufferMap.FindOrAdd(UB);
UniformBufferEntry.LayoutHash = (uint32)Hash;
if (!UniformBufferEntry.MemberNameBuffer)
{
TArray<TCHAR>* MemberNameBuffer = new TArray<TCHAR>();
UniformBufferEntry.MemberNameBuffer = MakeShareable(MemberNameBuffer);
}
}
// Need to iterate through Uniform Buffer Map to add strings to correct MemberNameBuffer storage
auto UniformBufferMapIt = OutEnvironment.UniformBufferMap.begin();
// If we exit parse early due to error, we still want to fixup the string names for the members we found,
// so the partial data isn't corrupt.
struct FFixupOnExit
{
FFixupOnExit(FShaderCompilerEnvironment& OutEnvironment) : Environment(OutEnvironment) {}
~FFixupOnExit()
{
Environment.ResourceTableMap.FixupOnLoad(Environment.UniformBufferMap);
}
FShaderCompilerEnvironment& Environment;
};
FFixupOnExit FixupOnExit(OutEnvironment);
while (Ptr < PtrEnd)
{
FString Name;
if (!CrossCompiler::ParseIdentifier(Ptr, Name))
{
return;
}
if (!CrossCompiler::Match(Ptr, TEXT(", ")))
{
return;
}
FString UB;
if (!CrossCompiler::ParseIdentifier(Ptr, UB))
{
return;
}
if (!CrossCompiler::Match(Ptr, TEXT(", ")))
{
return;
}
int32 Type;
if (!CrossCompiler::ParseSignedNumber(Ptr, Type))
{
return;
}
if (!CrossCompiler::Match(Ptr, TEXT(", ")))
{
return;
}
int32 ResourceIndex;
if (!CrossCompiler::ParseSignedNumber(Ptr, ResourceIndex))
{
return;
}
// Optional
CrossCompiler::Match(Ptr, '\r');
if (!CrossCompiler::Match(Ptr, '\n'))
{
return;
}
if (Name == TEXT("NULL") && UB == TEXT("NULL") && Type == 0 && ResourceIndex == 0)
{
break;
}
// Advance the uniform buffer map if this is a different UB name
while (UniformBufferMapIt.Key() != UB)
{
++UniformBufferMapIt;
if (UniformBufferMapIt == OutEnvironment.UniformBufferMap.end())
{
return;
}
}
// Append the Name we parsed to the member name buffer
TArray<TCHAR>& Buffer = *UniformBufferMapIt.Value().MemberNameBuffer.Get();
uint32 MemberNameLength = Name.Len();
Buffer.Append(*Name, MemberNameLength + 1);
// The member name field of the entries is initialized at the end of parsing by the FixupOnLoad call from FFixupOnExit, so we can set it to nullptr here
OutEnvironment.ResourceTableMap.Resources.Add({
nullptr,
(uint8)UB.Len(),
(uint8)Type,
(uint16)ResourceIndex
});
}
}
/**
* Parse an error emitted by the HLSL cross-compiler.
* @param OutErrors - Array into which compiler errors may be added.
* @param InLine - A line from the compile log.
*/
void ParseHlslccError(TArray<FShaderCompilerError>& OutErrors, const FString& InLine, bool bUseAbsolutePaths)
{
const TCHAR* p = *InLine;
FShaderCompilerError& Error = OutErrors.AddDefaulted_GetRef();
// Copy the filename.
while (*p && *p != TEXT('('))
{
Error.ErrorVirtualFilePath += (*p++);
}
if (!bUseAbsolutePaths)
{
Error.ErrorVirtualFilePath = ParseVirtualShaderFilename(Error.ErrorVirtualFilePath);
}
p++;
// Parse the line number.
int32 LineNumber = 0;
while (*p && *p >= TEXT('0') && *p <= TEXT('9'))
{
LineNumber = 10 * LineNumber + (*p++ - TEXT('0'));
}
Error.ErrorLineString = *FString::Printf(TEXT("%d"), LineNumber);
// Skip to the warning message.
while (*p && (*p == TEXT(')') || *p == TEXT(':') || *p == TEXT(' ') || *p == TEXT('\t')))
{
p++;
}
Error.StrippedErrorMessage = p;
}
/** Map shader frequency -> string for messages. */
static const TCHAR* FrequencyStringTable[] =
{
TEXT("Vertex"),
TEXT("Mesh"),
TEXT("Amplification"),
TEXT("Pixel"),
TEXT("Geometry"),
TEXT("Compute"),
TEXT("RayGen"),
TEXT("RayMiss"),
TEXT("RayHitGroup"),
TEXT("RayCallable"),
};
/** Compile time check to verify that the GL mapping tables are up-to-date. */
static_assert(SF_NumFrequencies == UE_ARRAY_COUNT(FrequencyStringTable), "NumFrequencies changed. Please update tables.");
const TCHAR* GetFrequencyName(EShaderFrequency Frequency)
{
check((int32)Frequency >= 0 && Frequency < SF_NumFrequencies);
return FrequencyStringTable[Frequency];
}
FHlslccHeader::FHlslccHeader() :
Name(TEXT(""))
{
NumThreads[0] = NumThreads[1] = NumThreads[2] = 0;
}
bool FHlslccHeader::Read(const ANSICHAR*& ShaderSource, int32 SourceLen)
{
#define DEF_PREFIX_STR(Str) \
static const ANSICHAR* Str##Prefix = "// @" #Str ": "; \
static const int32 Str##PrefixLen = FCStringAnsi::Strlen(Str##Prefix)
DEF_PREFIX_STR(Inputs);
DEF_PREFIX_STR(Outputs);
DEF_PREFIX_STR(UniformBlocks);
DEF_PREFIX_STR(Uniforms);
DEF_PREFIX_STR(PackedGlobals);
DEF_PREFIX_STR(PackedUB);
DEF_PREFIX_STR(PackedUBCopies);
DEF_PREFIX_STR(PackedUBGlobalCopies);
DEF_PREFIX_STR(Samplers);
DEF_PREFIX_STR(UAVs);
DEF_PREFIX_STR(SamplerStates);
DEF_PREFIX_STR(AccelerationStructures);
DEF_PREFIX_STR(NumThreads);
#undef DEF_PREFIX_STR
// Skip any comments that come before the signature.
while (FCStringAnsi::Strncmp(ShaderSource, "//", 2) == 0 &&
FCStringAnsi::Strncmp(ShaderSource + 2, " !", 2) != 0 &&
FCStringAnsi::Strncmp(ShaderSource + 2, " @", 2) != 0)
{
ShaderSource += 2;
while (*ShaderSource && *ShaderSource++ != '\n')
{
// Do nothing
}
}
// Read shader name if any
if (FCStringAnsi::Strncmp(ShaderSource, "// !", 4) == 0)
{
ShaderSource += 4;
while (*ShaderSource && *ShaderSource != '\n')
{
Name += (TCHAR)*ShaderSource;
++ShaderSource;
}
if (*ShaderSource == '\n')
{
++ShaderSource;
}
}
// Skip any comments that come before the signature.
while (FCStringAnsi::Strncmp(ShaderSource, "//", 2) == 0 &&
FCStringAnsi::Strncmp(ShaderSource + 2, " @", 2) != 0)
{
ShaderSource += 2;
while (*ShaderSource && *ShaderSource++ != '\n')
{
// Do nothing
}
}
if (FCStringAnsi::Strncmp(ShaderSource, InputsPrefix, InputsPrefixLen) == 0)
{
ShaderSource += InputsPrefixLen;
if (!ReadInOut(ShaderSource, Inputs))
{
return false;
}
}
if (FCStringAnsi::Strncmp(ShaderSource, OutputsPrefix, OutputsPrefixLen) == 0)
{
ShaderSource += OutputsPrefixLen;
if (!ReadInOut(ShaderSource, Outputs))
{
return false;
}
}
if (FCStringAnsi::Strncmp(ShaderSource, UniformBlocksPrefix, UniformBlocksPrefixLen) == 0)
{
ShaderSource += UniformBlocksPrefixLen;
while (*ShaderSource && *ShaderSource != '\n')
{
FAttribute UniformBlock;
if (!ParseIdentifier(ShaderSource, UniformBlock.Name))
{
return false;
}
if (!Match(ShaderSource, '('))
{
return false;
}
if (!ParseIntegerNumber(ShaderSource, UniformBlock.Index))
{
return false;
}
if (!Match(ShaderSource, ')'))
{
return false;
}
UniformBlocks.Add(UniformBlock);
if (Match(ShaderSource, '\n'))
{
break;
}
if (Match(ShaderSource, ','))
{
continue;
}
//#todo-rco: Need a log here
//UE_LOG(ShaderCompilerCommon, Warning, TEXT("Invalid char '%c'"), *ShaderSource);
return false;
}
}
if (FCStringAnsi::Strncmp(ShaderSource, UniformsPrefix, UniformsPrefixLen) == 0)
{
// @todo-mobile: Will we ever need to support this code path?
check(0);
return false;
/*
ShaderSource += UniformsPrefixLen;
while (*ShaderSource && *ShaderSource != '\n')
{
uint16 ArrayIndex = 0;
uint16 Offset = 0;
uint16 NumComponents = 0;
FString ParameterName = ParseIdentifier(ShaderSource);
verify(ParameterName.Len() > 0);
verify(Match(ShaderSource, '('));
ArrayIndex = ParseNumber(ShaderSource);
verify(Match(ShaderSource, ':'));
Offset = ParseNumber(ShaderSource);
verify(Match(ShaderSource, ':'));
NumComponents = ParseNumber(ShaderSource);
verify(Match(ShaderSource, ')'));
ParameterMap.AddParameterAllocation(
*ParameterName,
ArrayIndex,
Offset * BytesPerComponent,
NumComponents * BytesPerComponent
);
if (ArrayIndex < OGL_NUM_PACKED_UNIFORM_ARRAYS)
{
PackedUniformSize[ArrayIndex] = FMath::Max<uint16>(
PackedUniformSize[ArrayIndex],
BytesPerComponent * (Offset + NumComponents)
);
}
// Skip the comma.
if (Match(ShaderSource, '\n'))
{
break;
}
verify(Match(ShaderSource, ','));
}
Match(ShaderSource, '\n');
*/
}
// @PackedGlobals: Global0(h:0,1),Global1(h:4,1),Global2(h:8,1)
if (FCStringAnsi::Strncmp(ShaderSource, PackedGlobalsPrefix, PackedGlobalsPrefixLen) == 0)
{
ShaderSource += PackedGlobalsPrefixLen;
while (*ShaderSource && *ShaderSource != '\n')
{
FPackedGlobal PackedGlobal;
if (!ParseIdentifier(ShaderSource, PackedGlobal.Name))
{
return false;
}
if (!Match(ShaderSource, '('))
{
return false;
}
PackedGlobal.PackedType = *ShaderSource++;
if (!Match(ShaderSource, ':'))
{
return false;
}
if (!ParseIntegerNumber(ShaderSource, PackedGlobal.Offset))
{
return false;
}
if (!Match(ShaderSource, ','))
{
return false;
}
if (!ParseIntegerNumber(ShaderSource, PackedGlobal.Count))
{
return false;
}
if (!Match(ShaderSource, ')'))
{
return false;
}
PackedGlobals.Add(PackedGlobal);
// Break if EOL
if (Match(ShaderSource, '\n'))
{
break;
}
// Has to be a comma!
if (Match(ShaderSource, ','))
{
continue;
}
//#todo-rco: Need a log here
//UE_LOG(ShaderCompilerCommon, Warning, TEXT("Invalid char '%c'"), *ShaderSource);
return false;
}
}
// Packed Uniform Buffers (Multiple lines)
// @PackedUB: CBuffer(0): CBMember0(0,1),CBMember1(1,1)
while (FCStringAnsi::Strncmp(ShaderSource, PackedUBPrefix, PackedUBPrefixLen) == 0)
{
ShaderSource += PackedUBPrefixLen;
FPackedUB PackedUB;
if (!ParseIdentifier(ShaderSource, PackedUB.Attribute.Name))
{
return false;
}
if (!Match(ShaderSource, '('))
{
return false;
}
if (!ParseIntegerNumber(ShaderSource, PackedUB.Attribute.Index))
{
return false;
}
if (!Match(ShaderSource, ')'))
{
return false;
}
if (!Match(ShaderSource, ':'))
{
return false;
}
if (!Match(ShaderSource, ' '))
{
return false;
}
while (*ShaderSource && *ShaderSource != '\n')
{
FPackedUB::FMember Member;
ParseIdentifier(ShaderSource, Member.Name);
if (!Match(ShaderSource, '('))
{
return false;
}
if (!ParseIntegerNumber(ShaderSource, Member.Offset))
{
return false;
}
if (!Match(ShaderSource, ','))
{
return false;
}
if (!ParseIntegerNumber(ShaderSource, Member.Count))
{
return false;
}
if (!Match(ShaderSource, ')'))
{
return false;
}
PackedUB.Members.Add(Member);
// Break if EOL
if (Match(ShaderSource, '\n'))
{
break;
}
// Has to be a comma!
if (Match(ShaderSource, ','))
{
continue;
}
//#todo-rco: Need a log here
//UE_LOG(ShaderCompilerCommon, Warning, TEXT("Invalid char '%c'"), *ShaderSource);
return false;
}
PackedUBs.Add(PackedUB);
}
// @PackedUBCopies: 0:0-0:h:0:1,0:1-0:h:4:1,1:0-1:h:0:1
if (FCStringAnsi::Strncmp(ShaderSource, PackedUBCopiesPrefix, PackedUBCopiesPrefixLen) == 0)
{
ShaderSource += PackedUBCopiesPrefixLen;
if (!ReadCopies(ShaderSource, false, PackedUBCopies))
{
return false;
}
}
// @PackedUBGlobalCopies: 0:0-h:12:1,0:1-h:16:1,1:0-h:20:1
if (FCStringAnsi::Strncmp(ShaderSource, PackedUBGlobalCopiesPrefix, PackedUBGlobalCopiesPrefixLen) == 0)
{
ShaderSource += PackedUBGlobalCopiesPrefixLen;
if (!ReadCopies(ShaderSource, true, PackedUBGlobalCopies))
{
return false;
}
}
if (FCStringAnsi::Strncmp(ShaderSource, SamplersPrefix, SamplersPrefixLen) == 0)
{
ShaderSource += SamplersPrefixLen;
while (*ShaderSource && *ShaderSource != '\n')
{
FSampler Sampler;
if (!ParseIdentifier(ShaderSource, Sampler.Name))
{
return false;
}
if (!Match(ShaderSource, '('))
{
return false;
}
if (!ParseIntegerNumber(ShaderSource, Sampler.Offset))
{
return false;
}
if (!Match(ShaderSource, ':'))
{
return false;
}
if (!ParseIntegerNumber(ShaderSource, Sampler.Count))
{
return false;
}
if (Match(ShaderSource, '['))
{
// Sampler States
do
{
FString SamplerState;
if (!ParseIdentifier(ShaderSource, SamplerState))
{
return false;
}
Sampler.SamplerStates.Add(SamplerState);
}
while (Match(ShaderSource, ','));
if (!Match(ShaderSource, ']'))
{
return false;
}
}
if (!Match(ShaderSource, ')'))
{
return false;
}
Samplers.Add(Sampler);
// Break if EOL
if (Match(ShaderSource, '\n'))
{
break;
}
// Has to be a comma!
if (Match(ShaderSource, ','))
{
continue;
}
//#todo-rco: Need a log here
//UE_LOG(ShaderCompilerCommon, Warning, TEXT("Invalid char '%c'"), *ShaderSource);
return false;
}
}
if (FCStringAnsi::Strncmp(ShaderSource, UAVsPrefix, UAVsPrefixLen) == 0)
{
ShaderSource += UAVsPrefixLen;
while (*ShaderSource && *ShaderSource != '\n')
{
FUAV UAV;
if (!ParseIdentifier(ShaderSource, UAV.Name))
{
return false;
}
if (!Match(ShaderSource, '('))
{
return false;
}
if (!ParseIntegerNumber(ShaderSource, UAV.Offset))
{
return false;
}
if (!Match(ShaderSource, ':'))
{
return false;
}
if (!ParseIntegerNumber(ShaderSource, UAV.Count))
{
return false;
}
if (!Match(ShaderSource, ')'))
{
return false;
}
UAVs.Add(UAV);
// Break if EOL
if (Match(ShaderSource, '\n'))
{
break;
}
// Has to be a comma!
if (Match(ShaderSource, ','))
{
continue;
}
//#todo-rco: Need a log here
//UE_LOG(ShaderCompilerCommon, Warning, TEXT("Invalid char '%c'"), *ShaderSource);
return false;
}
}
if (FCStringAnsi::Strncmp(ShaderSource, SamplerStatesPrefix, SamplerStatesPrefixLen) == 0)
{
ShaderSource += SamplerStatesPrefixLen;
while (*ShaderSource && *ShaderSource != '\n')
{
FAttribute SamplerState;
if (!ParseIntegerNumber(ShaderSource, SamplerState.Index))
{
return false;
}
if (!Match(ShaderSource, ':'))
{
return false;
}
if (!ParseIdentifier(ShaderSource, SamplerState.Name))
{
return false;
}
SamplerStates.Add(SamplerState);
// Break if EOL
if (Match(ShaderSource, '\n'))
{
break;
}
// Has to be a comma!
if (Match(ShaderSource, ','))
{
continue;
}
//#todo-rco: Need a log here
//UE_LOG(ShaderCompilerCommon, Warning, TEXT("Invalid char '%c'"), *ShaderSource);
return false;
}
}
if (FCStringAnsi::Strncmp(ShaderSource, AccelerationStructuresPrefix, AccelerationStructuresPrefixLen) == 0)
{
ShaderSource += AccelerationStructuresPrefixLen;
while (*ShaderSource && *ShaderSource != '\n')
{
FAccelerationStructure AccelerationStructure;
if (!ParseIntegerNumber(ShaderSource, AccelerationStructure.Offset))
{
return false;
}
if (!Match(ShaderSource, ':'))
{
return false;
}
if (!ParseIdentifier(ShaderSource, AccelerationStructure.Name))
{
return false;
}
AccelerationStructures.Add(AccelerationStructure);
if (Match(ShaderSource, '\n'))
{
break;
}
if (Match(ShaderSource, ','))
{
continue;
}
return false;
}
}
if (FCStringAnsi::Strncmp(ShaderSource, NumThreadsPrefix, NumThreadsPrefixLen) == 0)
{
ShaderSource += NumThreadsPrefixLen;
if (!ParseIntegerNumber(ShaderSource, NumThreads[0]))
{
return false;
}
if (!Match(ShaderSource, ','))
{
return false;
}
if (!Match(ShaderSource, ' '))
{
return false;
}
if (!ParseIntegerNumber(ShaderSource, NumThreads[1]))
{
return false;
}
if (!Match(ShaderSource, ','))
{
return false;
}
if (!Match(ShaderSource, ' '))
{
return false;
}
if (!ParseIntegerNumber(ShaderSource, NumThreads[2]))
{
return false;
}
if (!Match(ShaderSource, '\n'))
{
return false;
}
}
return ParseCustomHeaderEntries(ShaderSource);
}
bool FHlslccHeader::ReadCopies(const ANSICHAR*& ShaderSource, bool bGlobals, TArray<FPackedUBCopy>& OutCopies)
{
while (*ShaderSource && *ShaderSource != '\n')
{
FPackedUBCopy PackedUBCopy;
PackedUBCopy.DestUB = 0;
if (!ParseIntegerNumber(ShaderSource, PackedUBCopy.SourceUB))
{
return false;
}
if (!Match(ShaderSource, ':'))
{
return false;
}
if (!ParseIntegerNumber(ShaderSource, PackedUBCopy.SourceOffset))
{
return false;
}
if (!Match(ShaderSource, '-'))
{
return false;
}
if (!bGlobals)
{
if (!ParseIntegerNumber(ShaderSource, PackedUBCopy.DestUB))
{
return false;
}
if (!Match(ShaderSource, ':'))
{
return false;
}
}
PackedUBCopy.DestPackedType = *ShaderSource++;
if (!Match(ShaderSource, ':'))
{
return false;
}
if (!ParseIntegerNumber(ShaderSource, PackedUBCopy.DestOffset))
{
return false;
}
if (!Match(ShaderSource, ':'))
{
return false;
}
if (!ParseIntegerNumber(ShaderSource, PackedUBCopy.Count))
{
return false;
}
OutCopies.Add(PackedUBCopy);
// Break if EOL
if (Match(ShaderSource, '\n'))
{
break;
}
// Has to be a comma!
if (Match(ShaderSource, ','))
{
continue;
}
//#todo-rco: Need a log here
//UE_LOG(ShaderCompilerCommon, Warning, TEXT("Invalid char '%c'"), *ShaderSource);
return false;
}
return true;
}
bool FHlslccHeader::ReadInOut(const ANSICHAR*& ShaderSource, TArray<FInOut>& OutAttributes)
{
while (*ShaderSource && *ShaderSource != '\n')
{
FInOut Attribute;
if (!ParseIdentifier(ShaderSource, Attribute.Type))
{
return false;
}
if (Match(ShaderSource, '['))
{
if (!ParseIntegerNumber(ShaderSource, Attribute.ArrayCount))
{
return false;
}
if (!Match(ShaderSource, ']'))
{
return false;
}
}
else
{
Attribute.ArrayCount = 0;
}
if (Match(ShaderSource, ';'))
{
if (!ParseSignedNumber(ShaderSource, Attribute.Index))
{
return false;
}
}
if (!Match(ShaderSource, ':'))
{
return false;
}
if (!ParseIdentifier(ShaderSource, Attribute.Name))
{
return false;
}
// Optional array suffix
if (Match(ShaderSource, '['))
{
Attribute.Name += '[';
while (*ShaderSource)
{
Attribute.Name += *ShaderSource;
if (Match(ShaderSource, ']'))
{
break;
}
++ShaderSource;
}
}
OutAttributes.Add(Attribute);
// Break if EOL
if (Match(ShaderSource, '\n'))
{
return true;
}
// Has to be a comma!
if (Match(ShaderSource, ','))
{
continue;
}
//#todo-rco: Need a log here
//UE_LOG(ShaderCompilerCommon, Warning, TEXT("Invalid char '%c'"), *ShaderSource);
return false;
}
// Last character must be EOL
return Match(ShaderSource, '\n');
}
} // namespace CrossCompiler
Reference in New Issue View Git Blame Copy Permalink