Merge pull request #13975 from iwubcode/shader_includes

VideoBackends / VideoCommon: add support for specifying include files in shader code

Source/Core/DolphinLib.props

@@ -749,6 +749,7 @@
     <ClInclude Include="VideoCommon\Present.h" />
     <ClInclude Include="VideoCommon\RenderState.h" />
     <ClInclude Include="VideoCommon\ShaderCache.h" />
+    <ClInclude Include="VideoCommon\ShaderCompileUtils.h" />
     <ClInclude Include="VideoCommon\ShaderGenCommon.h" />
     <ClInclude Include="VideoCommon\Spirv.h" />
     <ClInclude Include="VideoCommon\Statistics.h" />
@@ -1396,6 +1397,7 @@
     <ClCompile Include="VideoCommon\Present.cpp" />
     <ClCompile Include="VideoCommon\RenderState.cpp" />
     <ClCompile Include="VideoCommon\ShaderCache.cpp" />
+    <ClCompile Include="VideoCommon\ShaderCompileUtils.cpp" />
     <ClCompile Include="VideoCommon\ShaderGenCommon.cpp" />
     <ClCompile Include="VideoCommon\Spirv.cpp" />
     <ClCompile Include="VideoCommon\Statistics.cpp" />

Source/Core/VideoBackends/D3D/D3DGfx.cpp

@@ -71,9 +71,10 @@ Gfx::CreateFramebuffer(AbstractTexture* color_attachment, AbstractTexture* depth
 }
 
 std::unique_ptr<AbstractShader>
-Gfx::CreateShaderFromSource(ShaderStage stage, std::string_view source, std::string_view name)
+Gfx::CreateShaderFromSource(ShaderStage stage, std::string_view source,
+                            VideoCommon::ShaderIncluder* shader_includer, std::string_view name)
 {
-  auto bytecode = DXShader::CompileShader(D3D::feature_level, stage, source);
+  auto bytecode = DXShader::CompileShader(D3D::feature_level, stage, source, shader_includer);
   if (!bytecode)
     return nullptr;
 

Source/Core/VideoBackends/D3D/D3DGfx.h

@@ -31,8 +31,10 @@ public:
                                                  std::string_view name) override;
   std::unique_ptr<AbstractStagingTexture>
   CreateStagingTexture(StagingTextureType type, const TextureConfig& config) override;
-  std::unique_ptr<AbstractShader> CreateShaderFromSource(ShaderStage stage, std::string_view source,
-                                                         std::string_view name) override;
+  std::unique_ptr<AbstractShader>
+  CreateShaderFromSource(ShaderStage stage, std::string_view source,
+                         VideoCommon::ShaderIncluder* shader_includer,
+                         std::string_view name) override;
   std::unique_ptr<AbstractShader> CreateShaderFromBinary(ShaderStage stage, const void* data,
                                                          size_t length,
                                                          std::string_view name) override;

Source/Core/VideoBackends/D3D12/D3D12Gfx.cpp

@@ -68,9 +68,10 @@ Gfx::CreateFramebuffer(AbstractTexture* color_attachment, AbstractTexture* depth
 }
 
 std::unique_ptr<AbstractShader>
-Gfx::CreateShaderFromSource(ShaderStage stage, std::string_view source, std::string_view name)
+Gfx::CreateShaderFromSource(ShaderStage stage, std::string_view source,
+                            VideoCommon::ShaderIncluder* shader_includer, std::string_view name)
 {
-  return DXShader::CreateFromSource(stage, source, name);
+  return DXShader::CreateFromSource(stage, source, shader_includer, name);
 }
 
 std::unique_ptr<AbstractShader> Gfx::CreateShaderFromBinary(ShaderStage stage, const void* data,

Source/Core/VideoBackends/D3D12/D3D12Gfx.h

@@ -36,8 +36,10 @@ public:
   CreateFramebuffer(AbstractTexture* color_attachment, AbstractTexture* depth_attachment,
                     std::vector<AbstractTexture*> additional_color_attachments) override;
 
-  std::unique_ptr<AbstractShader> CreateShaderFromSource(ShaderStage stage, std::string_view source,
-                                                         std::string_view name) override;
+  std::unique_ptr<AbstractShader>
+  CreateShaderFromSource(ShaderStage stage, std::string_view source,
+                         VideoCommon::ShaderIncluder* shader_includer,
+                         std::string_view name) override;
   std::unique_ptr<AbstractShader> CreateShaderFromBinary(ShaderStage stage, const void* data,
                                                          size_t length,
                                                          std::string_view name) override;

Source/Core/VideoBackends/D3D12/DX12Shader.cpp

@@ -29,9 +29,10 @@ std::unique_ptr<DXShader> DXShader::CreateFromBytecode(ShaderStage stage, Binary
 }
 
 std::unique_ptr<DXShader> DXShader::CreateFromSource(ShaderStage stage, std::string_view source,
+                                                     VideoCommon::ShaderIncluder* shader_includer,
                                                      std::string_view name)
 {
-  auto bytecode = CompileShader(g_dx_context->GetFeatureLevel(), stage, source);
+  auto bytecode = CompileShader(g_dx_context->GetFeatureLevel(), stage, source, shader_includer);
   if (!bytecode)
     return nullptr;
 

Source/Core/VideoBackends/D3D12/DX12Shader.h

@@ -23,6 +23,7 @@ public:
   static std::unique_ptr<DXShader> CreateFromBytecode(ShaderStage stage, BinaryData bytecode,
                                                       std::string_view name);
   static std::unique_ptr<DXShader> CreateFromSource(ShaderStage stage, std::string_view source,
+                                                    VideoCommon::ShaderIncluder* shader_includer,
                                                     std::string_view name);
 
 private:

Source/Core/VideoBackends/D3DCommon/Shader.cpp

@@ -20,6 +20,7 @@
 #include "Common/StringUtil.h"
 #include "Common/Version.h"
 
+#include "VideoCommon/ShaderCompileUtils.h"
 #include "VideoCommon/Spirv.h"
 #include "VideoCommon/VideoBackendBase.h"
 #include "VideoCommon/VideoConfig.h"
@@ -35,6 +36,9 @@ namespace
 constexpr std::string_view SHADER_HEADER = R"(
   // Target GLSL 4.5.
   #version 450 core
+
+  #extension GL_ARB_shading_language_include : enable
+
   #define ATTRIBUTE_LOCATION(x) layout(location = x)
   #define FRAGMENT_OUTPUT_LOCATION(x) layout(location = x)
   #define FRAGMENT_OUTPUT_LOCATION_INDEXED(x, y) layout(location = x, index = y)
@@ -107,14 +111,16 @@ std::optional<std::string> GetHLSLFromSPIRV(SPIRV::CodeVector spv, D3D_FEATURE_L
   return compiler.compile();
 }
 
-std::optional<SPIRV::CodeVector> GetSpirv(ShaderStage stage, std::string_view source)
+std::optional<SPIRV::CodeVector> GetSpirv(ShaderStage stage, std::string_view source,
+                                          glslang::TShader::Includer* shader_includer)
 {
   switch (stage)
   {
   case ShaderStage::Vertex:
   {
     const auto full_source = fmt::format("{}{}", SHADER_HEADER, source);
-    return SPIRV::CompileVertexShader(full_source, APIType::D3D, glslang::EShTargetSpv_1_0);
+    return SPIRV::CompileVertexShader(full_source, APIType::D3D, glslang::EShTargetSpv_1_0,
+                                      shader_includer);
   }
 
   case ShaderStage::Geometry:
@@ -126,13 +132,15 @@ std::optional<SPIRV::CodeVector> GetSpirv(ShaderStage stage, std::string_view so
   case ShaderStage::Pixel:
   {
     const auto full_source = fmt::format("{}{}", SHADER_HEADER, source);
-    return SPIRV::CompileFragmentShader(full_source, APIType::D3D, glslang::EShTargetSpv_1_0);
+    return SPIRV::CompileFragmentShader(full_source, APIType::D3D, glslang::EShTargetSpv_1_0,
+                                        shader_includer);
   }
 
   case ShaderStage::Compute:
   {
     const auto full_source = fmt::format("{}{}", COMPUTE_SHADER_HEADER, source);
-    return SPIRV::CompileComputeShader(full_source, APIType::D3D, glslang::EShTargetSpv_1_0);
+    return SPIRV::CompileComputeShader(full_source, APIType::D3D, glslang::EShTargetSpv_1_0,
+                                       shader_includer);
   }
   };
 
@@ -140,13 +148,14 @@ std::optional<SPIRV::CodeVector> GetSpirv(ShaderStage stage, std::string_view so
 }
 
 std::optional<std::string> GetHLSL(D3D_FEATURE_LEVEL feature_level, ShaderStage stage,
-                                   std::string_view source)
+                                   std::string_view source,
+                                   VideoCommon::ShaderIncluder* shader_includer)
 {
   if (stage == ShaderStage::Geometry)
   {
     return std::string{source};
   }
-  else if (const auto spirv = GetSpirv(stage, source))
+  else if (const auto spirv = GetSpirv(stage, source, shader_includer))
   {
     return GetHLSLFromSPIRV(std::move(*spirv), feature_level);
   }
@@ -230,10 +239,11 @@ static const char* GetCompileTarget(D3D_FEATURE_LEVEL feature_level, ShaderStage
   }
 }
 
-std::optional<Shader::BinaryData> Shader::CompileShader(D3D_FEATURE_LEVEL feature_level,
-                                                        ShaderStage stage, std::string_view source)
+std::optional<Shader::BinaryData>
+Shader::CompileShader(D3D_FEATURE_LEVEL feature_level, ShaderStage stage, std::string_view source,
+                      VideoCommon::ShaderIncluder* shader_includer)
 {
-  const auto hlsl = GetHLSL(feature_level, stage, source);
+  const auto hlsl = GetHLSL(feature_level, stage, source, shader_includer);
   if (!hlsl)
     return std::nullopt;
 
@@ -260,7 +270,7 @@ std::optional<Shader::BinaryData> Shader::CompileShader(D3D_FEATURE_LEVEL featur
     file << "Dolphin Version: " + Common::GetScmRevStr() + "\n";
     file << "Video Backend: " + g_video_backend->GetDisplayName();
 
-    if (const auto spirv = GetSpirv(stage, source))
+    if (const auto spirv = GetSpirv(stage, source, shader_includer))
     {
       file << "\nOriginal Source: \n";
       file << source << std::endl;

Source/Core/VideoBackends/D3DCommon/Shader.h

@@ -20,7 +20,8 @@ public:
   BinaryData GetBinary() const override;
 
   static std::optional<BinaryData> CompileShader(D3D_FEATURE_LEVEL feature_level, ShaderStage stage,
-                                                 std::string_view source);
+                                                 std::string_view source,
+                                                 VideoCommon::ShaderIncluder* shader_includer);
 
   static BinaryData CreateByteCode(const void* data, size_t length);
 

Source/Core/VideoBackends/Metal/MTLGfx.h

@@ -31,8 +31,10 @@ public:
   CreateFramebuffer(AbstractTexture* color_attachment, AbstractTexture* depth_attachment,
                     std::vector<AbstractTexture*> additional_color_attachments) override;
 
-  std::unique_ptr<AbstractShader> CreateShaderFromSource(ShaderStage stage, std::string_view source,
-                                                         std::string_view name) override;
+  std::unique_ptr<AbstractShader>
+  CreateShaderFromSource(ShaderStage stage, std::string_view source,
+                         VideoCommon::ShaderIncluder* shader_includer,
+                         std::string_view name) override;
   std::unique_ptr<AbstractShader> CreateShaderFromBinary(ShaderStage stage, const void* data,
                                                          size_t length,
                                                          std::string_view name) override;

Source/Core/VideoBackends/Metal/MTLGfx.mm

@@ -120,11 +120,12 @@ Metal::Gfx::CreateFramebuffer(AbstractTexture* color_attachment, AbstractTexture
 
 // MARK: Pipeline Creation
 
-std::unique_ptr<AbstractShader> Metal::Gfx::CreateShaderFromSource(ShaderStage stage,
-                                                                   std::string_view source,
-                                                                   std::string_view name)
+std::unique_ptr<AbstractShader>
+Metal::Gfx::CreateShaderFromSource(ShaderStage stage, std::string_view source,
+                                   VideoCommon::ShaderIncluder* shader_includer,
+                                   std::string_view name)
 {
-  std::optional<std::string> msl = Util::TranslateShaderToMSL(stage, source);
+  std::optional<std::string> msl = Util::TranslateShaderToMSL(stage, source, shader_includer);
   if (!msl.has_value())
   {
     PanicAlertFmt("Failed to convert shader {} to MSL", name);

Source/Core/VideoBackends/Metal/MTLUtil.h

@@ -54,7 +54,8 @@ static inline bool HasStencil(AbstractTextureFormat format)
   return format == AbstractTextureFormat::D24_S8 || format == AbstractTextureFormat::D32F_S8;
 }
 
-std::optional<std::string> TranslateShaderToMSL(ShaderStage stage, std::string_view source);
+std::optional<std::string> TranslateShaderToMSL(ShaderStage stage, std::string_view source,
+                                                VideoCommon::ShaderIncluder* shader_includer);
 
 }  // namespace Util
 }  // namespace Metal

Source/Core/VideoBackends/Metal/MTLUtil.mm

@@ -13,6 +13,7 @@
 
 #include "VideoCommon/Constants.h"
 #include "VideoCommon/DriverDetails.h"
+#include "VideoCommon/ShaderCompileUtils.h"
 #include "VideoCommon/Spirv.h"
 
 Metal::DeviceFeatures Metal::g_features;
@@ -490,8 +491,9 @@ MakeResourceBinding(spv::ExecutionModel stage, u32 set, u32 binding,  //
   return resource;
 }
 
-std::optional<std::string> Metal::Util::TranslateShaderToMSL(ShaderStage stage,
-                                                             std::string_view source)
+std::optional<std::string>
+Metal::Util::TranslateShaderToMSL(ShaderStage stage, std::string_view source,
+                                  VideoCommon::ShaderIncluder* shader_includer)
 {
   std::string full_source;
 
@@ -514,16 +516,19 @@ std::optional<std::string> Metal::Util::TranslateShaderToMSL(ShaderStage stage,
   switch (stage)
   {
   case ShaderStage::Vertex:
-    code = SPIRV::CompileVertexShader(full_source, APIType::Metal, glslang::EShTargetSpv_1_5);
+    code = SPIRV::CompileVertexShader(full_source, APIType::Metal, glslang::EShTargetSpv_1_5,
+                                      shader_includer);
     break;
   case ShaderStage::Geometry:
     PanicAlertFmt("Tried to compile geometry shader for Metal, but Metal doesn't support them!");
     break;
   case ShaderStage::Pixel:
-    code = SPIRV::CompileFragmentShader(full_source, APIType::Metal, glslang::EShTargetSpv_1_5);
+    code = SPIRV::CompileFragmentShader(full_source, APIType::Metal, glslang::EShTargetSpv_1_5,
+                                        shader_includer);
     break;
   case ShaderStage::Compute:
-    code = SPIRV::CompileComputeShader(full_source, APIType::Metal, glslang::EShTargetSpv_1_5);
+    code = SPIRV::CompileComputeShader(full_source, APIType::Metal, glslang::EShTargetSpv_1_5,
+                                       shader_includer);
     break;
   }
   if (!code.has_value())

Source/Core/VideoBackends/Null/NullGfx.cpp

@@ -53,6 +53,7 @@ public:
 
 std::unique_ptr<AbstractShader>
 NullGfx::CreateShaderFromSource(ShaderStage stage, [[maybe_unused]] std::string_view source,
+                                [[maybe_unused]] VideoCommon::ShaderIncluder* shader_includer,
                                 [[maybe_unused]] std::string_view name)
 {
   return std::make_unique<NullShader>(stage);

Source/Core/VideoBackends/Null/NullGfx.h

@@ -25,8 +25,10 @@ public:
   CreateFramebuffer(AbstractTexture* color_attachment, AbstractTexture* depth_attachment,
                     std::vector<AbstractTexture*> additional_color_attachments) override;
 
-  std::unique_ptr<AbstractShader> CreateShaderFromSource(ShaderStage stage, std::string_view source,
-                                                         std::string_view name) override;
+  std::unique_ptr<AbstractShader>
+  CreateShaderFromSource(ShaderStage stage, std::string_view source,
+                         VideoCommon::ShaderIncluder* shader_includer,
+                         std::string_view name) override;
   std::unique_ptr<AbstractShader> CreateShaderFromBinary(ShaderStage stage, const void* data,
                                                          size_t length,
                                                          std::string_view name) override;

Source/Core/VideoBackends/OGL/OGLGfx.cpp

@@ -231,9 +231,10 @@ OGLGfx::CreateFramebuffer(AbstractTexture* color_attachment, AbstractTexture* de
 }
 
 std::unique_ptr<AbstractShader>
-OGLGfx::CreateShaderFromSource(ShaderStage stage, std::string_view source, std::string_view name)
+OGLGfx::CreateShaderFromSource(ShaderStage stage, std::string_view source,
+                               VideoCommon::ShaderIncluder* shader_includer, std::string_view name)
 {
-  return OGLShader::CreateFromSource(stage, source, name);
+  return OGLShader::CreateFromSource(stage, source, shader_includer, name);
 }
 
 std::unique_ptr<AbstractShader>

Source/Core/VideoBackends/OGL/OGLGfx.h

@@ -25,8 +25,10 @@ public:
                                                  std::string_view name) override;
   std::unique_ptr<AbstractStagingTexture>
   CreateStagingTexture(StagingTextureType type, const TextureConfig& config) override;
-  std::unique_ptr<AbstractShader> CreateShaderFromSource(ShaderStage stage, std::string_view source,
-                                                         std::string_view name) override;
+  std::unique_ptr<AbstractShader>
+  CreateShaderFromSource(ShaderStage stage, std::string_view source,
+                         VideoCommon::ShaderIncluder* shader_includer,
+                         std::string_view name) override;
   std::unique_ptr<AbstractShader> CreateShaderFromBinary(ShaderStage stage, const void* data,
                                                          size_t length,
                                                          std::string_view name) override;

Source/Core/VideoBackends/OGL/OGLShader.cpp

@@ -3,12 +3,110 @@
 
 #include "VideoBackends/OGL/OGLShader.h"
 
+#include <algorithm>
+#include <cctype>
+
 #include "VideoBackends/OGL/ProgramShaderCache.h"
 
+#include "VideoCommon/ShaderCompileUtils.h"
+#include "VideoCommon/ShaderGenCommon.h"
 #include "VideoCommon/VideoConfig.h"
 
 namespace OGL
 {
+namespace
+{
+std::string ResolveIncludeStatements(glslang::TShader::Includer* shader_includer,
+                                     std::string_view source, const char* includer_name = "",
+                                     std::size_t depth = 1)
+{
+  if (!shader_includer)
+  {
+    return std::string{source};
+  }
+
+  std::string source_str(source);
+  std::istringstream iss(source_str);
+
+  ShaderCode output;
+  std::string line;
+  while (std::getline(iss, line))
+  {
+    if (line.empty())
+    {
+      output.Write("{}\n", line);
+      continue;
+    }
+
+    std::string_view include_preprocessor = "#include";
+    if (!line.starts_with(include_preprocessor))
+    {
+      output.Write("{}\n", line);
+      continue;
+    }
+
+    const std::string after_include = line.substr(include_preprocessor.size());
+
+    bool local_include = true;
+    std::string_view filename;
+    // First non-whitespace character after include
+    const std::string::const_iterator non_whitespace_iter = std::find_if_not(
+        after_include.begin(), after_include.end(), [](char ch) { return std::isspace(ch); });
+    if (*non_whitespace_iter == '<')
+    {
+      const auto after_less = std::next(non_whitespace_iter);
+      if (after_less == after_include.end())
+      {
+        // Found less-than at the end, malformed
+        output.Write("{}\n", line);
+        continue;
+      }
+      const auto end_iter = std::find(after_less, after_include.end(), '>');
+      if (end_iter == after_include.end())
+      {
+        // Include spans multiple lines or is malformed, just pass it along
+        output.Write("{}\n", line);
+        continue;
+      }
+      filename = std::string_view(after_less, end_iter);
+      local_include = false;
+    }
+    else if (*non_whitespace_iter == '"')
+    {
+      const auto after_quote = std::next(non_whitespace_iter);
+      if (after_quote == after_include.end())
+      {
+        // Found quote at the end, malformed
+        output.Write("{}\n", line);
+        continue;
+      }
+      const auto end_iter = std::find(after_quote, after_include.end(), '"');
+      if (end_iter == after_include.end())
+      {
+        // Include spans multiple lines or is malformed, just pass it along
+        output.Write("{}\n", line);
+        continue;
+      }
+      filename = std::string_view(after_quote, end_iter);
+    }
+
+    const std::string header_path = std::string{filename};
+    auto include_result =
+        local_include ? shader_includer->includeLocal(header_path.c_str(), includer_name, depth) :
+                        shader_includer->includeSystem(header_path.c_str(), includer_name, depth);
+    if (!include_result)
+    {
+      output.Write("{}\n", line);
+      continue;
+    }
+
+    output.Write("{}", ResolveIncludeStatements(shader_includer, include_result->headerData,
+                                                header_path.c_str(), ++depth));
+  }
+
+  return output.GetBuffer();
+}
+}  // namespace
 static GLenum GetGLShaderTypeForStage(ShaderStage stage)
 {
   switch (stage)
@@ -57,9 +155,12 @@ OGLShader::~OGLShader()
 }
 
 std::unique_ptr<OGLShader> OGLShader::CreateFromSource(ShaderStage stage, std::string_view source,
+                                                       VideoCommon::ShaderIncluder* shader_includer,
                                                        std::string_view name)
 {
-  std::string source_str(source);
+  // Note: while the source will all be available, any errors will not
+  // reference the correct paths
+  std::string source_str = ResolveIncludeStatements(shader_includer, source);
   std::string name_str(name);
   if (stage != ShaderStage::Compute)
   {

Source/Core/VideoBackends/OGL/OGLShader.h

@@ -29,6 +29,7 @@ public:
   const std::string& GetSource() const { return m_source; }
 
   static std::unique_ptr<OGLShader> CreateFromSource(ShaderStage stage, std::string_view source,
+                                                     VideoCommon::ShaderIncluder* shader_includer,
                                                      std::string_view name);
 
 private:

Source/Core/VideoBackends/Software/SWGfx.cpp

@@ -78,6 +78,7 @@ public:
 
 std::unique_ptr<AbstractShader>
 SWGfx::CreateShaderFromSource(ShaderStage stage, [[maybe_unused]] std::string_view source,
+                              [[maybe_unused]] VideoCommon::ShaderIncluder* shader_includer,
                               [[maybe_unused]] std::string_view name)
 {
   return std::make_unique<SWShader>(stage);