[Vulkan] Add shader variants for bound color targets

Fixes vulkan validation warnings about unused fragment shader outputs in
depth-only render passes, but is arguably less efficient in terms of
shader cache usage.

src/xenia/gpu/spirv_shader_translator.cc

@@ -2386,8 +2386,12 @@ void SpirvShaderTranslator::StartFragmentShaderBeforeMain() {
           "xe_out_fragment_data_2",
           "xe_out_fragment_data_3",
       };
+      // Only create outputs for color targets that are both written by the
+      // shader and actually bound in the render pass.
+      Modification shader_modification = GetSpirvShaderModification();
       uint32_t color_targets_remaining =
-          current_shader().writes_color_targets();
+          current_shader().writes_color_targets() &
+          shader_modification.pixel.color_targets_used;
       uint32_t color_target_index;
       while (
           xe::bit_scan_forward(color_targets_remaining, &color_target_index)) {

src/xenia/gpu/spirv_shader_translator.h

@@ -94,6 +94,9 @@ class SpirvShaderTranslator : public ShaderTranslator {
       // pre-multiply. Only RT0 is supported for now.
       xenos::BlendFactor rt0_blend_rgb_factor_for_premult : 5;
       xenos::BlendFactor rt0_blend_a_factor_for_premult : 5;
+      // For host render targets - which color render targets are actually
+      // bound.
+      uint32_t color_targets_used : xenos::kMaxColorRenderTargets;
     } pixel;
     uint64_t value = 0;
 

src/xenia/gpu/vulkan/vulkan_command_processor.cc

@@ -2885,6 +2885,7 @@ bool VulkanCommandProcessor::IssueDraw(xenos::PrimitiveType prim_type,
   SpirvShaderTranslator::Modification pixel_shader_modification;
   VulkanShader::VulkanTranslation* vertex_shader_translation;
   VulkanShader::VulkanTranslation* pixel_shader_translation;
+  uint32_t normalized_color_mask;
 
   // Two iterations because a submission (even the current one - in which case
   // it needs to be ended, and a new one must be started) may need to be awaited
@@ -2919,6 +2920,12 @@ bool VulkanCommandProcessor::IssueDraw(xenos::PrimitiveType prim_type,
       return false;
     }
 
+    // Compute which color render targets are used.
+    normalized_color_mask =
+        pixel_shader ? draw_util::GetNormalizedColorMask(
+                           regs, pixel_shader->writes_color_targets())
+                     : 0;
+
     // Shader modifications.
     vertex_shader_modification =
         pipeline_cache_->GetCurrentVertexShaderModification(
@@ -2926,7 +2933,8 @@ bool VulkanCommandProcessor::IssueDraw(xenos::PrimitiveType prim_type,
             interpolator_mask, ps_param_gen_pos != UINT32_MAX);
     pixel_shader_modification =
         pixel_shader ? pipeline_cache_->GetCurrentPixelShaderModification(
-                           *pixel_shader, interpolator_mask, ps_param_gen_pos)
+                           *pixel_shader, interpolator_mask, ps_param_gen_pos,
+                           normalized_color_mask)
                      : SpirvShaderTranslator::Modification(0);
 
     // Translate the shaders now to obtain the sampler bindings.
@@ -3017,10 +3025,6 @@ bool VulkanCommandProcessor::IssueDraw(xenos::PrimitiveType prim_type,
   // Set up the render targets - this may perform dispatches and draws.
   reg::RB_DEPTHCONTROL normalized_depth_control =
       draw_util::GetNormalizedDepthControl(regs);
-  uint32_t normalized_color_mask =
-      pixel_shader ? draw_util::GetNormalizedColorMask(
-                         regs, pixel_shader->writes_color_targets())
-                   : 0;
   if (!render_target_cache_->Update(is_rasterization_done,
                                     normalized_depth_control,
                                     normalized_color_mask, *vertex_shader)) {

src/xenia/gpu/vulkan/vulkan_pipeline_cache.cc

@@ -410,8 +410,8 @@ VulkanPipelineCache::GetCurrentVertexShaderModification(
 
 SpirvShaderTranslator::Modification
 VulkanPipelineCache::GetCurrentPixelShaderModification(
-    const Shader& shader, uint32_t interpolator_mask,
-    uint32_t param_gen_pos) const {
+    const Shader& shader, uint32_t interpolator_mask, uint32_t param_gen_pos,
+    uint32_t normalized_color_mask) const {
   assert_true(shader.type() == xenos::ShaderType::kPixel);
   assert_true(shader.is_ucode_analyzed());
   const auto& regs = register_file_;
@@ -485,6 +485,13 @@ VulkanPipelineCache::GetCurrentPixelShaderModification(
             xenos::BlendFactor::kSrcAlpha;
       }
     }
+
+    // Extract 1 bit per RT from the 4-bits-per-RT normalized_color_mask.
+    modification.pixel.color_targets_used =
+        (((normalized_color_mask >> 0) & 0xF) ? 1 : 0) |
+        (((normalized_color_mask >> 4) & 0xF) ? 2 : 0) |
+        (((normalized_color_mask >> 8) & 0xF) ? 4 : 0) |
+        (((normalized_color_mask >> 12) & 0xF) ? 8 : 0);
   }
 
   return modification;

src/xenia/gpu/vulkan/vulkan_pipeline_cache.h

@@ -112,8 +112,8 @@ class VulkanPipelineCache {
       Shader::HostVertexShaderType host_vertex_shader_type,
       uint32_t interpolator_mask, bool ps_param_gen_used) const;
   SpirvShaderTranslator::Modification GetCurrentPixelShaderModification(
-      const Shader& shader, uint32_t interpolator_mask,
-      uint32_t param_gen_pos) const;
+      const Shader& shader, uint32_t interpolator_mask, uint32_t param_gen_pos,
+      uint32_t normalized_color_mask) const;
 
   bool EnsureShadersTranslated(VulkanShader::VulkanTranslation* vertex_shader,
                                VulkanShader::VulkanTranslation* pixel_shader);