vkd3d-shader/d3dbc: Normalise I/O register write masks when not disassembling.

Sometimes SM1-3 shaders contain write masks that exceed the
signature element masks. That happens because SM1-3 shaders do not
have a concept of signature and signature masks, and OTOH aren't
always able to express any given write mask.

In VSIR we don't want to deal with I/O register masks exceeding the
corresponding signature element mask or usage mask, because, for
instance, for higher shader models it can complicate dealing with
DCL_INDEX_RANGE. In order to have uniform rules for all shader
models we normalise masks coming from SM1-3 shaders.

We don't do that normalisation when disassembling, in order to
preserve the expected output.

libs/vkd3d-shader/d3dbc.c

@@ -633,7 +633,32 @@ static void add_signature_mask(struct vkd3d_shader_sm1_parser *sm1, bool output,
         return;
     }
 
+    /* Normally VSIR mandates that the register mask is a subset of the usage
+     * mask, and the usage mask is a subset of the signature mask. This is
+     * doesn't always happen with SM1-3 registers, because of the limited
+     * flexibility with expressing swizzles.
+     *
+     * For example it's easy to find shaders like this:
+     *   ps_3_0
+     *   [...]
+     *   dcl_texcoord0 v0
+     *   [...]
+     *   texld r2.xyzw, v0.xyzw, s1.xyzw
+     *   [...]
+     *
+     * The dcl_textcoord0 instruction secretly has a .xy mask, which is used to
+     * compute the signature mask, but the texld instruction apparently uses all
+     * the components. Of course the last two components are ignored, but
+     * formally they seem to be used. So we end up with a signature element with
+     * mask .xy and usage mask .xyzw.
+     *
+     * In order to avoid this problem, when generating VSIR code with SM4
+     * normalisation level we remove the unused components in the write mask. We
+     * don't do that when targetting the SM1 normalisation level (i.e., when
+     * disassembling) so as to generate the same disassembly code as native. */
     element->used_mask |= mask;
+    if (program->normalisation_level >= VSIR_NORMALISED_SM4)
+        element->used_mask &= element->mask;
 }
 
 static bool add_signature_element_from_register(struct vkd3d_shader_sm1_parser *sm1,
@@ -1265,6 +1290,7 @@ static enum vkd3d_result shader_sm1_init(struct vkd3d_shader_sm1_parser *sm1, st
         const struct vkd3d_shader_compile_info *compile_info, struct vkd3d_shader_message_context *message_context)
 {
     const struct vkd3d_shader_location location = {.source_name = compile_info->source_name};
+    enum vsir_normalisation_level normalisation_level;
     const uint32_t *code = compile_info->source.code;
     size_t code_size = compile_info->source.size;
     struct vkd3d_shader_version version;
@@ -1315,9 +1341,13 @@ static enum vkd3d_result shader_sm1_init(struct vkd3d_shader_sm1_parser *sm1, st
     sm1->start = &code[1];
     sm1->end = &code[token_count];
 
+    normalisation_level = VSIR_NORMALISED_SM1;
+    if (compile_info->target_type != VKD3D_SHADER_TARGET_D3D_ASM)
+        normalisation_level = VSIR_NORMALISED_SM4;
+
     /* Estimate instruction count to avoid reallocation in most shaders. */
     if (!vsir_program_init(program, compile_info, &version,
-            code_size != ~(size_t)0 ? token_count / 4u + 4 : 16, VSIR_CF_STRUCTURED, VSIR_NORMALISED_SM4))
+            code_size != ~(size_t)0 ? token_count / 4u + 4 : 16, VSIR_CF_STRUCTURED, normalisation_level))
         return VKD3D_ERROR_OUT_OF_MEMORY;
 
     vkd3d_shader_parser_init(&sm1->p, program, message_context, compile_info->source_name);

libs/vkd3d-shader/ir.c

@@ -8098,29 +8098,20 @@ static void vsir_validate_signature_element(struct validation_context *ctx,
                 "element %u of %s signature: Non-contiguous mask %#x.",
                 idx, signature_type_name, element->mask);
 
-    /* Here we'd likely want to validate that the usage mask is a subset of the
+    if (ctx->program->normalisation_level >= VSIR_NORMALISED_SM4)
-     * signature mask. Unfortunately the D3DBC parser sometimes violates this.
+    {
-     * For example I've seen a shader like this:
+        if ((element->used_mask & element->mask) != element->used_mask)
-     *   ps_3_0
+            validator_error(ctx, VKD3D_SHADER_ERROR_VSIR_INVALID_SIGNATURE,
-     *   [...]
+                    "element %u of %s signature: Invalid usage mask %#x with mask %#x.",
-     *   dcl_texcoord0 v0
+                    idx, signature_type_name, element->used_mask, element->mask);
-     *   [...]
+    }
-     *   texld r2.xyzw, v0.xyzw, s1.xyzw
+    else
-     *   [...]
+    {
-     *
+        if (element->used_mask & ~0xf)
-     * The dcl_textcoord0 instruction secretly has a .xy mask, which is used to
+            validator_error(ctx, VKD3D_SHADER_ERROR_VSIR_INVALID_SIGNATURE,
-     * compute the signature mask, but the texld instruction apparently uses all
+                    "element %u of %s signature: Invalid usage mask %#x.",
-     * the components. Of course the last two components are ignored, but
+                    idx, signature_type_name, element->used_mask);
-     * formally they seem to be used. So we end up with a signature element with
+    }
-     * mask .xy and usage mask .xyzw.
-     *
-     * The correct fix would probably be to make the D3DBC parser aware of which
-     * components are really used for each instruction, but that would take some
-     * time. */
-    if (element->used_mask & ~0xf)
-        validator_error(ctx, VKD3D_SHADER_ERROR_VSIR_INVALID_SIGNATURE,
-                "element %u of %s signature: Invalid usage mask %#x.",
-                idx, signature_type_name, element->used_mask);
 
     switch (element->sysval_semantic)
     {

libs/vkd3d-shader/vkd3d_shader_private.h

@@ -1411,6 +1411,7 @@ enum vsir_control_flow_type
 
 enum vsir_normalisation_level
 {
+    VSIR_NORMALISED_SM1,
     VSIR_NORMALISED_SM4,
     VSIR_NORMALISED_HULL_CONTROL_POINT_IO,
     VSIR_NORMALISED_SM6,