tests/shader_runner: Record HLSL todo/fail state for each shader model.

When a shader fails to compile for a range of versions, we want to validate that
we are correctly implementing that behaviour. E.g. if a feature requires shader
model 5.0, we should validate that it compiles correctly with 5.0 (which we do),
but also that it *fails* to compile with 4.1 (which we do not).

The obvious and simple solution is to simply run compile tests for each version.
There are, however, at least 12 versions of HLSL up to and including 6.0, at
least 10 of which are known to introduce new features. Shader compilation takes
about 10-15% of the time that draw and dispatch does, both for native and
(currently) vkd3d. Testing every version for every shader would add a
noticeable amount of time to the tests.

In practice, the interesting versions to test for most shaders are:

* At least one from each range 1-3, 4-5, and 6. It's common enough for the
  semantics of the HLSL to differ between bytecode formats, or for features to
  be added or removed across those boundaries.

* If the shader requires a given feature, we want to test both sides of the cusp
  to ensure we're requiring the same version for the feature.

In practice this is 3 or 4 versions, which is measurably less than the 12 we'd
otherwise be running.

In order to achieve this goal of testing only the 3 or 4 interesting versions
for a shader, we need to know what version is actually required for a feature.
This is encoded in the shader itself using e.g. [pixel shader fail(sm<5)].

This patch therefore implements the first step towards this goal, namely,
determining which versions succeed and fail, so we can figure out which ones are
interesting.

We could require the test writer to specify which versions are interesting ahead
of time (e.g. "for version in 2.0 4.1 5.0 6.0") but this is both redundant (and
there are a *lot* of tests that need some feature gate) and easy for a test
writer to get wrong by missing interesting versions.

tests/shader_runner.c

@@ -108,6 +108,27 @@ enum parse_state
     STATE_TEST,
 };
 
+static enum shader_model match_shader_model_string(const char *string, const char **rest)
+{
+    for (enum shader_model i = 0; i < ARRAY_SIZE(model_strings); ++i)
+    {
+        if (!strncmp(string, model_strings[i], strlen(model_strings[i])))
+        {
+            *rest = string + strlen(model_strings[i]);
+            return i;
+        }
+        /* Allow e.g. "4" as a shorthand for "4.0". */
+        if (string[0] == model_strings[i][0] && !strcmp(&model_strings[i][1], ".0")
+                && string[1] != '.' && !isdigit(string[1]))
+        {
+            *rest = string + 1;
+            return i;
+        }
+    }
+
+    fatal_error("Unrecognized shader model '%s'.\n", string);
+}
+
 static bool match_tag(struct shader_runner *runner, const char *tag)
 {
     for (size_t i = 0; i < runner->caps->tag_count; ++i)
@@ -119,54 +140,55 @@ static bool match_tag(struct shader_runner *runner, const char *tag)
     return false;
 }
 
-static bool check_qualifier_args_conjunction(struct shader_runner *runner, const char *line, const char **const rest)
+static bool check_qualifier_args_conjunction(struct shader_runner *runner,
+        const char *line, const char **const rest, uint32_t *model_mask)
 {
+    static const char *const valid_tags[] = {"d3d12", "glsl", "msl", "mvk", "vulkan"};
     bool holds = true;
-    unsigned int i;
 
-    static const struct
+    *model_mask = ~0u;
-    {
-        const char *text;
-        enum shader_model sm_min, sm_max;
-        bool tag;
-    }
-    valid_args[] =
-    {
-        {"sm>=4", SHADER_MODEL_4_0, SHADER_MODEL_6_0},
-        {"sm>=6", SHADER_MODEL_6_0, SHADER_MODEL_6_0},
-        {"sm<4",  SHADER_MODEL_2_0, SHADER_MODEL_4_0 - 1},
-        {"sm<6",  SHADER_MODEL_2_0, SHADER_MODEL_6_0 - 1},
-        {"d3d12", 0, 0, true},
-        {"glsl", 0, 0, true},
-        {"msl", 0, 0, true},
-        {"mvk", 0, 0, true},
-        {"vulkan", 0, 0, true},
-    };
 
     while (*line != ')' && *line != '|')
     {
+        enum shader_model model;
         bool match = false;
 
         while (isspace(*line))
             ++line;
 
-        for (i = 0; i < ARRAY_SIZE(valid_args); ++i)
+        if (!strncmp(line, "sm>=", 4))
         {
-            const char *option_text = valid_args[i].text;
-            size_t option_len = strlen(option_text);
-
-            if (strncmp(line, option_text, option_len))
-                continue;
-
             match = true;
-            line += option_len;
+            line += 4;
-            if (valid_args[i].tag)
+            model = match_shader_model_string(line, &line);
-                holds &= match_tag(runner, option_text);
+            *model_mask &= ~((1u << model) - 1);
-            else if (runner->minimum_shader_model < valid_args[i].sm_min
+            if (runner->minimum_shader_model < model)
-                    || runner->minimum_shader_model > valid_args[i].sm_max)
                 holds = false;
+        }
+        else if (!strncmp(line, "sm<", 3))
+        {
+            match = true;
+            line += 3;
+            model = match_shader_model_string(line, &line);
+            *model_mask &= ((1u << model) - 1);
+            if (runner->minimum_shader_model >= model)
+                holds = false;
+        }
+        else
+        {
+            for (unsigned int i = 0; i < ARRAY_SIZE(valid_tags); ++i)
+            {
+                const char *option_text = valid_tags[i];
+                size_t option_len = strlen(option_text);
 
-            break;
+                if (strncmp(line, option_text, option_len))
+                    continue;
+
+                match = true;
+                line += option_len;
+                holds &= match_tag(runner, option_text);
+                break;
+            }
         }
 
         while (isspace(*line))
@@ -189,21 +211,31 @@ static bool check_qualifier_args_conjunction(struct shader_runner *runner, const
     return holds;
 }
 
-static bool check_qualifier_args(struct shader_runner *runner, const char *line, const char **const rest)
+static bool check_qualifier_args(struct shader_runner *runner,
+        const char *line, const char **const rest, uint32_t *model_mask)
 {
     bool first = true;
     bool holds = false;
 
-    assert(*line == '(');
+    if (*line != '(')
+    {
+        *model_mask = ~0u;
+        return true;
+    }
     ++line;
 
+    *model_mask = 0;
+
     while (*line != ')')
     {
+        uint32_t sub_mask;
+
         if (!first && *line == '|')
             ++line;
         first = false;
 
-        holds = check_qualifier_args_conjunction(runner, line, &line) || holds;
+        holds = check_qualifier_args_conjunction(runner, line, &line, &sub_mask) || holds;
+        *model_mask |= sub_mask;
     }
 
     assert(*line == ')');
@@ -217,6 +249,7 @@ static bool match_string_generic(struct shader_runner *runner, const char *line,
         const char *token, const char **const rest, bool allow_qualifier_args)
 {
     size_t len = strlen(token);
+    uint32_t model_mask;
     bool holds = true;
 
     while (isspace(*line))
@@ -226,8 +259,8 @@ static bool match_string_generic(struct shader_runner *runner, const char *line,
         return false;
     line += len;
 
-    if (allow_qualifier_args && *line == '(')
+    if (allow_qualifier_args)
-        holds = check_qualifier_args(runner, line, &line);
+        holds = check_qualifier_args(runner, line, &line, &model_mask);
 
     if (rest)
     {
@@ -1619,8 +1652,7 @@ ID3D10Blob *compile_hlsl(const struct shader_runner *runner, enum shader_type ty
     return blob;
 }
 
-static void compile_shader(struct shader_runner *runner, const char *source, size_t len,
+static void compile_shader(struct shader_runner *runner, const char *source, size_t len, enum shader_type type)
-        enum shader_type type, HRESULT expect)
 {
     bool use_dxcompiler = runner->minimum_shader_model >= SHADER_MODEL_6_0;
     ID3D10Blob *blob = NULL, *errors = NULL;
@@ -1660,8 +1692,8 @@ static void compile_shader(struct shader_runner *runner, const char *source, siz
         hr = D3DCompile(source, len, NULL, NULL, NULL, "main", profile, runner->compile_options, 0, &blob, &errors);
     }
     hr = map_special_hrs(hr);
-    todo_if (runner->is_todo)
+    todo_if (runner->hlsl_todo[runner->minimum_shader_model])
-        ok(hr == expect, "Got unexpected hr %#x.\n", hr);
+        ok(hr == runner->hlsl_hrs[runner->minimum_shader_model], "Got unexpected hr %#x.\n", hr);
     if (hr == S_OK)
     {
         ID3D10Blob_Release(blob);
@@ -1680,33 +1712,47 @@ static void compile_shader(struct shader_runner *runner, const char *source, siz
     }
 }
 
-static void read_shader_directive(struct shader_runner *runner,
+static void read_shader_directive(struct shader_runner *runner, const char *line, const char *src)
-        const char *line, const char *src, HRESULT *expect_hr)
 {
-    *expect_hr = S_OK;
+    for (unsigned int i = SHADER_MODEL_MIN; i <= SHADER_MODEL_MAX; ++i)
-    runner->is_todo = false;
+    {
+        runner->hlsl_hrs[i] = S_OK;
+        runner->hlsl_todo[i] = false;
+    }
 
     while (*src && *src != ']')
     {
-        const char *src_start = src;
+        uint32_t model_mask;
 
-        if (match_string_with_args(runner, src, "todo", &src))
+        if (match_string(src, "todo", &src))
         {
-            /* 'todo' is not meaningful when dxcompiler is in use. */
+            check_qualifier_args(runner, src, &src, &model_mask);
-            if (runner->minimum_shader_model >= SHADER_MODEL_6_0)
+            for (unsigned int i = SHADER_MODEL_MIN; i <= SHADER_MODEL_MAX; ++i)
-                continue;
+            {
-            runner->is_todo = true;
+                /* 'todo' is not meaningful when dxcompiler is in use. */
+                if (i < SHADER_MODEL_6_0 && (model_mask & (1u << i)))
+                    runner->hlsl_todo[i] = true;
+            }
         }
-        else if (match_string_with_args(runner, src, "fail", &src))
+        else if (match_string(src, "fail", &src))
         {
-            *expect_hr = E_FAIL;
+            check_qualifier_args(runner, src, &src, &model_mask);
+            for (unsigned int i = SHADER_MODEL_MIN; i <= SHADER_MODEL_MAX; ++i)
+            {
+                if (model_mask & (1u << i))
+                    runner->hlsl_hrs[i] = E_FAIL;
+            }
         }
-        else if (match_string_with_args(runner, src, "notimpl", &src))
+        else if (match_string(src, "notimpl", &src))
         {
-            *expect_hr = E_NOTIMPL;
+            check_qualifier_args(runner, src, &src, &model_mask);
+            for (unsigned int i = SHADER_MODEL_MIN; i <= SHADER_MODEL_MAX; ++i)
+            {
+                if (model_mask & (1u << i))
+                    runner->hlsl_hrs[i] = E_NOTIMPL;
+            }
         }
-
+        else
-        if (src == src_start)
         {
             fatal_error("Malformed line '%s'.\n", line);
         }
@@ -1865,7 +1911,6 @@ void run_shader_tests(struct shader_runner *runner, const struct shader_runner_c
     unsigned int i, line_number = 0, block_start_line_number = 0;
     enum test_action test_action = TEST_ACTION_RUN;
     char *shader_source = NULL;
-    HRESULT expect_hr = S_OK;
     char line_buffer[256];
     const char *testname;
     FILE *f;
@@ -1958,7 +2003,7 @@ void run_shader_tests(struct shader_runner *runner, const struct shader_runner_c
 
                 case STATE_SHADER:
                     if (test_action != TEST_ACTION_SKIP_COMPILATION)
-                        compile_shader(runner, shader_source, shader_source_len, shader_type, expect_hr);
+                        compile_shader(runner, shader_source, shader_source_len, shader_type);
                     free(runner->shader_source[shader_type]);
                     runner->shader_source[shader_type] = shader_source;
                     shader_source = NULL;
@@ -2189,7 +2234,7 @@ void run_shader_tests(struct shader_runner *runner, const struct shader_runner_c
             }
 
             if (state == STATE_SHADER)
-                read_shader_directive(runner, line_buffer, line, &expect_hr);
+                read_shader_directive(runner, line_buffer, line);
         }
         else if (line[0] != '%' && line[0] != '\n')
         {
@@ -2232,7 +2277,8 @@ void run_shader_tests(struct shader_runner *runner, const struct shader_runner_c
                 case STATE_TEST:
                     /* Compilation which fails with dxcompiler is not 'todo', therefore the tests are
                      * not 'todo' either. They cannot run, so skip them entirely. */
-                    if (!runner->failed_resource_count && test_action == TEST_ACTION_RUN && SUCCEEDED(expect_hr))
+                    if (!runner->failed_resource_count && test_action == TEST_ACTION_RUN
+                            && SUCCEEDED(runner->hlsl_hrs[runner->minimum_shader_model]))
                         parse_test_directive(runner, line);
                     break;
             }

tests/shader_runner.h

@@ -32,12 +32,14 @@
 enum shader_model
 {
     SHADER_MODEL_2_0,
+    SHADER_MODEL_MIN = SHADER_MODEL_2_0,
     SHADER_MODEL_3_0,
     SHADER_MODEL_4_0,
     SHADER_MODEL_4_1,
     SHADER_MODEL_5_0,
     SHADER_MODEL_5_1,
     SHADER_MODEL_6_0,
+    SHADER_MODEL_MAX = SHADER_MODEL_6_0,
 };
 
 enum shader_type
@@ -190,6 +192,8 @@ struct shader_runner
 
     bool is_todo;
     bool is_bug;
+    bool hlsl_todo[SHADER_MODEL_MAX + 1];
+    HRESULT hlsl_hrs[SHADER_MODEL_MAX + 1];
 
     char *shader_source[SHADER_TYPE_COUNT];
     enum shader_model minimum_shader_model;