vkd3d-shader/hlsl: Forbid recursive calls.

Approved-by: Giovanni Mascellani (@giomasce)
Approved-by: Francisco Casas (@fcasas)
Approved-by: Henri Verbeet (@hverbeet)
Approved-by: Alexandre Julliard (@julliard)
Merge-Request: https://gitlab.winehq.org/wine/vkd3d/-/merge_requests/63

libs/vkd3d-shader/hlsl_codegen.c

@@ -457,6 +457,48 @@ static bool transform_ir(struct hlsl_ctx *ctx, bool (*func)(struct hlsl_ctx *ctx
     return progress;
 }
 
+struct recursive_call_ctx
+{
+    const struct hlsl_ir_function_decl **backtrace;
+    size_t count, capacity;
+};
+
+static bool find_recursive_calls(struct hlsl_ctx *ctx, struct hlsl_ir_node *instr, void *context)
+{
+    struct recursive_call_ctx *call_ctx = context;
+    struct hlsl_ir_function_decl *decl;
+    const struct hlsl_ir_call *call;
+    size_t i;
+
+    if (instr->type != HLSL_IR_CALL)
+        return false;
+    call = hlsl_ir_call(instr);
+    decl = call->decl;
+
+    for (i = 0; i < call_ctx->count; ++i)
+    {
+        if (call_ctx->backtrace[i] == decl)
+        {
+            hlsl_error(ctx, &call->node.loc, VKD3D_SHADER_ERROR_HLSL_RECURSIVE_CALL,
+                    "Recursive call to \"%s\".", decl->func->name);
+            /* Native returns E_NOTIMPL instead of E_FAIL here. */
+            ctx->result = VKD3D_ERROR_NOT_IMPLEMENTED;
+            return false;
+        }
+    }
+
+    if (!hlsl_array_reserve(ctx, (void **)&call_ctx->backtrace, &call_ctx->capacity,
+            call_ctx->count + 1, sizeof(*call_ctx->backtrace)))
+        return false;
+    call_ctx->backtrace[call_ctx->count++] = decl;
+
+    transform_ir(ctx, find_recursive_calls, &decl->body, call_ctx);
+
+    --call_ctx->count;
+
+    return false;
+}
+
 /* Lower casts from vec1 to vecN to swizzles. */
 static bool lower_broadcasts(struct hlsl_ctx *ctx, struct hlsl_ir_node *instr, void *context)
 {
@@ -2628,12 +2670,17 @@ int hlsl_emit_bytecode(struct hlsl_ctx *ctx, struct hlsl_ir_function_decl *entry
 {
     const struct hlsl_profile_info *profile = ctx->profile;
     struct hlsl_block *const body = &entry_func->body;
+    struct recursive_call_ctx recursive_call_ctx;
     struct hlsl_ir_var *var;
     unsigned int i;
     bool progress;
 
     list_move_head(&body->instrs, &ctx->static_initializers);
 
+    memset(&recursive_call_ctx, 0, sizeof(recursive_call_ctx));
+    transform_ir(ctx, find_recursive_calls, body, &recursive_call_ctx);
+    vkd3d_free(recursive_call_ctx.backtrace);
+
     LIST_FOR_EACH_ENTRY(var, &ctx->globals->vars, struct hlsl_ir_var, scope_entry)
     {
         if (var->storage_modifiers & HLSL_STORAGE_UNIFORM)

libs/vkd3d-shader/vkd3d_shader_private.h

@@ -121,6 +121,7 @@ enum vkd3d_shader_error
     VKD3D_SHADER_ERROR_HLSL_NON_STATIC_OBJECT_REF       = 5022,
     VKD3D_SHADER_ERROR_HLSL_INVALID_THREAD_COUNT        = 5023,
     VKD3D_SHADER_ERROR_HLSL_MISSING_ATTRIBUTE           = 5024,
+    VKD3D_SHADER_ERROR_HLSL_RECURSIVE_CALL              = 5025,
 
     VKD3D_SHADER_WARNING_HLSL_IMPLICIT_TRUNCATION       = 5300,
     VKD3D_SHADER_WARNING_HLSL_DIVISION_BY_ZERO          = 5301,

tests/hlsl-function.shader_test

@@ -162,3 +162,41 @@ float4 main() : sv_target
 [test]
 draw quad
 todo probe all rgba (0.6, 0.1, 0.5, 0)
+
+% Recursion is forbidden.
+
+[pixel shader notimpl]
+
+void bar();
+
+void foo()
+{
+    bar();
+}
+
+void bar()
+{
+    foo();
+}
+
+float4 main() : sv_target
+{
+    foo();
+    return 0;
+}
+
+[pixel shader notimpl todo]
+
+% Even trivially finite recursion is forbidden.
+
+void func(bool x)
+{
+    if (x)
+        func(false);
+}
+
+float4 main() : sv_target
+{
+    func(true);
+    return 0;
+}

tests/shader_runner.c

@@ -893,6 +893,16 @@ void run_shader_tests(struct shader_runner *runner, int argc, char **argv, const
                 state = STATE_SHADER_PIXEL_TODO;
                 expect_hr = E_FAIL;
             }
+            else if (!strcmp(line, "[pixel shader notimpl]\n"))
+            {
+                state = STATE_SHADER_PIXEL;
+                expect_hr = E_NOTIMPL;
+            }
+            else if (!strcmp(line, "[pixel shader notimpl todo]\n"))
+            {
+                state = STATE_SHADER_PIXEL_TODO;
+                expect_hr = E_NOTIMPL;
+            }
             else if (sscanf(line, "[sampler %u]\n", &index))
             {
                 state = STATE_SAMPLER;