vkd3d-shader: Implement function-like macro expansion.

Signed-off-by: Zebediah Figura <zfigura@codeweavers.com>
Signed-off-by: Matteo Bruni <mbruni@codeweavers.com>
Signed-off-by: Henri Verbeet <hverbeet@codeweavers.com>
Signed-off-by: Alexandre Julliard <julliard@winehq.org>

Makefile.am

@@ -247,7 +247,6 @@ XFAIL_TESTS = \
 	tests/preproc-if-expr.shader_test \
 	tests/preproc-invalid.shader_test \
 	tests/preproc-macro.shader_test \
-	tests/preproc-misc.shader_test \
 	tests/swizzle-0.shader_test \
 	tests/swizzle-1.shader_test \
 	tests/swizzle-2.shader_test \

libs/vkd3d-shader/preproc.h

@@ -60,6 +60,9 @@ struct preproc_expansion
 {
     struct preproc_buffer buffer;
     const struct preproc_text *text;
+    /* Back-pointer to the macro, if this expansion a macro body. This is
+     * necessary so that argument tokens can be correctly replaced. */
+    struct preproc_macro *macro;
 };
 
 struct preproc_macro
@@ -67,6 +70,10 @@ struct preproc_macro
     struct rb_entry entry;
     char *name;
 
+    char **arg_names;
+    size_t arg_count;
+    struct preproc_text *arg_values;
+
     struct preproc_text body;
 };
 
@@ -86,8 +93,36 @@ struct preproc_ctx
 
     struct rb_tree macros;
 
+    /* It's possible to parse as many as two function-like macros at once: one
+     * in the main text, and another inside of #if directives. E.g.
+     *
+     * func1(
+     * #if func2(...)
+     * #endif
+     * )
+     *
+     * It's not possible to parse more than two, however. In the case of nested
+     * calls like "func1(func2(...))", we store everything inside the outer
+     * parentheses as unparsed text, and then parse it once the argument is
+     * actually invoked.
+     */
+    struct preproc_func_state
+    {
+        struct preproc_macro *macro;
+        size_t arg_count;
+        enum
+        {
+            STATE_NONE = 0,
+            STATE_IDENTIFIER,
+            STATE_ARGS,
+        } state;
+        unsigned int paren_depth;
+    } text_func, directive_func;
+
     int current_directive;
 
+    int lookahead_token;
+
     bool last_was_newline;
     bool last_was_eof;
 

libs/vkd3d-shader/preproc.l

@@ -142,7 +142,7 @@ IDENTIFIER      [A-Za-z_][A-Za-z0-9_]*
     }
 
 <INITIAL>{WS}+                      {}
-<INITIAL>[(),]                      {return yytext[0];}
+<INITIAL>[()\[\]{},]                {return yytext[0];}
 <INITIAL>.                          {return T_TEXT;}
 
 %%
@@ -169,6 +169,26 @@ static void update_location(struct preproc_ctx *ctx)
     }
 }
 
+static bool preproc_is_writing(struct preproc_ctx *ctx)
+{
+    const struct preproc_file *file;
+
+    /* This can happen while checking for unterminated macro invocation. */
+    if (!ctx->file_count)
+        return true;
+    file = preproc_get_top_file(ctx);
+    if (!file->if_count)
+        return true;
+    return file->if_stack[file->if_count - 1].current_true;
+}
+
+static struct preproc_macro *preproc_get_top_macro(struct preproc_ctx *ctx)
+{
+    if (!ctx->expansion_count)
+        return NULL;
+    return ctx->expansion_stack[ctx->expansion_count - 1].macro;
+}
+
 static void preproc_pop_buffer(struct preproc_ctx *ctx)
 {
     if (ctx->expansion_count)
@@ -209,15 +229,6 @@ static void preproc_pop_buffer(struct preproc_ctx *ctx)
         yy_switch_to_buffer(ctx->file_stack[ctx->file_count - 1].buffer.lexer_buffer, ctx->scanner);
 }
 
-static bool preproc_is_writing(struct preproc_ctx *ctx)
-{
-    const struct preproc_file *file = preproc_get_top_file(ctx);
-
-    if (!file->if_count)
-        return true;
-    return file->if_stack[file->if_count - 1].current_true;
-}
-
 static int return_token(int token, YYSTYPE *lval, const char *text)
 {
     switch (token)
@@ -235,7 +246,29 @@ static int return_token(int token, YYSTYPE *lval, const char *text)
     return token;
 }
 
-static bool preproc_push_expansion(struct preproc_ctx *ctx, const struct preproc_text *text)
+static const struct preproc_text *find_arg_expansion(struct preproc_ctx *ctx, const char *s)
+{
+    struct preproc_macro *macro;
+    unsigned int i;
+
+    if ((macro = preproc_get_top_macro(ctx)))
+    {
+        for (i = 0; i < macro->arg_count; ++i)
+        {
+            if (!strcmp(s, macro->arg_names[i]))
+                return &macro->arg_values[i];
+        }
+    }
+    return NULL;
+}
+
+static void preproc_text_add(struct preproc_text *text, const char *string)
+{
+    vkd3d_string_buffer_printf(&text->text, "%s", string);
+}
+
+static bool preproc_push_expansion(struct preproc_ctx *ctx,
+        const struct preproc_text *text, struct preproc_macro *macro)
 {
     struct preproc_expansion *exp;
 
@@ -246,6 +279,7 @@ static bool preproc_push_expansion(struct preproc_ctx *ctx, const struct preproc
     exp->text = text;
     exp->buffer.lexer_buffer = yy_scan_bytes(text->text.buffer, text->text.content_size, ctx->scanner);
     exp->buffer.location = text->location;
+    exp->macro = macro;
     TRACE("Expansion stack size is now %zu.\n", ctx->expansion_count);
     return true;
 }
@@ -256,58 +290,72 @@ int yylex(YYSTYPE *lval, YYLTYPE *lloc, yyscan_t scanner)
 
     for (;;)
     {
+        struct preproc_func_state *func_state;
         const char *text;
         int token;
 
-        if (ctx->last_was_eof)
+        if (ctx->lookahead_token)
         {
-            preproc_pop_buffer(ctx);
+            token = ctx->lookahead_token;
-            if (!ctx->file_count)
+            text = yyget_text(scanner);
-                return 0;
-        }
-        ctx->last_was_eof = false;
-
-        assert(ctx->file_count);
-        if (!(token = preproc_lexer_lex(lval, lloc, scanner)))
-        {
-            ctx->last_was_eof = true;
-
-            /* If we have reached the end of an included file, inject a newline. */
-            if (ctx->expansion_count)
-                continue;
-            token = T_NEWLINE;
-            text = "\n";
         }
         else
         {
-            text = yyget_text(scanner);
+            if (ctx->last_was_eof)
-        }
-
-        if (ctx->last_was_newline)
-        {
-            switch (token)
             {
-                case T_DEFINE:
+                preproc_pop_buffer(ctx);
-                case T_ELIF:
+                if (!ctx->file_count)
-                case T_ELSE:
+                    return 0;
-                case T_ENDIF:
-                case T_IF:
-                case T_IFDEF:
-                case T_IFNDEF:
-                case T_INCLUDE:
-                case T_UNDEF:
-                    ctx->current_directive = token;
-                    break;
-
-                default:
-                    ctx->current_directive = 0;
             }
+            ctx->last_was_eof = false;
+
+            assert(ctx->file_count);
+            if (!(token = preproc_lexer_lex(lval, lloc, scanner)))
+            {
+                ctx->last_was_eof = true;
+
+                /* If we have reached the end of an included file, inject a newline. */
+                if (ctx->expansion_count)
+                    continue;
+                token = T_NEWLINE;
+                text = "\n";
+            }
+            else
+            {
+                text = yyget_text(scanner);
+            }
+
+            if (ctx->last_was_newline)
+            {
+                switch (token)
+                {
+                    case T_DEFINE:
+                    case T_ELIF:
+                    case T_ELSE:
+                    case T_ENDIF:
+                    case T_IF:
+                    case T_IFDEF:
+                    case T_IFNDEF:
+                    case T_INCLUDE:
+                    case T_UNDEF:
+                        ctx->current_directive = token;
+                        break;
+
+                    default:
+                        ctx->current_directive = 0;
+                }
+            }
+
+            ctx->last_was_newline = (token == T_NEWLINE);
         }
 
-        ctx->last_was_newline = (token == T_NEWLINE);
+        func_state = ctx->current_directive ? &ctx->directive_func : &ctx->text_func;
 
-        TRACE("Parsing token %d, line %d, in directive %d, string %s.\n",
+        TRACE("Parsing token %d%s, line %d, in directive %d, state %#x, string %s.\n",
-                token, lloc->line, ctx->current_directive, debugstr_a(text));
+                token, ctx->lookahead_token ? " (lookahead)" : "", lloc->line,
+                ctx->current_directive, func_state->state, debugstr_a(text));
+
+        ctx->lookahead_token = 0;
 
         switch (ctx->current_directive)
         {
@@ -324,33 +372,154 @@ int yylex(YYSTYPE *lval, YYLTYPE *lloc, yyscan_t scanner)
                     continue;
         }
 
-        if (token == T_IDENTIFIER || token == T_IDENTIFIER_PAREN)
+        switch (func_state->state)
         {
-            struct preproc_macro *macro;
+            case STATE_NONE:
+                if (token == T_IDENTIFIER || token == T_IDENTIFIER_PAREN)
+                {
+                    const struct preproc_text *expansion;
+                    struct preproc_macro *macro;
 
-            switch (ctx->current_directive)
+                    switch (ctx->current_directive)
-            {
+                    {
-                case T_DEFINE:
+                        case T_DEFINE:
-                case T_IFDEF:
+                        case T_IFDEF:
-                case T_IFNDEF:
+                        case T_IFNDEF:
-                case T_UNDEF:
+                        case T_UNDEF:
-                    /* Return identifiers verbatim. */
+                            /* Return identifiers verbatim. */
+                            return return_token(token, lval, text);
+                    }
+
+                    /* Otherwise, expand a macro if there is one. */
+
+                    if ((expansion = find_arg_expansion(ctx, text)))
+                    {
+                        preproc_push_expansion(ctx, expansion, NULL);
+                        continue;
+                    }
+
+                    if ((macro = preproc_find_macro(ctx, text)))
+                    {
+                        if (!macro->arg_count)
+                        {
+                            preproc_push_expansion(ctx, &macro->body, macro);
+                        }
+                        else
+                        {
+                            func_state->state = STATE_IDENTIFIER;
+                            func_state->macro = macro;
+                        }
+                        continue;
+                    }
+                }
+
+                if (ctx->current_directive)
                     return return_token(token, lval, text);
-            }
 
-            /* Otherwise, expand a macro if there is one. */
+                vkd3d_string_buffer_printf(&ctx->buffer, "%s ", text);
+                break;
 
-            if ((macro = preproc_find_macro(ctx, text)))
+            case STATE_IDENTIFIER:
+                if (token == '(')
+                {
+                    struct preproc_text *first_arg = &func_state->macro->arg_values[0];
+                    unsigned int i;
+
+                    func_state->arg_count = 0;
+                    func_state->paren_depth = 1;
+                    func_state->state = STATE_ARGS;
+                    for (i = 0; i < func_state->macro->arg_count; ++i)
+                        func_state->macro->arg_values[i].text.content_size = 0;
+
+                    first_arg->location = *lloc;
+                }
+                else
+                {
+                    const char *name = func_state->macro->name;
+
+                    ctx->lookahead_token = token;
+                    func_state->macro = NULL;
+                    func_state->state = STATE_NONE;
+
+                    if (ctx->current_directive)
+                        return return_token(T_IDENTIFIER, lval, name);
+
+                    vkd3d_string_buffer_printf(&ctx->buffer, "%s ", name);
+                }
+                break;
+
+            case STATE_ARGS:
             {
-                preproc_push_expansion(ctx, &macro->body);
+                struct preproc_text *current_arg = NULL;
-                continue;
+
+                assert(func_state->macro->arg_count);
+
+                if (func_state->arg_count < func_state->macro->arg_count)
+                    current_arg = &func_state->macro->arg_values[func_state->arg_count];
+
+                switch (token)
+                {
+                    case T_NEWLINE:
+                        if (current_arg)
+                            preproc_text_add(current_arg, " ");
+                        break;
+
+                    case ')':
+                    case ']':
+                    case '}':
+                        if (!--func_state->paren_depth)
+                        {
+                            if (++func_state->arg_count == func_state->macro->arg_count)
+                            {
+                                preproc_push_expansion(ctx, &func_state->macro->body, func_state->macro);
+                            }
+                            else
+                            {
+                                preproc_warning(ctx, lloc, VKD3D_SHADER_WARNING_PP_ARGUMENT_COUNT_MISMATCH,
+                                        "Wrong number of arguments to macro \"%s\": expected %zu, got %zu.",
+                                        func_state->macro->name, func_state->macro->arg_count, func_state->arg_count);
+
+                                if (ctx->current_directive)
+                                    return return_token(T_IDENTIFIER, lval, func_state->macro->name);
+
+                                vkd3d_string_buffer_printf(&ctx->buffer, "%s ", func_state->macro->name);
+                            }
+                            func_state->macro = NULL;
+                            func_state->state = STATE_NONE;
+                        }
+                        else
+                        {
+                            if (current_arg)
+                                preproc_text_add(current_arg, text);
+                        }
+                        break;
+
+                    case ',':
+                        if (func_state->paren_depth == 1)
+                        {
+                            ++func_state->arg_count;
+                            if (current_arg)
+                                current_arg->location = *lloc;
+                        }
+                        else if (current_arg)
+                        {
+                            preproc_text_add(current_arg, text);
+                        }
+                        break;
+
+                    case '(':
+                    case '[':
+                    case '{':
+                        ++func_state->paren_depth;
+                        /* fall through */
+
+                    default:
+                        if (current_arg)
+                            preproc_text_add(current_arg, text);
+                }
+                break;
             }
         }
-
-        if (ctx->current_directive)
-            return return_token(token, lval, text);
-
-        vkd3d_string_buffer_printf(&ctx->buffer, "%s ", text);
     }
 }
 
@@ -418,6 +587,26 @@ int preproc_lexer_parse(const struct vkd3d_shader_compile_info *compile_info,
 
     preproc_yyparse(ctx.scanner, &ctx);
 
+    switch (ctx.text_func.state)
+    {
+        case STATE_NONE:
+            break;
+
+        case STATE_ARGS:
+        {
+            const struct vkd3d_shader_location loc = {.source_name = source_name};
+
+            preproc_warning(&ctx, &loc, VKD3D_SHADER_WARNING_PP_UNTERMINATED_MACRO,
+                    "Unterminated macro invocation.");
+        }
+        /* fall through */
+
+        case STATE_IDENTIFIER:
+            if (preproc_is_writing(&ctx))
+                vkd3d_string_buffer_printf(&ctx.buffer, "%s ", ctx.text_func.macro->name);
+            break;
+    }
+
     while (ctx.file_count)
         preproc_pop_buffer(&ctx);
     yylex_destroy(ctx.scanner);

libs/vkd3d-shader/preproc.y

@@ -84,9 +84,10 @@ struct preproc_macro *preproc_find_macro(struct preproc_ctx *ctx, const char *na
 }
 
 static bool preproc_add_macro(struct preproc_ctx *ctx, const struct vkd3d_shader_location *loc, char *name,
-        const struct vkd3d_shader_location *body_loc, struct vkd3d_string_buffer *body)
+        char **arg_names, size_t arg_count, const struct vkd3d_shader_location *body_loc, struct vkd3d_string_buffer *body)
 {
     struct preproc_macro *macro;
+    unsigned int i;
     int ret;
 
     if ((macro = preproc_find_macro(ctx, name)))
@@ -96,11 +97,21 @@ static bool preproc_add_macro(struct preproc_ctx *ctx, const struct vkd3d_shader
         preproc_free_macro(macro);
     }
 
-    TRACE("Defining new macro %s.\n", debugstr_a(name));
+    TRACE("Defining new macro %s with %zu arguments.\n", debugstr_a(name), arg_count);
 
     if (!(macro = vkd3d_malloc(sizeof(*macro))))
         return false;
     macro->name = name;
+    macro->arg_names = arg_names;
+    macro->arg_count = arg_count;
+    macro->arg_values = NULL;
+    if (arg_count && !(macro->arg_values = vkd3d_calloc(arg_count, sizeof(*macro->arg_values))))
+    {
+        vkd3d_free(macro);
+        return false;
+    }
+    for (i = 0; i < arg_count; ++i)
+        vkd3d_string_buffer_init(&macro->arg_values[i].text);
     macro->body.text = *body;
     macro->body.location = *body_loc;
     ret = rb_put(&ctx->macros, name, &macro->entry);
@@ -110,7 +121,16 @@ static bool preproc_add_macro(struct preproc_ctx *ctx, const struct vkd3d_shader
 
 void preproc_free_macro(struct preproc_macro *macro)
 {
+    unsigned int i;
+
     vkd3d_free(macro->name);
+    for (i = 0; i < macro->arg_count; ++i)
+    {
+        vkd3d_string_buffer_cleanup(&macro->arg_values[i].text);
+        vkd3d_free(macro->arg_names[i]);
+    }
+    vkd3d_free(macro->arg_names);
+    vkd3d_free(macro->arg_values);
     vkd3d_string_buffer_cleanup(&macro->body.text);
     vkd3d_free(macro);
 }
@@ -379,6 +399,22 @@ body_token_const
         {
             $$ = ")";
         }
+    | '['
+        {
+            $$ = "[";
+        }
+    | ']'
+        {
+            $$ = "]";
+        }
+    | '{'
+        {
+            $$ = "{";
+        }
+    | '}'
+        {
+            $$ = "}";
+        }
     | ','
         {
             $$ = ",";
@@ -387,7 +423,7 @@ body_token_const
 directive
     : T_DEFINE T_IDENTIFIER body_text T_NEWLINE
         {
-            if (!preproc_add_macro(ctx, &@$, $2, &@3, &$3))
+            if (!preproc_add_macro(ctx, &@$, $2, NULL, 0, &@3, &$3))
             {
                 vkd3d_free($2);
                 vkd3d_string_buffer_cleanup(&$3);
@@ -396,10 +432,10 @@ directive
         }
     | T_DEFINE T_IDENTIFIER_PAREN '(' identifier_list ')' body_text T_NEWLINE
         {
-            free_parse_arg_names(&$4);
+            if (!preproc_add_macro(ctx, &@6, $2, $4.args, $4.count, &@6, &$6))
-            if (!preproc_add_macro(ctx, &@6, $2, &@6, &$6))
             {
                 vkd3d_free($2);
+                free_parse_arg_names(&$4);
                 vkd3d_string_buffer_cleanup(&$6);
                 YYABORT;
             }

libs/vkd3d-shader/vkd3d_shader_private.h

@@ -85,7 +85,9 @@ enum vkd3d_shader_error
 
     VKD3D_SHADER_WARNING_PP_ALREADY_DEFINED             = 4300,
     VKD3D_SHADER_WARNING_PP_INVALID_DIRECTIVE           = 4301,
+    VKD3D_SHADER_WARNING_PP_ARGUMENT_COUNT_MISMATCH     = 4302,
     VKD3D_SHADER_WARNING_PP_UNKNOWN_DIRECTIVE           = 4303,
+    VKD3D_SHADER_WARNING_PP_UNTERMINATED_MACRO          = 4304,
     VKD3D_SHADER_WARNING_PP_UNTERMINATED_IF             = 4305,
 };
 

tests/hlsl_d3d12.c

@@ -352,7 +352,7 @@ static void test_preprocess(void)
         if (i == 10 || i == 11)
             continue;
         vkd3d_test_set_context("Source \"%s\"", tests[i].source);
-        todo_if (i <= 4 || (i >= 9 && i <= 14) || i == 43)
+        todo_if (i <= 4 || (i >= 9 && i <= 14))
             check_preprocess(tests[i].source, NULL, NULL, tests[i].present, tests[i].absent);
     }
     vkd3d_test_set_context(NULL);