diff --git a/Makefile.am b/Makefile.am index 514f0506..22151adf 100644 --- a/Makefile.am +++ b/Makefile.am @@ -247,7 +247,6 @@ XFAIL_TESTS = \ tests/preproc-if-expr.shader_test \ tests/preproc-invalid.shader_test \ tests/preproc-macro.shader_test \ - tests/preproc-misc.shader_test \ tests/swizzle-0.shader_test \ tests/swizzle-1.shader_test \ tests/swizzle-2.shader_test \ diff --git a/libs/vkd3d-shader/preproc.h b/libs/vkd3d-shader/preproc.h index 7d6bda2a..0f2d8ba9 100644 --- a/libs/vkd3d-shader/preproc.h +++ b/libs/vkd3d-shader/preproc.h @@ -60,6 +60,9 @@ struct preproc_expansion { struct preproc_buffer buffer; const struct preproc_text *text; + /* Back-pointer to the macro, if this expansion a macro body. This is + * necessary so that argument tokens can be correctly replaced. */ + struct preproc_macro *macro; }; struct preproc_macro @@ -67,6 +70,10 @@ struct preproc_macro struct rb_entry entry; char *name; + char **arg_names; + size_t arg_count; + struct preproc_text *arg_values; + struct preproc_text body; }; @@ -86,8 +93,36 @@ struct preproc_ctx struct rb_tree macros; + /* It's possible to parse as many as two function-like macros at once: one + * in the main text, and another inside of #if directives. E.g. + * + * func1( + * #if func2(...) + * #endif + * ) + * + * It's not possible to parse more than two, however. In the case of nested + * calls like "func1(func2(...))", we store everything inside the outer + * parentheses as unparsed text, and then parse it once the argument is + * actually invoked. + */ + struct preproc_func_state + { + struct preproc_macro *macro; + size_t arg_count; + enum + { + STATE_NONE = 0, + STATE_IDENTIFIER, + STATE_ARGS, + } state; + unsigned int paren_depth; + } text_func, directive_func; + int current_directive; + int lookahead_token; + bool last_was_newline; bool last_was_eof; diff --git a/libs/vkd3d-shader/preproc.l b/libs/vkd3d-shader/preproc.l index 77f4f0dc..9043adee 100644 --- a/libs/vkd3d-shader/preproc.l +++ b/libs/vkd3d-shader/preproc.l @@ -142,7 +142,7 @@ IDENTIFIER [A-Za-z_][A-Za-z0-9_]* } {WS}+ {} -[(),] {return yytext[0];} +[()\[\]{},] {return yytext[0];} . {return T_TEXT;} %% @@ -169,6 +169,26 @@ static void update_location(struct preproc_ctx *ctx) } } +static bool preproc_is_writing(struct preproc_ctx *ctx) +{ + const struct preproc_file *file; + + /* This can happen while checking for unterminated macro invocation. */ + if (!ctx->file_count) + return true; + file = preproc_get_top_file(ctx); + if (!file->if_count) + return true; + return file->if_stack[file->if_count - 1].current_true; +} + +static struct preproc_macro *preproc_get_top_macro(struct preproc_ctx *ctx) +{ + if (!ctx->expansion_count) + return NULL; + return ctx->expansion_stack[ctx->expansion_count - 1].macro; +} + static void preproc_pop_buffer(struct preproc_ctx *ctx) { if (ctx->expansion_count) @@ -209,15 +229,6 @@ static void preproc_pop_buffer(struct preproc_ctx *ctx) yy_switch_to_buffer(ctx->file_stack[ctx->file_count - 1].buffer.lexer_buffer, ctx->scanner); } -static bool preproc_is_writing(struct preproc_ctx *ctx) -{ - const struct preproc_file *file = preproc_get_top_file(ctx); - - if (!file->if_count) - return true; - return file->if_stack[file->if_count - 1].current_true; -} - static int return_token(int token, YYSTYPE *lval, const char *text) { switch (token) @@ -235,7 +246,29 @@ static int return_token(int token, YYSTYPE *lval, const char *text) return token; } -static bool preproc_push_expansion(struct preproc_ctx *ctx, const struct preproc_text *text) +static const struct preproc_text *find_arg_expansion(struct preproc_ctx *ctx, const char *s) +{ + struct preproc_macro *macro; + unsigned int i; + + if ((macro = preproc_get_top_macro(ctx))) + { + for (i = 0; i < macro->arg_count; ++i) + { + if (!strcmp(s, macro->arg_names[i])) + return ¯o->arg_values[i]; + } + } + return NULL; +} + +static void preproc_text_add(struct preproc_text *text, const char *string) +{ + vkd3d_string_buffer_printf(&text->text, "%s", string); +} + +static bool preproc_push_expansion(struct preproc_ctx *ctx, + const struct preproc_text *text, struct preproc_macro *macro) { struct preproc_expansion *exp; @@ -246,6 +279,7 @@ static bool preproc_push_expansion(struct preproc_ctx *ctx, const struct preproc exp->text = text; exp->buffer.lexer_buffer = yy_scan_bytes(text->text.buffer, text->text.content_size, ctx->scanner); exp->buffer.location = text->location; + exp->macro = macro; TRACE("Expansion stack size is now %zu.\n", ctx->expansion_count); return true; } @@ -256,58 +290,72 @@ int yylex(YYSTYPE *lval, YYLTYPE *lloc, yyscan_t scanner) for (;;) { + struct preproc_func_state *func_state; const char *text; int token; - if (ctx->last_was_eof) + if (ctx->lookahead_token) { - preproc_pop_buffer(ctx); - if (!ctx->file_count) - return 0; - } - ctx->last_was_eof = false; - - assert(ctx->file_count); - if (!(token = preproc_lexer_lex(lval, lloc, scanner))) - { - ctx->last_was_eof = true; - - /* If we have reached the end of an included file, inject a newline. */ - if (ctx->expansion_count) - continue; - token = T_NEWLINE; - text = "\n"; + token = ctx->lookahead_token; + text = yyget_text(scanner); } else { - text = yyget_text(scanner); - } - - if (ctx->last_was_newline) - { - switch (token) + if (ctx->last_was_eof) { - case T_DEFINE: - case T_ELIF: - case T_ELSE: - case T_ENDIF: - case T_IF: - case T_IFDEF: - case T_IFNDEF: - case T_INCLUDE: - case T_UNDEF: - ctx->current_directive = token; - break; - - default: - ctx->current_directive = 0; + preproc_pop_buffer(ctx); + if (!ctx->file_count) + return 0; } + ctx->last_was_eof = false; + + assert(ctx->file_count); + if (!(token = preproc_lexer_lex(lval, lloc, scanner))) + { + ctx->last_was_eof = true; + + /* If we have reached the end of an included file, inject a newline. */ + if (ctx->expansion_count) + continue; + token = T_NEWLINE; + text = "\n"; + } + else + { + text = yyget_text(scanner); + } + + if (ctx->last_was_newline) + { + switch (token) + { + case T_DEFINE: + case T_ELIF: + case T_ELSE: + case T_ENDIF: + case T_IF: + case T_IFDEF: + case T_IFNDEF: + case T_INCLUDE: + case T_UNDEF: + ctx->current_directive = token; + break; + + default: + ctx->current_directive = 0; + } + } + + ctx->last_was_newline = (token == T_NEWLINE); } - ctx->last_was_newline = (token == T_NEWLINE); + func_state = ctx->current_directive ? &ctx->directive_func : &ctx->text_func; - TRACE("Parsing token %d, line %d, in directive %d, string %s.\n", - token, lloc->line, ctx->current_directive, debugstr_a(text)); + TRACE("Parsing token %d%s, line %d, in directive %d, state %#x, string %s.\n", + token, ctx->lookahead_token ? " (lookahead)" : "", lloc->line, + ctx->current_directive, func_state->state, debugstr_a(text)); + + ctx->lookahead_token = 0; switch (ctx->current_directive) { @@ -324,33 +372,154 @@ int yylex(YYSTYPE *lval, YYLTYPE *lloc, yyscan_t scanner) continue; } - if (token == T_IDENTIFIER || token == T_IDENTIFIER_PAREN) + switch (func_state->state) { - struct preproc_macro *macro; + case STATE_NONE: + if (token == T_IDENTIFIER || token == T_IDENTIFIER_PAREN) + { + const struct preproc_text *expansion; + struct preproc_macro *macro; - switch (ctx->current_directive) - { - case T_DEFINE: - case T_IFDEF: - case T_IFNDEF: - case T_UNDEF: - /* Return identifiers verbatim. */ + switch (ctx->current_directive) + { + case T_DEFINE: + case T_IFDEF: + case T_IFNDEF: + case T_UNDEF: + /* Return identifiers verbatim. */ + return return_token(token, lval, text); + } + + /* Otherwise, expand a macro if there is one. */ + + if ((expansion = find_arg_expansion(ctx, text))) + { + preproc_push_expansion(ctx, expansion, NULL); + continue; + } + + if ((macro = preproc_find_macro(ctx, text))) + { + if (!macro->arg_count) + { + preproc_push_expansion(ctx, ¯o->body, macro); + } + else + { + func_state->state = STATE_IDENTIFIER; + func_state->macro = macro; + } + continue; + } + } + + if (ctx->current_directive) return return_token(token, lval, text); - } - /* Otherwise, expand a macro if there is one. */ + vkd3d_string_buffer_printf(&ctx->buffer, "%s ", text); + break; - if ((macro = preproc_find_macro(ctx, text))) + case STATE_IDENTIFIER: + if (token == '(') + { + struct preproc_text *first_arg = &func_state->macro->arg_values[0]; + unsigned int i; + + func_state->arg_count = 0; + func_state->paren_depth = 1; + func_state->state = STATE_ARGS; + for (i = 0; i < func_state->macro->arg_count; ++i) + func_state->macro->arg_values[i].text.content_size = 0; + + first_arg->location = *lloc; + } + else + { + const char *name = func_state->macro->name; + + ctx->lookahead_token = token; + func_state->macro = NULL; + func_state->state = STATE_NONE; + + if (ctx->current_directive) + return return_token(T_IDENTIFIER, lval, name); + + vkd3d_string_buffer_printf(&ctx->buffer, "%s ", name); + } + break; + + case STATE_ARGS: { - preproc_push_expansion(ctx, ¯o->body); - continue; + struct preproc_text *current_arg = NULL; + + assert(func_state->macro->arg_count); + + if (func_state->arg_count < func_state->macro->arg_count) + current_arg = &func_state->macro->arg_values[func_state->arg_count]; + + switch (token) + { + case T_NEWLINE: + if (current_arg) + preproc_text_add(current_arg, " "); + break; + + case ')': + case ']': + case '}': + if (!--func_state->paren_depth) + { + if (++func_state->arg_count == func_state->macro->arg_count) + { + preproc_push_expansion(ctx, &func_state->macro->body, func_state->macro); + } + else + { + preproc_warning(ctx, lloc, VKD3D_SHADER_WARNING_PP_ARGUMENT_COUNT_MISMATCH, + "Wrong number of arguments to macro \"%s\": expected %zu, got %zu.", + func_state->macro->name, func_state->macro->arg_count, func_state->arg_count); + + if (ctx->current_directive) + return return_token(T_IDENTIFIER, lval, func_state->macro->name); + + vkd3d_string_buffer_printf(&ctx->buffer, "%s ", func_state->macro->name); + } + func_state->macro = NULL; + func_state->state = STATE_NONE; + } + else + { + if (current_arg) + preproc_text_add(current_arg, text); + } + break; + + case ',': + if (func_state->paren_depth == 1) + { + ++func_state->arg_count; + if (current_arg) + current_arg->location = *lloc; + } + else if (current_arg) + { + preproc_text_add(current_arg, text); + } + break; + + case '(': + case '[': + case '{': + ++func_state->paren_depth; + /* fall through */ + + default: + if (current_arg) + preproc_text_add(current_arg, text); + } + break; } } - - if (ctx->current_directive) - return return_token(token, lval, text); - - vkd3d_string_buffer_printf(&ctx->buffer, "%s ", text); } } @@ -418,6 +587,26 @@ int preproc_lexer_parse(const struct vkd3d_shader_compile_info *compile_info, preproc_yyparse(ctx.scanner, &ctx); + switch (ctx.text_func.state) + { + case STATE_NONE: + break; + + case STATE_ARGS: + { + const struct vkd3d_shader_location loc = {.source_name = source_name}; + + preproc_warning(&ctx, &loc, VKD3D_SHADER_WARNING_PP_UNTERMINATED_MACRO, + "Unterminated macro invocation."); + } + /* fall through */ + + case STATE_IDENTIFIER: + if (preproc_is_writing(&ctx)) + vkd3d_string_buffer_printf(&ctx.buffer, "%s ", ctx.text_func.macro->name); + break; + } + while (ctx.file_count) preproc_pop_buffer(&ctx); yylex_destroy(ctx.scanner); diff --git a/libs/vkd3d-shader/preproc.y b/libs/vkd3d-shader/preproc.y index 3cb7de61..4173af0f 100644 --- a/libs/vkd3d-shader/preproc.y +++ b/libs/vkd3d-shader/preproc.y @@ -84,9 +84,10 @@ struct preproc_macro *preproc_find_macro(struct preproc_ctx *ctx, const char *na } static bool preproc_add_macro(struct preproc_ctx *ctx, const struct vkd3d_shader_location *loc, char *name, - const struct vkd3d_shader_location *body_loc, struct vkd3d_string_buffer *body) + char **arg_names, size_t arg_count, const struct vkd3d_shader_location *body_loc, struct vkd3d_string_buffer *body) { struct preproc_macro *macro; + unsigned int i; int ret; if ((macro = preproc_find_macro(ctx, name))) @@ -96,11 +97,21 @@ static bool preproc_add_macro(struct preproc_ctx *ctx, const struct vkd3d_shader preproc_free_macro(macro); } - TRACE("Defining new macro %s.\n", debugstr_a(name)); + TRACE("Defining new macro %s with %zu arguments.\n", debugstr_a(name), arg_count); if (!(macro = vkd3d_malloc(sizeof(*macro)))) return false; macro->name = name; + macro->arg_names = arg_names; + macro->arg_count = arg_count; + macro->arg_values = NULL; + if (arg_count && !(macro->arg_values = vkd3d_calloc(arg_count, sizeof(*macro->arg_values)))) + { + vkd3d_free(macro); + return false; + } + for (i = 0; i < arg_count; ++i) + vkd3d_string_buffer_init(¯o->arg_values[i].text); macro->body.text = *body; macro->body.location = *body_loc; ret = rb_put(&ctx->macros, name, ¯o->entry); @@ -110,7 +121,16 @@ static bool preproc_add_macro(struct preproc_ctx *ctx, const struct vkd3d_shader void preproc_free_macro(struct preproc_macro *macro) { + unsigned int i; + vkd3d_free(macro->name); + for (i = 0; i < macro->arg_count; ++i) + { + vkd3d_string_buffer_cleanup(¯o->arg_values[i].text); + vkd3d_free(macro->arg_names[i]); + } + vkd3d_free(macro->arg_names); + vkd3d_free(macro->arg_values); vkd3d_string_buffer_cleanup(¯o->body.text); vkd3d_free(macro); } @@ -379,6 +399,22 @@ body_token_const { $$ = ")"; } + | '[' + { + $$ = "["; + } + | ']' + { + $$ = "]"; + } + | '{' + { + $$ = "{"; + } + | '}' + { + $$ = "}"; + } | ',' { $$ = ","; @@ -387,7 +423,7 @@ body_token_const directive : T_DEFINE T_IDENTIFIER body_text T_NEWLINE { - if (!preproc_add_macro(ctx, &@$, $2, &@3, &$3)) + if (!preproc_add_macro(ctx, &@$, $2, NULL, 0, &@3, &$3)) { vkd3d_free($2); vkd3d_string_buffer_cleanup(&$3); @@ -396,10 +432,10 @@ directive } | T_DEFINE T_IDENTIFIER_PAREN '(' identifier_list ')' body_text T_NEWLINE { - free_parse_arg_names(&$4); - if (!preproc_add_macro(ctx, &@6, $2, &@6, &$6)) + if (!preproc_add_macro(ctx, &@6, $2, $4.args, $4.count, &@6, &$6)) { vkd3d_free($2); + free_parse_arg_names(&$4); vkd3d_string_buffer_cleanup(&$6); YYABORT; } diff --git a/libs/vkd3d-shader/vkd3d_shader_private.h b/libs/vkd3d-shader/vkd3d_shader_private.h index 5224c929..157dc5e2 100644 --- a/libs/vkd3d-shader/vkd3d_shader_private.h +++ b/libs/vkd3d-shader/vkd3d_shader_private.h @@ -85,7 +85,9 @@ enum vkd3d_shader_error VKD3D_SHADER_WARNING_PP_ALREADY_DEFINED = 4300, VKD3D_SHADER_WARNING_PP_INVALID_DIRECTIVE = 4301, + VKD3D_SHADER_WARNING_PP_ARGUMENT_COUNT_MISMATCH = 4302, VKD3D_SHADER_WARNING_PP_UNKNOWN_DIRECTIVE = 4303, + VKD3D_SHADER_WARNING_PP_UNTERMINATED_MACRO = 4304, VKD3D_SHADER_WARNING_PP_UNTERMINATED_IF = 4305, }; diff --git a/tests/hlsl_d3d12.c b/tests/hlsl_d3d12.c index ac616c23..6980b998 100644 --- a/tests/hlsl_d3d12.c +++ b/tests/hlsl_d3d12.c @@ -352,7 +352,7 @@ static void test_preprocess(void) if (i == 10 || i == 11) continue; vkd3d_test_set_context("Source \"%s\"", tests[i].source); - todo_if (i <= 4 || (i >= 9 && i <= 14) || i == 43) + todo_if (i <= 4 || (i >= 9 && i <= 14)) check_preprocess(tests[i].source, NULL, NULL, tests[i].present, tests[i].absent); } vkd3d_test_set_context(NULL);