vkd3d-shader: Implement #include.

Signed-off-by: Zebediah Figura <zfigura@codeweavers.com>
Signed-off-by: Matteo Bruni <mbruni@codeweavers.com>
Signed-off-by: Henri Verbeet <hverbeet@codeweavers.com>
Signed-off-by: Alexandre Julliard <julliard@winehq.org>
This commit is contained in:
Zebediah Figura 2021-01-07 11:48:09 -06:00 committed by Alexandre Julliard
parent 568983596e
commit 8c927c491c
5 changed files with 281 additions and 46 deletions

View File

@ -34,6 +34,22 @@ struct preproc_if_state
bool seen_else;
};
struct preproc_buffer
{
void *lexer_buffer;
struct vkd3d_shader_location location;
};
struct preproc_file
{
struct preproc_buffer buffer;
struct vkd3d_shader_code code;
char *filename;
struct preproc_if_state *if_stack;
size_t if_count, if_stack_size;
};
struct preproc_macro
{
struct rb_entry entry;
@ -42,14 +58,14 @@ struct preproc_macro
struct preproc_ctx
{
const struct vkd3d_shader_preprocess_info *preprocess_info;
void *scanner;
struct vkd3d_shader_message_context *message_context;
struct vkd3d_string_buffer buffer;
struct vkd3d_shader_location location;
struct preproc_if_state *if_stack;
size_t if_count, if_stack_size;
struct preproc_file *file_stack;
size_t file_count, file_stack_size;
struct rb_tree macros;
@ -61,8 +77,16 @@ struct preproc_ctx
bool error;
};
void preproc_close_include(struct preproc_ctx *ctx, const struct vkd3d_shader_code *code) DECLSPEC_HIDDEN;
void preproc_free_macro(struct preproc_macro *macro) DECLSPEC_HIDDEN;
bool preproc_push_include(struct preproc_ctx *ctx, char *filename, const struct vkd3d_shader_code *code) DECLSPEC_HIDDEN;
void preproc_warning(struct preproc_ctx *ctx, const struct vkd3d_shader_location *loc,
enum vkd3d_shader_error error, const char *format, ...) VKD3D_PRINTF_FUNC(4, 5) DECLSPEC_HIDDEN;
static inline struct preproc_file *preproc_get_top_file(struct preproc_ctx *ctx)
{
assert(ctx->file_count);
return &ctx->file_stack[ctx->file_count - 1];
}
#endif

View File

@ -50,6 +50,8 @@ static void update_location(struct preproc_ctx *ctx);
%s C_COMMENT
%s CXX_COMMENT
%s INCLUDE
NEWLINE \r?\n
WS [ \t]
IDENTIFIER [A-Za-z_][A-Za-z0-9_]*
@ -61,6 +63,7 @@ IDENTIFIER [A-Za-z_][A-Za-z0-9_]*
<CXX_COMMENT>\\{NEWLINE} {}
<CXX_COMMENT>\n {
yy_pop_state(yyscanner);
BEGIN(INITIAL);
return T_NEWLINE;
}
<C_COMMENT>"*/" {yy_pop_state(yyscanner);}
@ -87,6 +90,9 @@ IDENTIFIER [A-Za-z_][A-Za-z0-9_]*
<INITIAL>">>"=? {return T_TEXT;}
<INITIAL>[-+*/%&|^=><!]= {return T_TEXT;}
<INCLUDE>\"[^"]*\" {return T_STRING;}
<INCLUDE>\<[^>]*\> {return T_STRING;}
/* C strings (including escaped quotes). */
<INITIAL>\"([^"\\]|\\.)*\" {return T_TEXT;}
@ -100,6 +106,12 @@ IDENTIFIER [A-Za-z_][A-Za-z0-9_]*
for (p = yytext + 1; strchr(" \t", *p); ++p)
;
if (!strcmp(p, "include"))
{
BEGIN(INCLUDE);
return T_INCLUDE;
}
if (!strcmp(p, "define"))
return T_DEFINE;
if (!strcmp(p, "elif"))
@ -120,8 +132,11 @@ IDENTIFIER [A-Za-z_][A-Za-z0-9_]*
return T_TEXT;
}
<INITIAL>\\{NEWLINE} {}
<INITIAL>{NEWLINE} {return T_NEWLINE;}
<INITIAL,INCLUDE>\\{NEWLINE} {}
<INITIAL,INCLUDE>{NEWLINE} {
BEGIN(INITIAL);
return T_NEWLINE;
}
<INITIAL>{WS}+ {}
<INITIAL>. {return T_TEXT;}
@ -130,30 +145,59 @@ IDENTIFIER [A-Za-z_][A-Za-z0-9_]*
static void update_location(struct preproc_ctx *ctx)
{
struct preproc_buffer *buffer = &preproc_get_top_file(ctx)->buffer;
unsigned int i, leng = yyget_leng(ctx->scanner);
const char *text = yyget_text(ctx->scanner);
/* We want to do this here, rather than before calling yylex(), because
* some tokens are skipped by the lexer. */
*yyget_lloc(ctx->scanner) = ctx->location;
*yyget_lloc(ctx->scanner) = buffer->location;
for (i = 0; i < leng; ++i)
{
++ctx->location.column;
++buffer->location.column;
if (text[i] == '\n')
{
ctx->location.column = 1;
++ctx->location.line;
buffer->location.column = 1;
++buffer->location.line;
}
}
}
static void preproc_pop_buffer(struct preproc_ctx *ctx)
{
struct preproc_file *file = preproc_get_top_file(ctx);
if (ctx->file_count > 1)
preproc_close_include(ctx, &file->code);
if (file->if_count)
{
const struct vkd3d_shader_location loc = {.source_name = file->filename};
preproc_warning(ctx, &loc, VKD3D_SHADER_WARNING_PP_UNTERMINATED_IF, "Unterminated #if block.");
}
vkd3d_free(file->if_stack);
vkd3d_free(file->filename);
yy_delete_buffer(file->buffer.lexer_buffer, ctx->scanner);
--ctx->file_count;
TRACE("File stack size is now %zu.\n", ctx->file_count);
if (ctx->file_count)
yy_switch_to_buffer(ctx->file_stack[ctx->file_count - 1].buffer.lexer_buffer, ctx->scanner);
}
static bool preproc_is_writing(struct preproc_ctx *ctx)
{
if (!ctx->if_count)
const struct preproc_file *file = preproc_get_top_file(ctx);
if (!file->if_count)
return true;
return ctx->if_stack[ctx->if_count - 1].current_true;
return file->if_stack[file->if_count - 1].current_true;
}
static int return_token(int token, YYSTYPE *lval, const char *text)
@ -162,6 +206,7 @@ static int return_token(int token, YYSTYPE *lval, const char *text)
{
case T_IDENTIFIER:
case T_INTEGER:
case T_STRING:
case T_TEXT:
if (!(lval->string = vkd3d_strdup(text)))
return 0;
@ -181,8 +226,14 @@ int yylex(YYSTYPE *lval, YYLTYPE *lloc, yyscan_t scanner)
int token;
if (ctx->last_was_eof)
{
preproc_pop_buffer(ctx);
if (!ctx->file_count)
return 0;
}
ctx->last_was_eof = false;
assert(ctx->file_count);
if (!(token = preproc_lexer_lex(lval, lloc, scanner)))
{
ctx->last_was_eof = true;
@ -205,6 +256,7 @@ int yylex(YYSTYPE *lval, YYLTYPE *lloc, yyscan_t scanner)
case T_IF:
case T_IFDEF:
case T_IFNDEF:
case T_INCLUDE:
ctx->current_directive = token;
break;
@ -240,6 +292,26 @@ int yylex(YYSTYPE *lval, YYLTYPE *lloc, yyscan_t scanner)
}
}
bool preproc_push_include(struct preproc_ctx *ctx, char *filename, const struct vkd3d_shader_code *code)
{
struct preproc_file *file;
if (!vkd3d_array_reserve((void **)&ctx->file_stack, &ctx->file_stack_size,
ctx->file_count + 1, sizeof(*ctx->file_stack)))
return false;
file = &ctx->file_stack[ctx->file_count++];
memset(file, 0, sizeof(*file));
file->code = *code;
file->filename = filename;
file->buffer.lexer_buffer = yy_scan_bytes(code->code, code->size, ctx->scanner);
file->buffer.location.source_name = file->filename;
file->buffer.location.line = 1;
file->buffer.location.column = 1;
TRACE("File stack size is now %zu.\n", ctx->file_count);
ctx->last_was_newline = true;
return true;
}
static int preproc_macro_compare(const void *key, const struct rb_entry *entry)
{
const struct preproc_macro *macro = RB_ENTRY_VALUE(entry, struct preproc_macro, entry);
@ -256,35 +328,40 @@ static void preproc_macro_rb_free(struct rb_entry *entry, void *ctx)
int preproc_lexer_parse(const struct vkd3d_shader_compile_info *compile_info,
struct vkd3d_shader_code *out, struct vkd3d_shader_message_context *message_context)
{
static const struct vkd3d_shader_preprocess_info default_preprocess_info = {0};
struct preproc_ctx ctx = {0};
YY_BUFFER_STATE top_buffer;
char *source_name;
void *output_code;
vkd3d_string_buffer_init(&ctx.buffer);
rb_init(&ctx.macros, preproc_macro_compare);
if (!(ctx.preprocess_info = vkd3d_find_struct(compile_info->next, PREPROCESS_INFO)))
ctx.preprocess_info = &default_preprocess_info;
ctx.message_context = message_context;
ctx.location.source_name = compile_info->source_name;
ctx.location.line = 1;
ctx.location.column = 1;
if (!(source_name = vkd3d_strdup(compile_info->source_name ? compile_info->source_name : "<anonymous>")))
{
vkd3d_string_buffer_cleanup(&ctx.buffer);
return VKD3D_ERROR_OUT_OF_MEMORY;
}
yylex_init_extra(&ctx, &ctx.scanner);
top_buffer = yy_scan_bytes(compile_info->source.code, compile_info->source.size, ctx.scanner);
ctx.last_was_newline = true;
if (!preproc_push_include(&ctx, source_name, &compile_info->source))
{
yylex_destroy(ctx.scanner);
vkd3d_free(source_name);
vkd3d_string_buffer_cleanup(&ctx.buffer);
return VKD3D_ERROR_OUT_OF_MEMORY;
}
preproc_yyparse(ctx.scanner, &ctx);
yy_delete_buffer(top_buffer, ctx.scanner);
while (ctx.file_count)
preproc_pop_buffer(&ctx);
yylex_destroy(ctx.scanner);
if (ctx.if_count)
{
const struct vkd3d_shader_location loc = {.source_name = ctx.location.source_name};
preproc_warning(&ctx, &loc, VKD3D_SHADER_WARNING_PP_UNTERMINATED_IF, "Unterminated #if block.");
}
vkd3d_free(ctx.if_stack);
rb_destroy(&ctx.macros, preproc_macro_rb_free, NULL);
vkd3d_free(ctx.file_stack);
if (ctx.error)
{

View File

@ -23,6 +23,8 @@
#include "vkd3d_shader_private.h"
#include "preproc.h"
#include <stdio.h>
#include <sys/stat.h>
#define PREPROC_YYLTYPE struct vkd3d_shader_location
@ -105,18 +107,24 @@ void preproc_free_macro(struct preproc_macro *macro)
static bool preproc_was_writing(struct preproc_ctx *ctx)
{
if (ctx->if_count < 2)
const struct preproc_file *file = preproc_get_top_file(ctx);
/* This applies across files, since we can't #include anyway if we weren't
* writing. */
if (file->if_count < 2)
return true;
return ctx->if_stack[ctx->if_count - 2].current_true;
return file->if_stack[file->if_count - 2].current_true;
}
static bool preproc_push_if(struct preproc_ctx *ctx, bool condition)
{
struct preproc_file *file = preproc_get_top_file(ctx);
struct preproc_if_state *state;
if (!vkd3d_array_reserve((void **)&ctx->if_stack, &ctx->if_stack_size, ctx->if_count + 1, sizeof(*ctx->if_stack)))
if (!vkd3d_array_reserve((void **)&file->if_stack, &file->if_stack_size,
file->if_count + 1, sizeof(*file->if_stack)))
return false;
state = &ctx->if_stack[ctx->if_count++];
state = &file->if_stack[file->if_count++];
state->current_true = condition && preproc_was_writing(ctx);
state->seen_true = condition;
state->seen_else = false;
@ -155,6 +163,93 @@ static uint32_t preproc_parse_integer(const char *s)
return ret;
}
static int default_open_include(const char *filename, bool local,
const char *parent_data, void *context, struct vkd3d_shader_code *out)
{
uint8_t *data, *new_data;
size_t size = 4096;
struct stat st;
size_t pos = 0;
size_t ret;
FILE *f;
if (!(f = fopen(filename, "rb")))
{
ERR("Unable to open %s for reading.\n", debugstr_a(filename));
return VKD3D_ERROR;
}
if (fstat(fileno(f), &st) == -1)
{
ERR("Could not stat file %s.\n", debugstr_a(filename));
fclose(f);
return VKD3D_ERROR;
}
if (S_ISREG(st.st_mode))
size = st.st_size;
if (!(data = vkd3d_malloc(size)))
{
fclose(f);
return VKD3D_ERROR_OUT_OF_MEMORY;
}
for (;;)
{
if (pos >= size)
{
if (size > SIZE_MAX / 2 || !(new_data = vkd3d_realloc(data, size * 2)))
{
vkd3d_free(data);
fclose(f);
return VKD3D_ERROR_OUT_OF_MEMORY;
}
data = new_data;
size *= 2;
}
if (!(ret = fread(&data[pos], 1, size - pos, f)))
break;
pos += ret;
}
if (!feof(f))
{
vkd3d_free(data);
return VKD3D_ERROR;
}
fclose(f);
out->code = data;
out->size = pos;
return VKD3D_OK;
}
static void default_close_include(const struct vkd3d_shader_code *code, void *context)
{
vkd3d_free((void *)code->code);
}
void preproc_close_include(struct preproc_ctx *ctx, const struct vkd3d_shader_code *code)
{
PFN_vkd3d_shader_close_include close_include = ctx->preprocess_info->pfn_close_include;
if (!close_include)
close_include = default_close_include;
close_include(code, ctx->preprocess_info->include_context);
}
static const void *get_parent_data(struct preproc_ctx *ctx)
{
if (ctx->file_count == 1)
return NULL;
return preproc_get_top_file(ctx)->code.code;
}
}
%define api.prefix {preproc_yy}
@ -174,6 +269,7 @@ static uint32_t preproc_parse_integer(const char *s)
%token <string> T_IDENTIFIER
%token <string> T_INTEGER
%token <string> T_STRING
%token <string> T_TEXT
%token T_NEWLINE
@ -185,6 +281,7 @@ static uint32_t preproc_parse_integer(const char *s)
%token T_IF "#if"
%token T_IFDEF "#ifdef"
%token T_IFNDEF "#ifndef"
%token T_INCLUDE "#include"
%type <integer> expr
%type <string> body_token
@ -233,9 +330,11 @@ directive
}
| T_ELIF expr T_NEWLINE
{
if (ctx->if_count)
const struct preproc_file *file = preproc_get_top_file(ctx);
if (file->if_count)
{
struct preproc_if_state *state = &ctx->if_stack[ctx->if_count - 1];
struct preproc_if_state *state = &file->if_stack[file->if_count - 1];
if (state->seen_else)
{
@ -255,9 +354,11 @@ directive
}
| T_ELSE T_NEWLINE
{
if (ctx->if_count)
const struct preproc_file *file = preproc_get_top_file(ctx);
if (file->if_count)
{
struct preproc_if_state *state = &ctx->if_stack[ctx->if_count - 1];
struct preproc_if_state *state = &file->if_stack[file->if_count - 1];
if (state->seen_else)
{
@ -277,12 +378,46 @@ directive
}
| T_ENDIF T_NEWLINE
{
if (ctx->if_count)
--ctx->if_count;
struct preproc_file *file = preproc_get_top_file(ctx);
if (file->if_count)
--file->if_count;
else
preproc_warning(ctx, &@$, VKD3D_SHADER_WARNING_PP_INVALID_DIRECTIVE,
"Ignoring #endif without prior #if.");
}
| T_INCLUDE T_STRING T_NEWLINE
{
PFN_vkd3d_shader_open_include open_include = ctx->preprocess_info->pfn_open_include;
struct vkd3d_shader_code code;
char *filename;
int result;
if (!(filename = vkd3d_malloc(strlen($2) - 1)))
YYABORT;
if (!open_include)
open_include = default_open_include;
memcpy(filename, $2 + 1, strlen($2) - 2);
filename[strlen($2) - 2] = 0;
if (!(result = open_include(filename, $2[0] == '"', get_parent_data(ctx),
ctx->preprocess_info->include_context, &code)))
{
if (!preproc_push_include(ctx, filename, &code))
{
preproc_close_include(ctx, &code);
vkd3d_free(filename);
}
}
else
{
preproc_error(ctx, &@$, VKD3D_SHADER_ERROR_PP_INCLUDE_FAILED, "Failed to open %s.", $2);
vkd3d_free(filename);
}
vkd3d_free($2);
}
expr
: T_INTEGER

View File

@ -81,6 +81,7 @@ enum vkd3d_shader_error
VKD3D_SHADER_ERROR_RS_MIXED_DESCRIPTOR_RANGE_TYPES = 3004,
VKD3D_SHADER_ERROR_PP_INVALID_SYNTAX = 4000,
VKD3D_SHADER_ERROR_PP_INCLUDE_FAILED = 4002,
VKD3D_SHADER_WARNING_PP_ALREADY_DEFINED = 4300,
VKD3D_SHADER_WARNING_PP_INVALID_DIRECTIVE = 4301,

View File

@ -408,19 +408,17 @@ static void test_preprocess(void)
todo ok(include_count_file2 == 2, "file2 was included %u times.\n", include_count_file2);
/* Macro invocation spread across multiple files. */
if (0)
todo check_preprocess(test_include2_top, NULL, &test_include, "pass", NULL);
blob = errors = (ID3D10Blob *)0xdeadbeef;
hr = D3DPreprocess(test_include_top, strlen(test_include_top), NULL, NULL, &test_include_fail, &blob, &errors);
todo ok(hr == E_FAIL, "Got hr %#x.\n", hr);
todo ok(blob == (ID3D10Blob *)0xdeadbeef, "Expected no compiled shader blob.\n");
ok(hr == E_FAIL, "Got hr %#x.\n", hr);
ok(blob == (ID3D10Blob *)0xdeadbeef, "Expected no compiled shader blob.\n");
ok(!!errors, "Expected non-NULL error blob.\n");
if (errors)
{
if (vkd3d_test_state.debug_level)
trace("%s\n", (char *)ID3D10Blob_GetBufferPointer(errors));
ID3D10Blob_Release(errors);
}
}
START_TEST(hlsl_d3d12)