vkd3d-shader: Implement an initial pass-through HLSL preprocessor.

Signed-off-by: Zebediah Figura <zfigura@codeweavers.com>
Signed-off-by: Henri Verbeet <hverbeet@codeweavers.com>
Signed-off-by: Alexandre Julliard <julliard@winehq.org>

.gitignore

@@ -19,7 +19,10 @@ vkd3d-*.tar.xz
 *.log
 *.o
 *.pc
+*.tab.c
+*.tab.h
 *.trs
+*.yy.c
 *~
 
 .deps

Makefile.am

@@ -108,6 +108,33 @@ libvkd3d_common_la_SOURCES = \
 
 lib_LTLIBRARIES = libvkd3d-shader.la libvkd3d.la libvkd3d-utils.la
 
+VKD3D_V_FLEX = $(vkd3d_v_flex_@AM_V@)
+vkd3d_v_flex_ = $(vkd3d_v_flex_@AM_DEFAULT_V@)
+vkd3d_v_flex_0 = @echo "  FLEX    " $@;
+vkd3d_v_flex_1 =
+
+VKD3D_V_BISON = $(vkd3d_v_bison_@AM_V@)
+vkd3d_v_bison_ = $(vkd3d_v_bison_@AM_DEFAULT_V@)
+vkd3d_v_bison_0 = @echo "  BISON   " $@;
+vkd3d_v_bison_1 =
+
+libs/vkd3d-shader/preproc.yy.c: libs/vkd3d-shader/preproc.l
+	$(VKD3D_V_FLEX)$(FLEX) $(LFLAGS) -o $@ $<
+
+libs/vkd3d-shader/preproc.tab.c libs/vkd3d-shader/preproc.tab.h &: libs/vkd3d-shader/preproc.y
+	$(VKD3D_V_BISON)$(BISON) $(YFLAGS) -d -o libs/vkd3d-shader/preproc.tab.c $<
+
+BUILT_SOURCES += libs/vkd3d-shader/preproc.tab.h
+
+vkd3d_shader_yyfiles = \
+	libs/vkd3d-shader/preproc.tab.c \
+	libs/vkd3d-shader/preproc.tab.h \
+	libs/vkd3d-shader/preproc.yy.c
+
+CLEANFILES = $(vkd3d_shader_yyfiles)
+
+nodist_libvkd3d_shader_la_SOURCES = $(vkd3d_shader_yyfiles)
+
 libvkd3d_shader_la_SOURCES = \
 	include/private/list.h \
 	include/private/rbtree.h \
@@ -122,7 +149,7 @@ libvkd3d_shader_la_SOURCES = \
 	libs/vkd3d-shader/vkd3d_shader.map \
 	libs/vkd3d-shader/vkd3d_shader_main.c \
 	libs/vkd3d-shader/vkd3d_shader_private.h
-libvkd3d_shader_la_CFLAGS = $(AM_CFLAGS) @SPIRV_TOOLS_CFLAGS@
+libvkd3d_shader_la_CFLAGS = $(AM_CFLAGS) -I$(srcdir)/libs/vkd3d-shader @SPIRV_TOOLS_CFLAGS@
 libvkd3d_shader_la_LDFLAGS = $(AM_LDFLAGS) -version-info 1:0:0
 libvkd3d_shader_la_LIBADD = libvkd3d-common.la @SPIRV_TOOLS_LIBS@
 if HAVE_LD_VERSION_SCRIPT
@@ -173,7 +200,7 @@ EXTRA_DIST = ANNOUNCE LICENSE
 pkgconfigdir = $(libdir)/pkgconfig
 pkginclude_HEADERS = $(vkd3d_public_headers)
 nodist_pkgconfig_DATA = libvkd3d.pc libvkd3d-shader.pc libvkd3d-utils.pc
-CLEANFILES = libvkd3d.pc libvkd3d-shader.pc libvkd3d-utils.pc
+CLEANFILES += libvkd3d.pc libvkd3d-shader.pc libvkd3d-utils.pc
 EXTRA_DIST += \
 	libs/vkd3d/libvkd3d.pc.in \
 	libs/vkd3d-shader/libvkd3d-shader.pc.in \

configure.ac

@@ -30,6 +30,12 @@ AC_PROG_MKDIR_P
 VKD3D_PROG_WIDL(3, 20)
 AS_IF([test "x$WIDL" = "xno"], [AC_MSG_WARN([widl is required to build header files.])])
 
+AC_CHECK_PROGS([FLEX], [flex], [none])
+AS_IF([test "$FLEX" = "none"], [AC_MSG_ERROR([no suitable flex found. Please install the 'flex' package.])])
+
+AC_CHECK_PROGS([BISON], [bison], [none])
+AS_IF([test "$BISON" = "none"], [AC_MSG_ERROR([no suitable bison found. Please install the 'bison' package.])])
+
 DX_PS_FEATURE([OFF])
 DX_INIT_DOXYGEN([vkd3d], [Doxyfile], [doc])
 AC_CONFIG_FILES([Doxyfile])

libs/vkd3d-shader/preproc.h

@@ -0,0 +1,33 @@
+/*
+ * HLSL preprocessor
+ *
+ * Copyright 2020 Zebediah Figura for CodeWeavers
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA
+ */
+
+#ifndef __VKD3D_SHADER_PREPROC_H
+#define __VKD3D_SHADER_PREPROC_H
+
+#include "vkd3d_shader_private.h"
+
+struct preproc_ctx
+{
+    void *scanner;
+
+    struct vkd3d_string_buffer buffer;
+};
+
+#endif

libs/vkd3d-shader/preproc.l

@@ -0,0 +1,99 @@
+/*
+ * HLSL preprocessor
+ *
+ * Copyright 2020 Zebediah Figura for CodeWeavers
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA
+ */
+
+%{
+
+#include "preproc.tab.h"
+
+#define YYSTYPE PREPROC_YYSTYPE
+#define YYLTYPE PREPROC_YYLTYPE
+
+#define YY_DECL static int preproc_lexer_lex(YYSTYPE *yylval_param, YYLTYPE *yylloc_param, yyscan_t yyscanner)
+
+%}
+
+%option 8bit
+%option bison-bridge
+%option bison-locations
+%option extra-type="struct preproc_ctx *"
+%option never-interactive
+%option noinput
+%option nounput
+%option noyywrap
+%option prefix="preproc_yy"
+%option reentrant
+
+WS              [ \t]
+
+%%
+
+{WS}+                               {}
+.                                   {return T_TEXT;}
+
+%%
+
+int yylex(YYSTYPE *lval, YYLTYPE *lloc, yyscan_t scanner)
+{
+    struct preproc_ctx *ctx = yyget_extra(scanner);
+
+    for (;;)
+    {
+        const char *text;
+        int token;
+
+        if (!(token = preproc_lexer_lex(lval, lloc, scanner)))
+            return 0;
+        text = yyget_text(scanner);
+
+        TRACE("Parsing token %d, string %s.\n", token, debugstr_a(text));
+
+        vkd3d_string_buffer_printf(&ctx->buffer, "%s ", text);
+    }
+}
+
+int preproc_lexer_parse(const struct vkd3d_shader_compile_info *compile_info,
+        struct vkd3d_shader_code *out, struct vkd3d_shader_message_context *message_context)
+{
+    struct preproc_ctx ctx = {0};
+    YY_BUFFER_STATE top_buffer;
+    void *output_code;
+
+    vkd3d_string_buffer_init(&ctx.buffer);
+
+    yylex_init_extra(&ctx, &ctx.scanner);
+    top_buffer = yy_scan_bytes(compile_info->source.code, compile_info->source.size, ctx.scanner);
+
+    preproc_yyparse(ctx.scanner, &ctx);
+
+    yy_delete_buffer(top_buffer, ctx.scanner);
+    yylex_destroy(ctx.scanner);
+
+    if (!(output_code = vkd3d_malloc(ctx.buffer.content_size)))
+    {
+        vkd3d_string_buffer_cleanup(&ctx.buffer);
+        return VKD3D_ERROR_OUT_OF_MEMORY;
+    }
+    memcpy(output_code, ctx.buffer.buffer, ctx.buffer.content_size);
+    out->size = ctx.buffer.content_size;
+    out->code = output_code;
+    vkd3d_string_buffer_trace(&ctx.buffer);
+    vkd3d_string_buffer_cleanup(&ctx.buffer);
+    return VKD3D_OK;
+}

libs/vkd3d-shader/preproc.y

@@ -0,0 +1,60 @@
+/*
+ * HLSL preprocessor
+ *
+ * Copyright 2020 Zebediah Figura for CodeWeavers
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA
+ */
+
+%code requires
+{
+
+#include "vkd3d_shader_private.h"
+#include "preproc.h"
+
+}
+
+%code provides
+{
+
+int preproc_yylex(PREPROC_YYSTYPE *yylval_param, PREPROC_YYLTYPE *yylloc_param, void *scanner);
+
+}
+
+%code
+{
+
+static void yyerror(const YYLTYPE *loc, void *scanner, struct preproc_ctx *ctx, const char *string)
+{
+    FIXME("Error reporting is not implemented.\n");
+}
+
+}
+
+%define api.prefix {preproc_yy}
+%define api.pure full
+%define parse.error verbose
+%expect 0
+%locations
+%lex-param {yyscan_t scanner}
+%parse-param {void *scanner}
+%parse-param {struct preproc_ctx *ctx}
+
+%token T_TEXT
+
+%%
+
+shader_text
+    : %empty

libs/vkd3d-shader/vkd3d_shader_main.c

@@ -78,8 +78,7 @@ int vkd3d_string_buffer_vprintf(struct vkd3d_string_buffer *buffer, const char *
     }
 }
 
-static int VKD3D_PRINTF_FUNC(2, 3) vkd3d_string_buffer_printf(struct vkd3d_string_buffer *buffer,
+int vkd3d_string_buffer_printf(struct vkd3d_string_buffer *buffer, const char *format, ...)
-        const char *format, ...)
 {
     va_list args;
     int ret;
@@ -91,7 +90,7 @@ static int VKD3D_PRINTF_FUNC(2, 3) vkd3d_string_buffer_printf(struct vkd3d_strin
     return ret;
 }
 
-static void vkd3d_string_buffer_trace_(const struct vkd3d_string_buffer *buffer, const char *function)
+void vkd3d_string_buffer_trace_(const struct vkd3d_string_buffer *buffer, const char *function)
 {
     const char *p, *q, *end = buffer->buffer + buffer->content_size;
 
@@ -1154,6 +1153,7 @@ const enum vkd3d_shader_target_type *vkd3d_shader_get_supported_target_types(
 int vkd3d_shader_preprocess(const struct vkd3d_shader_compile_info *compile_info,
         struct vkd3d_shader_code *out, char **messages)
 {
+    struct vkd3d_shader_message_context message_context;
     int ret;
 
     TRACE("compile_info %p, out %p, messages %p.\n", compile_info, out, messages);
@@ -1164,5 +1164,13 @@ int vkd3d_shader_preprocess(const struct vkd3d_shader_compile_info *compile_info
     if ((ret = vkd3d_shader_validate_compile_info(compile_info, false)) < 0)
         return ret;
 
-    return VKD3D_ERROR_NOT_IMPLEMENTED;
+    vkd3d_shader_message_context_init(&message_context, compile_info->log_level, compile_info->source_name);
+
+    ret = preproc_lexer_parse(compile_info, out, &message_context);
+
+    vkd3d_shader_message_context_trace_messages(&message_context);
+    if (!vkd3d_shader_message_context_copy_messages(&message_context, messages))
+        ret = VKD3D_ERROR_OUT_OF_MEMORY;
+    vkd3d_shader_message_context_cleanup(&message_context);
+    return ret;
 }

libs/vkd3d-shader/vkd3d_shader_private.h

@@ -837,6 +837,11 @@ struct vkd3d_string_buffer
 enum vkd3d_result vkd3d_dxbc_binary_to_text(void *data, struct vkd3d_shader_code *out) DECLSPEC_HIDDEN;
 void vkd3d_string_buffer_cleanup(struct vkd3d_string_buffer *buffer) DECLSPEC_HIDDEN;
 void vkd3d_string_buffer_init(struct vkd3d_string_buffer *buffer) DECLSPEC_HIDDEN;
+int vkd3d_string_buffer_printf(struct vkd3d_string_buffer *buffer,
+        const char *format, ...) VKD3D_PRINTF_FUNC(2, 3) DECLSPEC_HIDDEN;
+#define vkd3d_string_buffer_trace(buffer) \
+        vkd3d_string_buffer_trace_(buffer, __FUNCTION__)
+void vkd3d_string_buffer_trace_(const struct vkd3d_string_buffer *buffer, const char *function) DECLSPEC_HIDDEN;
 int vkd3d_string_buffer_vprintf(struct vkd3d_string_buffer *buffer, const char *format, va_list args) DECLSPEC_HIDDEN;
 
 struct vkd3d_shader_message_context
@@ -882,6 +887,9 @@ void vkd3d_dxbc_compiler_destroy(struct vkd3d_dxbc_compiler *compiler) DECLSPEC_
 
 void vkd3d_compute_dxbc_checksum(const void *dxbc, size_t size, uint32_t checksum[4]) DECLSPEC_HIDDEN;
 
+int preproc_lexer_parse(const struct vkd3d_shader_compile_info *compile_info,
+        struct vkd3d_shader_code *out, struct vkd3d_shader_message_context *message_context) DECLSPEC_HIDDEN;
+
 static inline enum vkd3d_shader_component_type vkd3d_component_type_from_data_type(
         enum vkd3d_data_type data_type)
 {

tests/hlsl_d3d12.c

@@ -30,22 +30,20 @@ static void check_preprocess_(int line, const char *source, const D3D_SHADER_MAC
     HRESULT hr;
 
     hr = D3DPreprocess(source, strlen(source), NULL, macros, include, &blob, &errors);
-    todo ok_(line)(hr == S_OK, "Failed to preprocess shader, hr %#x.\n", hr);
+    assert_that_(line)(hr == S_OK, "Failed to preprocess shader, hr %#x.\n", hr);
     if (errors)
     {
         if (vkd3d_test_state.debug_level)
             trace_(line)("%s\n", (char *)ID3D10Blob_GetBufferPointer(errors));
         ID3D10Blob_Release(errors);
     }
-    if (hr != S_OK)
-        return;
     code = ID3D10Blob_GetBufferPointer(blob);
     size = ID3D10Blob_GetBufferSize(blob);
     if (present)
         ok_(line)(vkd3d_memmem(code, size, present, strlen(present)),
                 "\"%s\" not found in preprocessed shader.\n", present);
     if (absent)
-        ok_(line)(!vkd3d_memmem(code, size, absent, strlen(absent)),
+        assert_that_(line)(!vkd3d_memmem(code, size, absent, strlen(absent)),
                 "\"%s\" found in preprocessed shader.\n", absent);
     ID3D10Blob_Release(blob);
 }
@@ -352,6 +350,7 @@ static void test_preprocess(void)
     for (i = 0; i < ARRAY_SIZE(tests); ++i)
     {
         vkd3d_test_set_context("Source \"%s\"", tests[i].source);
+        todo_if (i != 5 && i != 8 && i != 42)
             check_preprocess(tests[i].source, NULL, NULL, tests[i].present, tests[i].absent);
     }
     vkd3d_test_set_context(NULL);
@@ -360,16 +359,16 @@ static void test_preprocess(void)
     macros[0].Definition = "value";
     macros[1].Name = NULL;
     macros[1].Definition = NULL;
-    check_preprocess("KEY", macros, NULL, "value", "KEY");
+    todo check_preprocess("KEY", macros, NULL, "value", "KEY");
 
-    check_preprocess("#undef KEY\nKEY", macros, NULL, "KEY", "value");
+    todo check_preprocess("#undef KEY\nKEY", macros, NULL, "KEY", "value");
 
     macros[0].Name = NULL;
-    check_preprocess("KEY", macros, NULL, "KEY", "value");
+    todo check_preprocess("KEY", macros, NULL, "KEY", "value");
 
     macros[0].Name = "KEY";
     macros[0].Definition = NULL;
-    check_preprocess("KEY", macros, NULL, NULL, "KEY");
+    todo check_preprocess("KEY", macros, NULL, NULL, "KEY");
 
     macros[0].Name = "0";
     macros[0].Definition = "value";
@@ -377,7 +376,7 @@ static void test_preprocess(void)
 
     macros[0].Name = "KEY(a)";
     macros[0].Definition = "value";
-    check_preprocess("KEY(a)", macros, NULL, "KEY", "value");
+    todo check_preprocess("KEY(a)", macros, NULL, "KEY", "value");
 
     macros[0].Name = "KEY";
     macros[0].Definition = "value1";
@@ -385,33 +384,33 @@ static void test_preprocess(void)
     macros[1].Definition = "value2";
     macros[2].Name = NULL;
     macros[2].Definition = NULL;
-    check_preprocess("KEY", macros, NULL, "value2", NULL);
+    todo check_preprocess("KEY", macros, NULL, "value2", NULL);
 
     macros[0].Name = "KEY";
     macros[0].Definition = "KEY2";
     macros[1].Name = "KEY2";
     macros[1].Definition = "value";
-    check_preprocess("KEY", macros, NULL, "value", NULL);
+    todo check_preprocess("KEY", macros, NULL, "value", NULL);
 
     macros[0].Name = "KEY2";
     macros[0].Definition = "value";
     macros[1].Name = "KEY";
     macros[1].Definition = "KEY2";
-    check_preprocess("KEY", macros, NULL, "value", NULL);
+    todo check_preprocess("KEY", macros, NULL, "value", NULL);
 
-    check_preprocess(test_include_top, NULL, &test_include, "pass", "fail");
+    todo check_preprocess(test_include_top, NULL, &test_include, "pass", "fail");
     ok(!refcount_file1, "Got %d references to file1.\n", refcount_file1);
     ok(!refcount_file2, "Got %d references to file1.\n", refcount_file2);
     ok(!refcount_file3, "Got %d references to file1.\n", refcount_file3);
     todo ok(include_count_file2 == 2, "file2 was included %u times.\n", include_count_file2);
 
     /* Macro invocation spread across multiple files. */
-    check_preprocess(test_include2_top, NULL, &test_include, "pass", NULL);
+    todo check_preprocess(test_include2_top, NULL, &test_include, "pass", NULL);
 
     blob = errors = (ID3D10Blob *)0xdeadbeef;
     hr = D3DPreprocess(test_include_top, strlen(test_include_top), NULL, NULL, &test_include_fail, &blob, &errors);
     todo ok(hr == E_FAIL, "Got hr %#x.\n", hr);
-    ok(blob == (ID3D10Blob *)0xdeadbeef, "Expected no compiled shader blob.\n");
+    todo ok(blob == (ID3D10Blob *)0xdeadbeef, "Expected no compiled shader blob.\n");
     todo ok(!!errors, "Expected non-NULL error blob.\n");
     if (errors)
     {