vkd3d-shader: Add separate binary target type for effects.

Signed-off-by: Nikolay Sivov <nsivov@codeweavers.com>

include/private/vkd3d_common.h

@@ -52,6 +52,7 @@
 #define TAG_AON9 VKD3D_MAKE_TAG('A', 'o', 'n', '9')
 #define TAG_DXBC VKD3D_MAKE_TAG('D', 'X', 'B', 'C')
 #define TAG_DXIL VKD3D_MAKE_TAG('D', 'X', 'I', 'L')
+#define TAG_FX10 VKD3D_MAKE_TAG('F', 'X', '1', '0')
 #define TAG_ISG1 VKD3D_MAKE_TAG('I', 'S', 'G', '1')
 #define TAG_ISGN VKD3D_MAKE_TAG('I', 'S', 'G', 'N')
 #define TAG_OSG1 VKD3D_MAKE_TAG('O', 'S', 'G', '1')

include/vkd3d_shader.h

@@ -785,6 +785,11 @@ enum vkd3d_shader_target_type
      * An 'OpenGL Shading Language' shader. \since 1.3
      */
     VKD3D_SHADER_TARGET_GLSL,
+    /**
+     * Binary format used by Direct3D 9/10.x/11 effects profiles.
+     * Output is a raw FX section without container. \since 1.11
+     */
+    VKD3D_SHADER_TARGET_FX,
 
     VKD3D_FORCE_32_BIT_ENUM(VKD3D_SHADER_TARGET_TYPE),
 };

libs/vkd3d-shader/hlsl.c

@@ -3631,7 +3631,13 @@ int hlsl_compile_shader(const struct vkd3d_shader_code *hlsl, const struct vkd3d
         return VKD3D_ERROR_NOT_IMPLEMENTED;
     }
 
-    if (compile_info->target_type == VKD3D_SHADER_TARGET_D3D_BYTECODE && profile->major_version > 3)
+    if (compile_info->target_type != VKD3D_SHADER_TARGET_FX && profile->type == VKD3D_SHADER_TYPE_EFFECT)
+    {
+        vkd3d_shader_error(message_context, NULL, VKD3D_SHADER_ERROR_HLSL_INCOMPATIBLE_PROFILE,
+                "The '%s' target profile is only compatible with the 'fx' target type.", profile->name);
+        return VKD3D_ERROR_INVALID_ARGUMENT;
+    }
+    else if (compile_info->target_type == VKD3D_SHADER_TARGET_D3D_BYTECODE && profile->major_version > 3)
     {
         vkd3d_shader_error(message_context, NULL, VKD3D_SHADER_ERROR_HLSL_INCOMPATIBLE_PROFILE,
                 "The '%s' target profile is incompatible with the 'd3dbc' target type.", profile->name);
@@ -3643,6 +3649,12 @@ int hlsl_compile_shader(const struct vkd3d_shader_code *hlsl, const struct vkd3d
                 "The '%s' target profile is incompatible with the 'dxbc-tpf' target type.", profile->name);
         return VKD3D_ERROR_INVALID_ARGUMENT;
     }
+    else if (compile_info->target_type == VKD3D_SHADER_TARGET_FX && profile->type != VKD3D_SHADER_TYPE_EFFECT)
+    {
+        vkd3d_shader_error(message_context, NULL, VKD3D_SHADER_ERROR_HLSL_INCOMPATIBLE_PROFILE,
+                "The '%s' target profile is incompatible with the 'fx' target type.", profile->name);
+        return VKD3D_ERROR_INVALID_ARGUMENT;
+    }
 
     if (!hlsl_ctx_init(&ctx, compile_info, profile, message_context))
         return VKD3D_ERROR_OUT_OF_MEMORY;

libs/vkd3d-shader/vkd3d_shader_main.c

@@ -1913,6 +1913,7 @@ const enum vkd3d_shader_target_type *vkd3d_shader_get_supported_target_types(
     {
         VKD3D_SHADER_TARGET_D3D_BYTECODE,
         VKD3D_SHADER_TARGET_DXBC_TPF,
+        VKD3D_SHADER_TARGET_FX,
     };
 
     static const enum vkd3d_shader_target_type d3dbc_types[] =

libs/vkd3d-utils/vkd3d_utils_main.c

@@ -185,28 +185,12 @@ static void close_include(const struct vkd3d_shader_code *code, void *context)
     ID3DInclude_Close(iface, code->code);
 }
 
-HRESULT WINAPI D3DCompile2(const void *data, SIZE_T data_size, const char *filename,
+static enum vkd3d_shader_target_type get_target_for_profile(const char *profile)
-        const D3D_SHADER_MACRO *macros, ID3DInclude *include, const char *entry_point,
-        const char *profile, UINT flags, UINT effect_flags, UINT secondary_flags,
-        const void *secondary_data, SIZE_T secondary_data_size, ID3DBlob **shader_blob,
-        ID3DBlob **messages_blob)
 {
-    struct vkd3d_shader_preprocess_info preprocess_info;
-    struct vkd3d_shader_hlsl_source_info hlsl_info;
-    struct vkd3d_shader_compile_option options[4];
-    struct vkd3d_shader_compile_info compile_info;
-    struct vkd3d_shader_compile_option *option;
-    struct vkd3d_shader_code byte_code;
-    const D3D_SHADER_MACRO *macro;
     size_t profile_len, i;
-    char *messages;
-    HRESULT hr;
-    int ret;
 
     static const char * const d3dbc_profiles[] =
     {
-        "fx_2_",
-
         "ps.1.",
         "ps.2.",
         "ps.3.",
@@ -226,6 +210,49 @@ HRESULT WINAPI D3DCompile2(const void *data, SIZE_T data_size, const char *filen
         "tx_1_",
     };
 
+    static const char * const fx_profiles[] =
+    {
+        "fx_2_0",
+        "fx_4_0",
+        "fx_4_1",
+        "fx_5_0",
+    };
+
+    profile_len = strlen(profile);
+    for (i = 0; i < ARRAY_SIZE(d3dbc_profiles); ++i)
+    {
+        size_t len = strlen(d3dbc_profiles[i]);
+
+        if (len <= profile_len && !memcmp(profile, d3dbc_profiles[i], len))
+            return VKD3D_SHADER_TARGET_D3D_BYTECODE;
+    }
+
+    for (i = 0; i < ARRAY_SIZE(fx_profiles); ++i)
+    {
+        if (!strcmp(profile, fx_profiles[i]))
+            return VKD3D_SHADER_TARGET_FX;
+    }
+
+    return VKD3D_SHADER_TARGET_DXBC_TPF;
+}
+
+HRESULT WINAPI D3DCompile2(const void *data, SIZE_T data_size, const char *filename,
+        const D3D_SHADER_MACRO *macros, ID3DInclude *include, const char *entry_point,
+        const char *profile, UINT flags, UINT effect_flags, UINT secondary_flags,
+        const void *secondary_data, SIZE_T secondary_data_size, ID3DBlob **shader_blob,
+        ID3DBlob **messages_blob)
+{
+    struct vkd3d_shader_preprocess_info preprocess_info;
+    struct vkd3d_shader_hlsl_source_info hlsl_info;
+    struct vkd3d_shader_compile_option options[4];
+    struct vkd3d_shader_compile_info compile_info;
+    struct vkd3d_shader_compile_option *option;
+    struct vkd3d_shader_code byte_code;
+    const D3D_SHADER_MACRO *macro;
+    char *messages;
+    HRESULT hr;
+    int ret;
+
     TRACE("data %p, data_size %lu, filename %s, macros %p, include %p, entry_point %s, "
             "profile %s, flags %#x, effect_flags %#x, secondary_flags %#x, secondary_data %p, "
             "secondary_data_size %lu, shader_blob %p, messages_blob %p.\n",
@@ -252,24 +279,12 @@ HRESULT WINAPI D3DCompile2(const void *data, SIZE_T data_size, const char *filen
     compile_info.source.code = data;
     compile_info.source.size = data_size;
     compile_info.source_type = VKD3D_SHADER_SOURCE_HLSL;
-    compile_info.target_type = VKD3D_SHADER_TARGET_DXBC_TPF;
+    compile_info.target_type = get_target_for_profile(profile);
     compile_info.options = options;
     compile_info.option_count = 1;
     compile_info.log_level = VKD3D_SHADER_LOG_INFO;
     compile_info.source_name = filename;
 
-    profile_len = strlen(profile);
-    for (i = 0; i < ARRAY_SIZE(d3dbc_profiles); ++i)
-    {
-        size_t len = strlen(d3dbc_profiles[i]);
-
-        if (len <= profile_len && !memcmp(profile, d3dbc_profiles[i], len))
-        {
-            compile_info.target_type = VKD3D_SHADER_TARGET_D3D_BYTECODE;
-            break;
-        }
-    }
-
     preprocess_info.type = VKD3D_SHADER_STRUCTURE_TYPE_PREPROCESS_INFO;
     preprocess_info.next = &hlsl_info;
     preprocess_info.macros = (const struct vkd3d_shader_macro *)macros;
@@ -331,6 +346,20 @@ HRESULT WINAPI D3DCompile2(const void *data, SIZE_T data_size, const char *filen
 
     if (!ret)
     {
+        /* Unlike other effect profiles fx_4_x is using DXBC container. */
+        if (!strcmp(profile, "fx_4_0") || !strcmp(profile, "fx_4_1"))
+        {
+            struct vkd3d_shader_dxbc_section_desc section = { .tag = TAG_FX10, .data = byte_code };
+            struct vkd3d_shader_code dxbc;
+
+            ret = vkd3d_shader_serialize_dxbc(1, &section, &dxbc, NULL);
+            vkd3d_shader_free_shader_code(&byte_code);
+            if (ret)
+                return hresult_from_vkd3d_result(ret);
+
+            byte_code = dxbc;
+        }
+
         if (FAILED(hr = vkd3d_blob_create((void *)byte_code.code, byte_code.size, shader_blob)))
         {
             vkd3d_shader_free_shader_code(&byte_code);

programs/vkd3d-compiler/main.c

@@ -106,6 +106,9 @@ target_type_info[] =
         "dxbc-tpf",     "A 'Tokenized Program Format' shader embedded in a DXBC container.\n"
         "                This is the format used for Direct3D shader model 4 and 5 shaders.\n",
         true},
+    {VKD3D_SHADER_TARGET_FX,
+        "fx",           "Binary format used by Direct3D 9/10.x/11 effects.\n",
+        true},
     {VKD3D_SHADER_TARGET_GLSL,
         "glsl", "An 'OpenGL Shading Language' shader.\n",
         false}