vkd3d-utils: Implement D3DStripShader().

Approved-by: Matteo Bruni (@Mystral)
Approved-by: Giovanni Mascellani (@giomasce)
Approved-by: Henri Verbeet (@hverbeet)
Approved-by: Alexandre Julliard (@julliard)
Merge-Request: https://gitlab.winehq.org/wine/vkd3d/-/merge_requests/420

include/private/vkd3d_common.h

@@ -65,6 +65,7 @@
 #define TAG_SDBG VKD3D_MAKE_TAG('S', 'D', 'B', 'G')
 #define TAG_SHDR VKD3D_MAKE_TAG('S', 'H', 'D', 'R')
 #define TAG_SHEX VKD3D_MAKE_TAG('S', 'H', 'E', 'X')
+#define TAG_STAT VKD3D_MAKE_TAG('S', 'T', 'A', 'T')
 #define TAG_TEXT VKD3D_MAKE_TAG('T', 'E', 'X', 'T')
 #define TAG_XNAP VKD3D_MAKE_TAG('X', 'N', 'A', 'P')
 #define TAG_XNAS VKD3D_MAKE_TAG('X', 'N', 'A', 'S')

include/vkd3d_d3dcompiler.h

@@ -79,6 +79,16 @@ typedef enum D3D_BLOB_PART
     D3D_BLOB_TEST_COMPILE_REPORT
 } D3D_BLOB_PART;
 
+typedef enum D3DCOMPILER_STRIP_FLAGS
+{
+    D3DCOMPILER_STRIP_REFLECTION_DATA = 0x00000001,
+    D3DCOMPILER_STRIP_DEBUG_INFO      = 0x00000002,
+    D3DCOMPILER_STRIP_TEST_BLOBS      = 0x00000004,
+    D3DCOMPILER_STRIP_PRIVATE_DATA    = 0x00000008,
+    D3DCOMPILER_STRIP_ROOT_SIGNATURE  = 0x00000010,
+    D3DCOMPILER_STRIP_FORCE_DWORD     = 0x7fffffff,
+} D3DCOMPILER_STRIP_FLAGS;
+
 HRESULT WINAPI D3DCompile(const void *data, SIZE_T data_size, const char *filename,
         const D3D_SHADER_MACRO *macros, ID3DInclude *include, const char *entrypoint,
         const char *profile, UINT flags, UINT effect_flags, ID3DBlob **shader, ID3DBlob **error_messages);
@@ -95,6 +105,7 @@ HRESULT WINAPI D3DGetInputSignatureBlob(const void *data, SIZE_T data_size, ID3D
 HRESULT WINAPI D3DGetOutputSignatureBlob(const void *data, SIZE_T data_size, ID3DBlob **blob);
 HRESULT WINAPI D3DPreprocess(const void *data, SIZE_T size, const char *filename, const D3D_SHADER_MACRO *macros,
         ID3DInclude *include, ID3DBlob **shader, ID3DBlob **error_messages);
+HRESULT WINAPI D3DStripShader(const void *data, SIZE_T data_size, UINT flags, ID3DBlob **blob);
 
 #endif /* __D3DCOMPILER_H__ */
 #endif /* __VKD3D_D3DCOMPILER_H */

include/vkd3d_utils.h

@@ -115,6 +115,8 @@ VKD3D_UTILS_API HRESULT WINAPI D3DGetInputAndOutputSignatureBlob(const void *dat
 VKD3D_UTILS_API HRESULT WINAPI D3DGetInputSignatureBlob(const void *data, SIZE_T data_size, ID3DBlob **blob);
 /** \since 1.10 */
 VKD3D_UTILS_API HRESULT WINAPI D3DGetOutputSignatureBlob(const void *data, SIZE_T data_size, ID3DBlob **blob);
+/** \since 1.10 */
+VKD3D_UTILS_API HRESULT WINAPI D3DStripShader(const void *data, SIZE_T data_size, UINT flags, ID3DBlob **blob);
 
 #ifdef __cplusplus
 }

libs/vkd3d-utils/vkd3d_utils.map

@@ -17,6 +17,7 @@ global:
     D3DGetInputSignatureBlob;
     D3DGetOutputSignatureBlob;
     D3DPreprocess;
+    D3DStripShader;
     vkd3d_create_event;
     vkd3d_destroy_event;
     vkd3d_signal_event;

libs/vkd3d-utils/vkd3d_utils_main.c

@@ -794,6 +794,100 @@ HRESULT WINAPI D3DGetOutputSignatureBlob(const void *data, SIZE_T data_size, ID3
     return get_blob_part(data, data_size, D3D_BLOB_OUTPUT_SIGNATURE_BLOB, 0, blob);
 }
 
+static bool check_blob_strip(uint32_t tag, uint32_t flags)
+{
+    bool add = true;
+
+    switch (tag)
+    {
+        case TAG_RDEF:
+        case TAG_STAT:
+            if (flags & D3DCOMPILER_STRIP_REFLECTION_DATA)
+                add = false;
+            break;
+
+        case TAG_SDBG:
+            if (flags & D3DCOMPILER_STRIP_DEBUG_INFO)
+                add = false;
+            break;
+
+        default:
+            break;
+    }
+
+    TRACE("%s tag %s.\n", add ? "Add" : "Skip", debugstr_an((const char *)&tag, 4));
+
+    return add;
+}
+
+HRESULT WINAPI D3DStripShader(const void *data, SIZE_T data_size, UINT flags, ID3DBlob **blob)
+{
+    const struct vkd3d_shader_code src_dxbc = {.code = data, .size = data_size};
+    struct vkd3d_shader_dxbc_section_desc *sections;
+    struct vkd3d_shader_dxbc_desc src_dxbc_desc;
+    struct vkd3d_shader_code dst_dxbc;
+    unsigned int section_count, i;
+    HRESULT hr;
+    int ret;
+
+    TRACE("data %p, data_size %lu, flags %#x, blob %p.\n", data, data_size, flags, blob);
+
+    if (!blob)
+    {
+        WARN("Invalid 'blob' pointer specified.\n");
+        return E_FAIL;
+    }
+
+    if (!data || !data_size)
+    {
+        WARN("Invalid arguments: data %p, data_size %lu.\n", data, data_size);
+        return D3DERR_INVALIDCALL;
+    }
+
+    if ((ret = vkd3d_shader_parse_dxbc(&src_dxbc, 0, &src_dxbc_desc, NULL)) < 0)
+    {
+        WARN("Failed to parse source data, ret %d.\n", ret);
+        return D3DERR_INVALIDCALL;
+    }
+
+    if (!(sections = vkd3d_calloc(src_dxbc_desc.section_count, sizeof(*sections))))
+    {
+        ERR("Failed to allocate sections memory.\n");
+        vkd3d_shader_free_dxbc(&src_dxbc_desc);
+        return E_OUTOFMEMORY;
+    }
+
+    if (flags & ~(D3DCOMPILER_STRIP_REFLECTION_DATA | D3DCOMPILER_STRIP_DEBUG_INFO))
+        FIXME("Unhandled flags %#x.\n", flags);
+
+    for (i = 0, section_count = 0; i < src_dxbc_desc.section_count; ++i)
+    {
+        const struct vkd3d_shader_dxbc_section_desc *src_section = &src_dxbc_desc.sections[i];
+
+        if (check_blob_strip(src_section->tag, flags))
+            sections[section_count++] = *src_section;
+    }
+
+    if ((ret = vkd3d_shader_serialize_dxbc(section_count, sections, &dst_dxbc, NULL) < 0))
+    {
+        WARN("Failed to serialise DXBC, ret %d.\n", ret);
+        hr = hresult_from_vkd3d_result(ret);
+        goto done;
+    }
+
+    if (FAILED(hr = D3DCreateBlob(dst_dxbc.size, blob)))
+        WARN("Failed to create blob, hr %#x.\n", hr);
+    else
+        memcpy(ID3D10Blob_GetBufferPointer(*blob), dst_dxbc.code, dst_dxbc.size);
+    vkd3d_shader_free_shader_code(&dst_dxbc);
+
+done:
+    vkd3d_free(sections);
+    vkd3d_shader_free_dxbc(&src_dxbc_desc);
+
+    return hr;
+}
+
 void vkd3d_utils_set_log_callback(PFN_vkd3d_log callback)
 {
     vkd3d_set_log_callback(callback);

tests/hlsl_d3d12.c

@@ -1095,6 +1095,80 @@ static void test_get_blob_part(void)
     refcount = ID3D10Blob_Release(blob);
     ok(!refcount, "Got refcount %u.\n", refcount);
 
+    /* D3DStripShader() corner cases. */
+    hr = D3DStripShader(test_blob_part, test_blob_part[6], 0xffffffff, &blob);
+    ok(hr == S_OK, "Got hr %#x.\n", hr);
+
+    refcount = ID3D10Blob_Release(blob);
+    ok(!refcount, "Got refcount %u.\n", refcount);
+
+    hr = D3DStripShader(test_blob_part, test_blob_part[6], 0, &blob);
+    ok(hr == S_OK, "Got hr %#x.\n", hr);
+
+    refcount = ID3D10Blob_Release(blob);
+    ok(!refcount, "Got refcount %u.\n", refcount);
+
+    hr = D3DStripShader(NULL, test_blob_part[6], 0, &blob);
+    ok(hr == D3DERR_INVALIDCALL, "Got hr %#x.\n", hr);
+
+    hr = D3DStripShader(test_blob_part, 7 * sizeof(DWORD), 0, &blob);
+    ok(hr == D3DERR_INVALIDCALL, "Got hr %#x.\n", hr);
+
+    hr = D3DStripShader(test_blob_part, 8 * sizeof(DWORD), 0, &blob);
+    ok(hr == D3DERR_INVALIDCALL, "Got hr %#x.\n", hr);
+
+    hr = D3DStripShader(test_blob_part, test_blob_part[6], 0, NULL);
+    ok(hr == E_FAIL, "Got hr %#x.\n", hr);
+
+    hr = D3DStripShader(NULL, test_blob_part[6], 0, NULL);
+    ok(hr == E_FAIL, "Got hr %#x.\n", hr);
+
+    hr = D3DStripShader(test_blob_part, 0, 0, NULL);
+    ok(hr == E_FAIL, "Got hr %#x.\n", hr);
+
+    /* D3DCOMPILER_STRIP_DEBUG_INFO */
+    hr = D3DStripShader(test_blob_part, test_blob_part[6], D3DCOMPILER_STRIP_DEBUG_INFO, &blob);
+    ok(hr == S_OK, "Got hr %#x.\n", hr);
+
+    size = ID3D10Blob_GetBufferSize(blob);
+    ok(size == 736, "Got size %u.\n", size);
+
+    u32 = ID3D10Blob_GetBufferPointer(blob);
+    ok(u32[0] == TAG_DXBC, "Got u32[0] 0x%08x, expected 0x%08x.\n", u32[0], TAG_DXBC);
+    ok(u32[16] == TAG_XNAS, "Got u32[16] 0x%08x, expected 0x%08x.\n", u32[16], TAG_XNAS);
+    ok(u32[35] == TAG_XNAP, "Got u32[35] 0x%08x, expected 0x%08x.\n", u32[35], TAG_XNAP);
+    ok(u32[54] == TAG_AON9, "Got u32[54] 0x%08x, expected 0x%08x.\n", u32[54], TAG_AON9);
+    ok(u32[79] == TAG_SHDR, "Got u32[79] 0x%08x, expected 0x%08x.\n", u32[79], TAG_SHDR);
+    ok(u32[96] == TAG_STAT, "Got u32[96] 0x%08x, expected 0x%08x.\n", u32[96], TAG_STAT);
+    ok(u32[127] == TAG_RDEF, "Got u32[127] 0x%08x, expected 0x%08x.\n", u32[127], TAG_RDEF);
+    ok(u32[149] == TAG_ISGN, "Got u32[149] 0x%08x, expected 0x%08x.\n", u32[149], TAG_ISGN);
+    ok(u32[171] == TAG_OSGN, "Got u32[171] 0x%08x, expected 0x%08x.\n", u32[171], TAG_OSGN);
+
+    hr = D3DGetBlobPart(u32, size, D3D_BLOB_DEBUG_INFO, 0, &blob2);
+    ok(hr == E_FAIL, "Got hr %#x.\n", hr);
+
+    refcount = ID3D10Blob_Release(blob);
+    ok(!refcount, "Got refcount %u.\n", refcount);
+
+    /* D3DCOMPILER_STRIP_REFLECTION_DATA */
+    hr = D3DStripShader(test_blob_part, test_blob_part[6], D3DCOMPILER_STRIP_REFLECTION_DATA, &blob);
+    ok(hr == S_OK, "Got hr %#x.\n", hr);
+
+    size = ID3D10Blob_GetBufferSize(blob);
+    ok(size == 516, "Got size %u.\n", size);
+
+    u32 = ID3D10Blob_GetBufferPointer(blob);
+    ok(u32[0] == TAG_DXBC, "Got u32[0] 0x%08x, expected 0x%08x.\n", u32[0], TAG_DXBC);
+    ok(u32[14] == TAG_XNAS, "Got u32[14] 0x%08x, expected 0x%08x.\n", u32[14], TAG_XNAS);
+    ok(u32[33] == TAG_XNAP, "Got u32[33] 0x%08x, expected 0x%08x.\n", u32[33], TAG_XNAP);
+    ok(u32[52] == TAG_AON9, "Got u32[52] 0x%08x, expected 0x%08x.\n", u32[52], TAG_AON9);
+    ok(u32[77] == TAG_SHDR, "Got u32[77] 0x%08x, expected 0x%08x.\n", u32[77], TAG_SHDR);
+    ok(u32[94] == TAG_ISGN, "Got u32[94] 0x%08x, expected 0x%08x.\n", u32[94], TAG_ISGN);
+    ok(u32[116] == TAG_OSGN, "Got u32[116] 0x%08x, expected 0x%08x.\n", u32[116], TAG_OSGN);
+
+    refcount = ID3D10Blob_Release(blob);
+    ok(!refcount, "Got refcount %u.\n", refcount);
+
     /* D3D_BLOB_PATCH_CONSTANT_SIGNATURE_BLOB */
     hr = D3DGetBlobPart(test_blob_part2, test_blob_part2[6], D3D_BLOB_PATCH_CONSTANT_SIGNATURE_BLOB, 0, &blob);
     ok(hr == S_OK, "Got hr %#x.\n", hr);
@@ -1200,6 +1274,46 @@ static void test_get_blob_part(void)
     /* D3D_BLOB_XNA_SHADER */
     hr = D3DGetBlobPart(test_blob_part2, test_blob_part2[6], D3D_BLOB_XNA_SHADER, 0, &blob);
     ok(hr == E_FAIL, "Got hr %#x.\n", hr);
+
+    /* D3DCOMPILER_STRIP_DEBUG_INFO */
+    hr = D3DStripShader(test_blob_part2, test_blob_part2[6], D3DCOMPILER_STRIP_DEBUG_INFO, &blob);
+    ok(hr == S_OK, "Got hr %#x.\n", hr);
+
+    size = ID3D10Blob_GetBufferSize(blob);
+    ok(size == 952, "Got size %u.\n", size);
+
+    u32 = ID3D10Blob_GetBufferPointer(blob);
+    ok(u32[0] == TAG_DXBC, "Got u32[0] 0x%08x, expected 0x%08x.\n", u32[0], TAG_DXBC);
+    ok(u32[14] == TAG_RDEF, "Got u32[14] 0x%08x, expected 0x%08x.\n", u32[14], TAG_RDEF);
+    ok(u32[44] == TAG_ISGN, "Got u32[44] 0x%08x, expected 0x%08x.\n", u32[44], TAG_ISGN);
+    ok(u32[57] == TAG_OSGN, "Got u32[57] 0x%08x, expected 0x%08x.\n", u32[57], TAG_OSGN);
+    ok(u32[70] == TAG_PCSG, "Got u32[70] 0x%08x, expected 0x%08x.\n", u32[70], TAG_PCSG);
+    ok(u32[119] == TAG_SHEX, "Got u32[119] 0x%08x, expected 0x%08x.\n", u32[119], TAG_SHEX);
+    ok(u32[199] == TAG_STAT, "Got u32[199] 0x%08x, expected 0x%08x.\n", u32[199], TAG_STAT);
+
+    hr = D3DGetBlobPart(u32, size, D3D_BLOB_DEBUG_INFO, 0, &blob2);
+    ok(hr == E_FAIL, "Got hr %#x.\n", hr);
+
+    refcount = ID3D10Blob_Release(blob);
+    ok(!refcount, "Got refcount %u.\n", refcount);
+
+    /* D3DCOMPILER_STRIP_REFLECTION_DATA */
+    hr = D3DStripShader(test_blob_part2, test_blob_part2[6], D3DCOMPILER_STRIP_REFLECTION_DATA, &blob);
+    ok(hr == S_OK, "Got hr %#x.\n", hr);
+
+    size = ID3D10Blob_GetBufferSize(blob);
+    ok(size == 4735, "Got size %u.\n", size);
+
+    u32 = ID3D10Blob_GetBufferPointer(blob);
+    ok(u32[0] == TAG_DXBC, "Got u32[0] 0x%08x, expected 0x%08x.\n", u32[0], TAG_DXBC);
+    ok(u32[13] == TAG_ISGN, "Got u32[13] 0x%08x, expected 0x%08x.\n", u32[13], TAG_ISGN);
+    ok(u32[26] == TAG_OSGN, "Got u32[26] 0x%08x, expected 0x%08x.\n", u32[26], TAG_OSGN);
+    ok(u32[39] == TAG_PCSG, "Got u32[39] 0x%08x, expected 0x%08x.\n", u32[39], TAG_PCSG);
+    ok(u32[88] == TAG_SHEX, "Got u32[88] 0x%08x, expected 0x%08x.\n", u32[88], TAG_SHEX);
+    ok(u32[168] == TAG_SDBG, "Got u32[168] 0x%08x, expected 0x%08x.\n", u32[168], TAG_SDBG);
+
+    refcount = ID3D10Blob_Release(blob);
+    ok(!refcount, "Got refcount %u.\n", refcount);
 }
 
 START_TEST(hlsl_d3d12)