libs/vkd3d: Synchronize access to Vulkan queues.

Access to VkQueue must be externally synchronized.

Signed-off-by: Józef Kucia <jkucia@codeweavers.com>
Signed-off-by: Henri Verbeet <hverbeet@codeweavers.com>
Signed-off-by: Alexandre Julliard <julliard@winehq.org>

libs/vkd3d/command.c

@@ -19,6 +19,63 @@
 
 #include "vkd3d_private.h"
 
+HRESULT vkd3d_queue_create(struct d3d12_device *device,
+        uint32_t family_index, uint32_t timestamp_bits, struct vkd3d_queue **queue)
+{
+    const struct vkd3d_vk_device_procs *vk_procs = &device->vk_procs;
+    struct vkd3d_queue *object;
+    int rc;
+
+    if (!(object = vkd3d_malloc(sizeof(*object))))
+        return E_OUTOFMEMORY;
+
+    if ((rc = pthread_mutex_init(&object->mutex, NULL)))
+    {
+        ERR("Failed to initialize mutex, error %d.\n", rc);
+        vkd3d_free(object);
+        return E_FAIL;
+    }
+
+    object->vk_family_index = family_index;
+    object->timestamp_bits = timestamp_bits;
+
+    VK_CALL(vkGetDeviceQueue(device->vk_device, family_index, 0, &object->vk_queue));
+
+    TRACE("Created queue %p for queue family index %u.\n", object, family_index);
+
+    *queue = object;
+
+    return S_OK;
+}
+
+void vkd3d_queue_destroy(struct vkd3d_queue *queue)
+{
+    pthread_mutex_destroy(&queue->mutex);
+    vkd3d_free(queue);
+}
+
+static VkQueue vkd3d_queue_acquire(struct vkd3d_queue *queue)
+{
+    int rc;
+
+    TRACE("queue %p.\n", queue);
+
+    if ((rc = pthread_mutex_lock(&queue->mutex)))
+    {
+        ERR("Failed to lock mutex, error %d.\n", rc);
+        return VK_NULL_HANDLE;
+    }
+
+    return queue->vk_queue;
+}
+
+static void vkd3d_queue_release(struct vkd3d_queue *queue)
+{
+    TRACE("queue %p.\n", queue);
+
+    pthread_mutex_unlock(&queue->mutex);
+}
+
 /* Fence worker thread */
 static HRESULT vkd3d_queue_gpu_fence(struct vkd3d_fence_worker *worker,
         VkFence vk_fence, ID3D12Fence *fence, UINT64 value)
@@ -955,13 +1012,34 @@ static struct d3d12_command_allocator *unsafe_impl_from_ID3D12CommandAllocator(I
     return impl_from_ID3D12CommandAllocator(iface);
 }
 
+static struct vkd3d_queue *d3d12_device_get_vkd3d_queue(struct d3d12_device *device,
+        D3D12_COMMAND_LIST_TYPE type)
+{
+    switch (type)
+    {
+        case D3D12_COMMAND_LIST_TYPE_DIRECT:
+            return device->direct_queue;
+        case D3D12_COMMAND_LIST_TYPE_COMPUTE:
+            return device->compute_queue;
+        case D3D12_COMMAND_LIST_TYPE_COPY:
+            return device->copy_queue;
+        default:
+            FIXME("Unhandled command list type %#x.\n", type);
+            return NULL;
+    }
+}
+
 static HRESULT d3d12_command_allocator_init(struct d3d12_command_allocator *allocator,
         struct d3d12_device *device, D3D12_COMMAND_LIST_TYPE type)
 {
     const struct vkd3d_vk_device_procs *vk_procs = &device->vk_procs;
     VkCommandPoolCreateInfo command_pool_info;
+    struct vkd3d_queue *queue;
     VkResult vr;
 
+    if (!(queue = d3d12_device_get_vkd3d_queue(device, type)))
+        queue = device->direct_queue;
+
     allocator->ID3D12CommandAllocator_iface.lpVtbl = &d3d12_command_allocator_vtbl;
     allocator->refcount = 1;
 
@@ -970,22 +1048,7 @@ static HRESULT d3d12_command_allocator_init(struct d3d12_command_allocator *allo
     command_pool_info.sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO;
     command_pool_info.pNext = NULL;
     command_pool_info.flags = VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT;
-    switch (type)
+    command_pool_info.queueFamilyIndex = queue->vk_family_index;
-    {
-        case D3D12_COMMAND_LIST_TYPE_DIRECT:
-            command_pool_info.queueFamilyIndex = device->direct_queue_family_index;
-            break;
-        case D3D12_COMMAND_LIST_TYPE_COMPUTE:
-            command_pool_info.queueFamilyIndex = device->compute_queue_family_index;
-            break;
-        case D3D12_COMMAND_LIST_TYPE_COPY:
-            command_pool_info.queueFamilyIndex = device->copy_queue_family_index;
-            break;
-        default:
-            FIXME("Unhandled command list type %#x.\n", type);
-            command_pool_info.queueFamilyIndex = device->direct_queue_family_index;
-            break;
-    }
 
     if ((vr = VK_CALL(vkCreateCommandPool(device->vk_device, &command_pool_info, NULL,
             &allocator->vk_command_pool))) < 0)
@@ -4077,6 +4140,7 @@ static void STDMETHODCALLTYPE d3d12_command_queue_ExecuteCommandLists(ID3D12Comm
     const struct vkd3d_vk_device_procs *vk_procs;
     struct VkSubmitInfo submit_desc;
     VkCommandBuffer *buffers;
+    VkQueue vk_queue;
     unsigned int i;
     VkResult vr;
 
@@ -4106,10 +4170,17 @@ static void STDMETHODCALLTYPE d3d12_command_queue_ExecuteCommandLists(ID3D12Comm
     submit_desc.signalSemaphoreCount = 0;
     submit_desc.pSignalSemaphores = NULL;
 
-    if ((vr = VK_CALL(vkQueueSubmit(command_queue->vk_queue,
+    if (!(vk_queue = vkd3d_queue_acquire(command_queue->vkd3d_queue)))
-            1, &submit_desc, VK_NULL_HANDLE))) < 0)
+    {
+        ERR("Failed to acquire queue %p.\n", command_queue->vkd3d_queue);
+        return;
+    }
+
+    if ((vr = VK_CALL(vkQueueSubmit(vk_queue, 1, &submit_desc, VK_NULL_HANDLE))) < 0)
         ERR("Failed to submit queue(s), vr %d.\n", vr);
 
+    vkd3d_queue_release(command_queue->vkd3d_queue);
+
     vkd3d_free(buffers);
 }
 
@@ -4140,6 +4211,7 @@ static HRESULT STDMETHODCALLTYPE d3d12_command_queue_Signal(ID3D12CommandQueue *
     VkFenceCreateInfo fence_info;
     struct d3d12_device *device;
     VkFence vk_fence;
+    VkQueue vk_queue;
     VkResult vr;
 
     TRACE("iface %p, fence %p, value %#"PRIx64".\n", iface, fence, value);
@@ -4158,7 +4230,14 @@ static HRESULT STDMETHODCALLTYPE d3d12_command_queue_Signal(ID3D12CommandQueue *
         return hresult_from_vk_result(vr);
     }
 
-    if ((vr = VK_CALL(vkQueueSubmit(command_queue->vk_queue, 0, NULL, vk_fence))) < 0)
+    if (!(vk_queue = vkd3d_queue_acquire(command_queue->vkd3d_queue)))
+    {
+        ERR("Failed to acquire queue %p.\n", command_queue->vkd3d_queue);
+        return E_FAIL;
+    }
+    vr = VK_CALL(vkQueueSubmit(vk_queue, 0, NULL, vk_fence));
+    vkd3d_queue_release(command_queue->vkd3d_queue);
+    if (vr < 0)
     {
         WARN("Failed to submit fence, vr %d.\n", vr);
         VK_CALL(vkDestroyFence(device->vk_device, vk_fence, NULL));
@@ -4197,8 +4276,11 @@ static HRESULT STDMETHODCALLTYPE d3d12_command_queue_GetTimestampFrequency(ID3D1
 
     TRACE("iface %p, frequency %p.\n", iface, frequency);
 
-    if (command_queue->vk_queue_timestamp_bits == 0)
+    if (!command_queue->vkd3d_queue->timestamp_bits)
+    {
+        WARN("Timestamp queries not supported.\n");
         return E_FAIL;
+    }
 
     *frequency = 1000000000 / device->vk_info.device_limits.timestampPeriod;
 
@@ -4255,10 +4337,6 @@ static const struct ID3D12CommandQueueVtbl d3d12_command_queue_vtbl =
 static HRESULT d3d12_command_queue_init(struct d3d12_command_queue *queue,
         struct d3d12_device *device, const D3D12_COMMAND_QUEUE_DESC *desc)
 {
-    const struct vkd3d_vk_device_procs *vk_procs = &device->vk_procs;
-    unsigned int queue_family_index;
-    uint32_t timestamp_bits;
-
     queue->ID3D12CommandQueue_iface.lpVtbl = &d3d12_command_queue_vtbl;
     queue->refcount = 1;
 
@@ -4266,35 +4344,14 @@ static HRESULT d3d12_command_queue_init(struct d3d12_command_queue *queue,
     if (!queue->desc.NodeMask)
         queue->desc.NodeMask = 0x1;
 
-    switch (desc->Type)
+    if (!(queue->vkd3d_queue = d3d12_device_get_vkd3d_queue(device, desc->Type)))
-    {
+        return E_NOTIMPL;
-        case D3D12_COMMAND_LIST_TYPE_DIRECT:
-            queue_family_index = device->direct_queue_family_index;
-            timestamp_bits = device->direct_queue_timestamp_bits;
-            break;
-        case D3D12_COMMAND_LIST_TYPE_COPY:
-            queue_family_index = device->copy_queue_family_index;
-            timestamp_bits = device->copy_queue_timestamp_bits;
-            break;
-        case D3D12_COMMAND_LIST_TYPE_COMPUTE:
-            queue_family_index = device->compute_queue_family_index;
-            timestamp_bits = device->compute_queue_timestamp_bits;
-            break;
-        default:
-            FIXME("Unhandled command list type %#x.\n", desc->Type);
-            return E_NOTIMPL;
-    }
 
     if (desc->Priority)
         FIXME("Ignoring priority %#x.\n", desc->Priority);
     if (desc->Flags)
         FIXME("Ignoring flags %#x.\n", desc->Flags);
 
-    /* FIXME: Access to VkQueue must be externally synchronized. */
-    VK_CALL(vkGetDeviceQueue(device->vk_device, queue_family_index, 0, &queue->vk_queue));
-    queue->vk_queue_family_index = queue_family_index;
-    queue->vk_queue_timestamp_bits = timestamp_bits;
-
     queue->device = device;
     ID3D12Device_AddRef(&device->ID3D12Device_iface);
 
@@ -4327,14 +4384,14 @@ VkQueue vkd3d_get_vk_queue(ID3D12CommandQueue *queue)
 {
     struct d3d12_command_queue *d3d12_queue = impl_from_ID3D12CommandQueue(queue);
 
-    return d3d12_queue->vk_queue;
+    return d3d12_queue->vkd3d_queue->vk_queue;
 }
 
 uint32_t vkd3d_get_vk_queue_family_index(ID3D12CommandQueue *queue)
 {
     struct d3d12_command_queue *d3d12_queue = impl_from_ID3D12CommandQueue(queue);
 
-    return d3d12_queue->vk_queue_family_index;
+    return d3d12_queue->vkd3d_queue->vk_family_index;
 }
 
 /* ID3D12CommandSignature */

libs/vkd3d/device.c

@@ -684,6 +684,62 @@ static HRESULT vkd3d_select_physical_device(struct vkd3d_instance *instance,
     return S_OK;
 }
 
+static void d3d12_device_destroy_vkd3d_queues(struct d3d12_device *device)
+{
+    if (device->direct_queue)
+        vkd3d_queue_destroy(device->direct_queue);
+    if (device->compute_queue && device->compute_queue != device->direct_queue)
+        vkd3d_queue_destroy(device->compute_queue);
+    if (device->copy_queue && device->copy_queue != device->direct_queue
+            && device->copy_queue != device->compute_queue)
+        vkd3d_queue_destroy(device->copy_queue);
+
+    device->direct_queue = NULL;
+    device->compute_queue = NULL;
+    device->copy_queue = NULL;
+}
+
+static HRESULT d3d12_device_create_vkd3d_queues(struct d3d12_device *device,
+        uint32_t direct_queue_family_index, uint32_t direct_queue_timestamp_bits,
+        uint32_t compute_queue_family_index, uint32_t compute_queue_timestamp_bits,
+        uint32_t copy_queue_family_index, uint32_t copy_queue_timestamp_bits)
+{
+    HRESULT hr;
+
+    device->direct_queue = NULL;
+    device->compute_queue = NULL;
+    device->copy_queue = NULL;
+
+    if (FAILED((hr = vkd3d_queue_create(device, direct_queue_family_index,
+            direct_queue_timestamp_bits, &device->direct_queue))))
+    {
+        d3d12_device_destroy_vkd3d_queues(device);
+        return hr;
+    }
+
+    if (compute_queue_family_index == direct_queue_family_index)
+        device->compute_queue = device->direct_queue;
+    else if (FAILED(hr = vkd3d_queue_create(device, compute_queue_family_index,
+            compute_queue_timestamp_bits, &device->compute_queue)))
+    {
+        d3d12_device_destroy_vkd3d_queues(device);
+        return hr;
+    }
+
+    if (copy_queue_family_index == direct_queue_family_index)
+        device->copy_queue = device->direct_queue;
+    else if (copy_queue_family_index == compute_queue_family_index)
+        device->copy_queue = device->compute_queue;
+    else if (FAILED(hr = vkd3d_queue_create(device, copy_queue_family_index,
+            copy_queue_timestamp_bits, &device->copy_queue)))
+    {
+        d3d12_device_destroy_vkd3d_queues(device);
+        return hr;
+    }
+
+    return S_OK;
+}
+
 static HRESULT vkd3d_create_vk_device(struct d3d12_device *device)
 {
     unsigned int direct_queue_family_index, copy_queue_family_index, compute_queue_family_index;
@@ -776,12 +832,6 @@ static HRESULT vkd3d_create_vk_device(struct d3d12_device *device)
         compute_queue_timestamp_bits = direct_queue_timestamp_bits;
     }
 
-    device->direct_queue_family_index = direct_queue_family_index;
-    device->copy_queue_family_index = copy_queue_family_index;
-    device->compute_queue_family_index = compute_queue_family_index;
-    device->direct_queue_timestamp_bits = direct_queue_timestamp_bits;
-    device->copy_queue_timestamp_bits = copy_queue_timestamp_bits;
-    device->compute_queue_timestamp_bits = compute_queue_timestamp_bits;
     TRACE("Using queue family %u for direct command queues.\n", direct_queue_family_index);
     TRACE("Using queue family %u for copy command queues.\n", copy_queue_family_index);
     TRACE("Using queue family %u for compute command queues.\n", compute_queue_family_index);
@@ -824,6 +874,16 @@ static HRESULT vkd3d_create_vk_device(struct d3d12_device *device)
 
     device->vk_device = vk_device;
 
+    if (FAILED(hr = d3d12_device_create_vkd3d_queues(device,
+            direct_queue_family_index, direct_queue_timestamp_bits,
+            compute_queue_family_index, compute_queue_timestamp_bits,
+            copy_queue_family_index, copy_queue_timestamp_bits)))
+    {
+        ERR("Failed to create queues, hr %#x.\n", hr);
+        vkDestroyDevice(vk_device, NULL);
+        return hr;
+    }
+
     TRACE("Created Vulkan device %p.\n", vk_device);
 
     return S_OK;
@@ -982,6 +1042,7 @@ static ULONG STDMETHODCALLTYPE d3d12_device_Release(ID3D12Device *iface)
         VK_CALL(vkDestroySampler(device->vk_device, device->vk_default_sampler, NULL));
         if (device->vk_pipeline_cache)
             VK_CALL(vkDestroyPipelineCache(device->vk_device, device->vk_pipeline_cache, NULL));
+        d3d12_device_destroy_vkd3d_queues(device);
         VK_CALL(vkDestroyDevice(device->vk_device, NULL));
         vkd3d_instance_decref(device->vkd3d_instance);
 

libs/vkd3d/vkd3d_private.h

@@ -578,6 +578,19 @@ HRESULT d3d12_command_list_create(struct d3d12_device *device,
         UINT node_mask, D3D12_COMMAND_LIST_TYPE type, ID3D12CommandAllocator *allocator_iface,
         ID3D12PipelineState *initial_pipeline_state, struct d3d12_command_list **list) DECLSPEC_HIDDEN;
 
+struct vkd3d_queue
+{
+    /* Access to VkQueue must be externally synchronized. */
+    pthread_mutex_t mutex;
+    VkQueue vk_queue;
+    uint32_t vk_family_index;
+    uint32_t timestamp_bits;
+};
+
+HRESULT vkd3d_queue_create(struct d3d12_device *device,
+        uint32_t family_index, uint32_t timestamp_bits, struct vkd3d_queue **queue) DECLSPEC_HIDDEN;
+void vkd3d_queue_destroy(struct vkd3d_queue *queue) DECLSPEC_HIDDEN;
+
 /* ID3D12CommandQueue */
 struct d3d12_command_queue
 {
@@ -586,9 +599,7 @@ struct d3d12_command_queue
 
     D3D12_COMMAND_QUEUE_DESC desc;
 
-    VkQueue vk_queue;
+    struct vkd3d_queue *vkd3d_queue;
-    uint32_t vk_queue_family_index;
-    uint32_t vk_queue_timestamp_bits;
 
     struct d3d12_device *device;
 };
@@ -631,18 +642,16 @@ struct d3d12_device
     /* A sampler used for SpvOpImageFetch. */
     VkSampler vk_default_sampler;
 
-    unsigned int direct_queue_family_index;
-    unsigned int copy_queue_family_index;
-    unsigned int compute_queue_family_index;
     VkPhysicalDeviceMemoryProperties memory_properties;
-    uint32_t direct_queue_timestamp_bits;
-    uint32_t copy_queue_timestamp_bits;
-    uint32_t compute_queue_timestamp_bits;
 
     D3D12_FEATURE_DATA_D3D12_OPTIONS feature_options;
 
     struct vkd3d_vulkan_info vk_info;
 
+    struct vkd3d_queue *direct_queue;
+    struct vkd3d_queue *compute_queue;
+    struct vkd3d_queue *copy_queue;
+
     struct vkd3d_instance *vkd3d_instance;
 
     vkd3d_create_thread_pfn create_thread;