From 55f4f996ddec2b3df63139eb037b1ce220ad690b Mon Sep 17 00:00:00 2001 From: "Herman S." <429230+has207@users.noreply.github.com> Date: Mon, 29 Dec 2025 15:07:32 +0900 Subject: [PATCH] [Vulkan] Resolve downscale shader to use rotating descriptor pool chain Avoids mid-frame GPU stalls on pool exhaustion --- .../gpu/vulkan/vulkan_command_processor.cc | 131 +++++------- .../gpu/vulkan/vulkan_command_processor.h | 7 +- .../ui/vulkan/vulkan_descriptor_pool_chain.cc | 195 ++++++++++++++++++ .../ui/vulkan/vulkan_descriptor_pool_chain.h | 81 ++++++++ 4 files changed, 336 insertions(+), 78 deletions(-) create mode 100644 src/xenia/ui/vulkan/vulkan_descriptor_pool_chain.cc create mode 100644 src/xenia/ui/vulkan/vulkan_descriptor_pool_chain.h diff --git a/src/xenia/gpu/vulkan/vulkan_command_processor.cc b/src/xenia/gpu/vulkan/vulkan_command_processor.cc index a7190f072..bee2c9c10 100644 --- a/src/xenia/gpu/vulkan/vulkan_command_processor.cc +++ b/src/xenia/gpu/vulkan/vulkan_command_processor.cc @@ -1302,27 +1302,16 @@ bool VulkanCommandProcessor::SetupContext() { return false; } - // Descriptor pool for resolve downscale shader. - // Max 16 sets, each with 2 storage buffers (source + destination). + // Descriptor pool chain for resolve downscale shader. + // Uses pool chain to avoid mid-frame GPU stalls on pool exhaustion. + // Each pool has 64 sets with 2 storage buffers each. VkDescriptorPoolSize resolve_downscale_pool_size; resolve_downscale_pool_size.type = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER; - resolve_downscale_pool_size.descriptorCount = 32; // 16 sets * 2 buffers - VkDescriptorPoolCreateInfo resolve_downscale_pool_create_info; - resolve_downscale_pool_create_info.sType = - VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO; - resolve_downscale_pool_create_info.pNext = nullptr; - resolve_downscale_pool_create_info.flags = - VK_DESCRIPTOR_POOL_CREATE_FREE_DESCRIPTOR_SET_BIT; - resolve_downscale_pool_create_info.maxSets = 16; - resolve_downscale_pool_create_info.poolSizeCount = 1; - resolve_downscale_pool_create_info.pPoolSizes = - &resolve_downscale_pool_size; - if (dfn.vkCreateDescriptorPool( - device, &resolve_downscale_pool_create_info, nullptr, - &resolve_downscale_descriptor_pool_) != VK_SUCCESS) { - XELOGE("Failed to create the resolve downscale descriptor pool"); - return false; - } + resolve_downscale_pool_size.descriptorCount = 128; // 64 sets * 2 buffers + resolve_downscale_descriptor_pool_chain_ = + std::make_unique( + vulkan_device, 0, 64, &resolve_downscale_pool_size, 1, + resolve_downscale_descriptor_set_layout_); } occlusion_query_resources_available_ = InitializeOcclusionQueryResources(); @@ -1392,8 +1381,7 @@ void VulkanCommandProcessor::ShutdownContext() { ui::vulkan::util::DestroyAndNullHandle( dfn.vkDestroyDescriptorSetLayout, device, resolve_downscale_descriptor_set_layout_); - ui::vulkan::util::DestroyAndNullHandle(dfn.vkDestroyDescriptorPool, device, - resolve_downscale_descriptor_pool_); + resolve_downscale_descriptor_pool_chain_.reset(); ui::vulkan::util::DestroyAndNullHandle(dfn.vkDestroyDescriptorPool, device, swap_descriptor_pool_); @@ -4201,43 +4189,22 @@ bool VulkanCommandProcessor::IssueCopy() { resolve_downscale_buffer_size_ = downscale_buffer_size; } - // Allocate descriptor set for source and destination buffers - VkDescriptorSetAllocateInfo descriptor_alloc_info = {}; - descriptor_alloc_info.sType = - VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO; - descriptor_alloc_info.descriptorPool = resolve_downscale_descriptor_pool_; - descriptor_alloc_info.descriptorSetCount = 1; - descriptor_alloc_info.pSetLayouts = - &resolve_downscale_descriptor_set_layout_; - - VkDescriptorSet descriptor_set; - if (dfn.vkAllocateDescriptorSets(device, &descriptor_alloc_info, - &descriptor_set) != VK_SUCCESS) { - // Pool may be exhausted, reset it and try again - if (!AwaitAllQueueOperationsCompletion()) { - XELOGE( - "VulkanCommandProcessor: Failed to wait for GPU before resetting " - "descriptor pool"); - if (debug_markers_enabled_) { - PopDebugMarker(); - } - return true; - } - dfn.vkResetDescriptorPool(device, resolve_downscale_descriptor_pool_, 0); - if (dfn.vkAllocateDescriptorSets(device, &descriptor_alloc_info, - &descriptor_set) != VK_SUCCESS) { - XELOGE( - "VulkanCommandProcessor: Failed to allocate resolve downscale " - "descriptor set"); - if (debug_markers_enabled_) { - PopDebugMarker(); - } - return true; + // Allocate descriptor set for source and destination buffers. + // Uses pool chain to avoid mid-frame GPU stalls on pool exhaustion. + VkDescriptorSet descriptor_set = + resolve_downscale_descriptor_pool_chain_->Allocate( + GetCurrentSubmission()); + if (descriptor_set == VK_NULL_HANDLE) { + XELOGE( + "VulkanCommandProcessor: Failed to allocate resolve downscale " + "descriptor set from pool chain"); + if (debug_markers_enabled_) { + PopDebugMarker(); } + return true; } - // Ensure submission is open after any potential - // AwaitAllQueueOperationsCompletion calls during buffer allocation above + // Ensure submission is open if (!BeginSubmission(true)) { XELOGE( "VulkanCommandProcessor: Failed to begin submission for scaled " @@ -4856,27 +4823,9 @@ void VulkanCommandProcessor::CheckSubmissionFenceAndDeviceLoss( size_t fences_total = submissions_in_flight_fences_.size(); size_t fences_awaited = 0; - if (await_submission > submission_completed_) { - // Await in a blocking way if requested. - // TODO(Triang3l): Await only one fence. "Fence signal operations that are - // defined by vkQueueSubmit additionally include in the first - // synchronization scope all commands that occur earlier in submission - // order." - VkResult wait_result = dfn.vkWaitForFences( - device, uint32_t(await_submission - submission_completed_), - submissions_in_flight_fences_.data(), VK_TRUE, UINT64_MAX); - if (wait_result == VK_SUCCESS) { - fences_awaited += await_submission - submission_completed_; - } else { - XELOGE("Failed to await submission completion Vulkan fences"); - if (wait_result == VK_ERROR_DEVICE_LOST) { - device_lost_ = true; - } - } - } - // Check how far into the submissions the GPU currently is, in order because - // submission themselves can be executed out of order, but Xenia serializes - // that for simplicity. + // Check how far into the submissions the GPU currently is (non-blocking). + // This is similar to D3D12's GetCompletedValue() - check first, wait only if + // needed. while (fences_awaited < fences_total) { VkResult fence_status = dfn.vkWaitForFences( device, 1, &submissions_in_flight_fences_[fences_awaited], VK_TRUE, 0); @@ -4888,6 +4837,25 @@ void VulkanCommandProcessor::CheckSubmissionFenceAndDeviceLoss( } ++fences_awaited; } + // If we need to wait for a specific submission and it's not done yet, block. + if (!device_lost_ && + await_submission > submission_completed_ + fences_awaited) { + size_t fences_to_wait = + size_t(await_submission - submission_completed_) - fences_awaited; + if (fences_to_wait > 0 && fences_awaited < fences_total) { + VkResult wait_result = dfn.vkWaitForFences( + device, uint32_t(fences_to_wait), + &submissions_in_flight_fences_[fences_awaited], VK_TRUE, UINT64_MAX); + if (wait_result == VK_SUCCESS) { + fences_awaited += fences_to_wait; + } else { + XELOGE("Failed to await submission completion Vulkan fences"); + if (wait_result == VK_ERROR_DEVICE_LOST) { + device_lost_ = true; + } + } + } + } if (device_lost_) { graphics_system_->OnHostGpuLossFromAnyThread(true); return; @@ -4937,6 +4905,11 @@ void VulkanCommandProcessor::CheckSubmissionFenceAndDeviceLoss( texture_cache_->CompletedSubmissionUpdated(submission_completed_); + // Reclaim descriptor pools that the GPU has finished using. + if (resolve_downscale_descriptor_pool_chain_) { + resolve_downscale_descriptor_pool_chain_->Reclaim(submission_completed_); + } + ProcessReadyOcclusionQueries(submission_completed_); // Destroy objects scheduled for destruction. @@ -5377,6 +5350,12 @@ bool VulkanCommandProcessor::EndSubmission(bool is_swap) { submissions_in_flight_fences_.push_back(fence); fences_free_.pop_back(); + // Mark descriptor pool chains with submission index for reclaim tracking. + if (resolve_downscale_descriptor_pool_chain_) { + resolve_downscale_descriptor_pool_chain_->EndSubmission( + submission_current); + } + submission_open_ = false; } diff --git a/src/xenia/gpu/vulkan/vulkan_command_processor.h b/src/xenia/gpu/vulkan/vulkan_command_processor.h index 75b58230f..7872c4db2 100644 --- a/src/xenia/gpu/vulkan/vulkan_command_processor.h +++ b/src/xenia/gpu/vulkan/vulkan_command_processor.h @@ -38,6 +38,7 @@ #include "xenia/gpu/xenos.h" #include "xenia/kernel/kernel_state.h" #include "xenia/ui/vulkan/linked_type_descriptor_set_allocator.h" +#include "xenia/ui/vulkan/vulkan_descriptor_pool_chain.h" #include "xenia/ui/vulkan/vulkan_presenter.h" #include "xenia/ui/vulkan/vulkan_provider.h" #include "xenia/ui/vulkan/vulkan_upload_buffer_pool.h" @@ -689,8 +690,10 @@ class VulkanCommandProcessor final : public CommandProcessor { VK_NULL_HANDLE; VkPipelineLayout resolve_downscale_pipeline_layout_ = VK_NULL_HANDLE; VkPipeline resolve_downscale_pipeline_ = VK_NULL_HANDLE; - // Descriptor pool for resolve downscale shader (2 storage buffers per set). - VkDescriptorPool resolve_downscale_descriptor_pool_ = VK_NULL_HANDLE; + // Descriptor pool chain for resolve downscale shader (2 storage buffers per + // set). Uses pool chain to avoid mid-frame GPU stalls on pool exhaustion. + std::unique_ptr + resolve_downscale_descriptor_pool_chain_; // Intermediate buffer for downscaled output (device local, for compute). VkBuffer resolve_downscale_buffer_ = VK_NULL_HANDLE; VkDeviceMemory resolve_downscale_buffer_memory_ = VK_NULL_HANDLE; diff --git a/src/xenia/ui/vulkan/vulkan_descriptor_pool_chain.cc b/src/xenia/ui/vulkan/vulkan_descriptor_pool_chain.cc new file mode 100644 index 000000000..4fa14d200 --- /dev/null +++ b/src/xenia/ui/vulkan/vulkan_descriptor_pool_chain.cc @@ -0,0 +1,195 @@ +/** + ****************************************************************************** + * Xenia : Xbox 360 Emulator Research Project * + ****************************************************************************** + * Copyright 2025 Ben Vanik. All rights reserved. * + * Released under the BSD license - see LICENSE in the root for more details. * + ****************************************************************************** + */ + +#include "xenia/ui/vulkan/vulkan_descriptor_pool_chain.h" + +#include "xenia/base/logging.h" + +namespace xe { +namespace ui { +namespace vulkan { + +VulkanDescriptorPoolChain::VulkanDescriptorPoolChain( + const VulkanDevice* device, VkDescriptorPoolCreateFlags flags, + uint32_t max_sets_per_pool, const VkDescriptorPoolSize* pool_sizes, + uint32_t pool_size_count, VkDescriptorSetLayout set_layout) + : device_(device), + flags_(flags), + max_sets_per_pool_(max_sets_per_pool), + set_layout_(set_layout) { + pool_sizes_.assign(pool_sizes, pool_sizes + pool_size_count); +} + +VulkanDescriptorPoolChain::~VulkanDescriptorPoolChain() { ClearCache(); } + +void VulkanDescriptorPoolChain::Reclaim(uint64_t completed_submission_index) { + // Move pools that the GPU has finished using back to the writable list. + while (submitted_first_) { + if (submitted_first_->last_submission_index > completed_submission_index) { + break; + } + // Reset the pool so it can be reused. + const VulkanDevice::Functions& dfn = device_->functions(); + dfn.vkResetDescriptorPool(device_->device(), submitted_first_->pool, 0); + submitted_first_->allocated_sets = 0; + + // Move to writable list. + if (writable_last_) { + writable_last_->next = submitted_first_; + } else { + writable_first_ = submitted_first_; + } + writable_last_ = submitted_first_; + submitted_first_ = submitted_first_->next; + writable_last_->next = nullptr; + } + if (!submitted_first_) { + submitted_last_ = nullptr; + } +} + +void VulkanDescriptorPoolChain::ClearCache() { + const VulkanDevice::Functions& dfn = device_->functions(); + VkDevice vk_device = device_->device(); + + while (submitted_first_) { + Pool* next = submitted_first_->next; + DestroyPool(submitted_first_); + submitted_first_ = next; + } + submitted_last_ = nullptr; + + while (writable_first_) { + Pool* next = writable_first_->next; + DestroyPool(writable_first_); + writable_first_ = next; + } + writable_last_ = nullptr; +} + +VkDescriptorSet VulkanDescriptorPoolChain::Allocate(uint64_t submission_index) { + const VulkanDevice::Functions& dfn = device_->functions(); + VkDevice vk_device = device_->device(); + + // If current pool is full, move to next writable pool or create new. + if (writable_first_ && + writable_first_->allocated_sets >= max_sets_per_pool_) { + // Current pool is full, move it to submitted list. + if (submitted_last_) { + submitted_last_->next = writable_first_; + } else { + submitted_first_ = writable_first_; + } + submitted_last_ = writable_first_; + writable_first_ = writable_first_->next; + submitted_last_->next = nullptr; + if (!writable_first_) { + writable_last_ = nullptr; + } + } + + // Create new pool if needed. + if (!writable_first_) { + writable_first_ = CreatePool(); + if (!writable_first_) { + return VK_NULL_HANDLE; + } + writable_last_ = writable_first_; + } + + // Allocate from current pool. + VkDescriptorSetAllocateInfo alloc_info = {}; + alloc_info.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO; + alloc_info.descriptorPool = writable_first_->pool; + alloc_info.descriptorSetCount = 1; + alloc_info.pSetLayouts = &set_layout_; + + VkDescriptorSet descriptor_set; + VkResult result = + dfn.vkAllocateDescriptorSets(vk_device, &alloc_info, &descriptor_set); + + if (result != VK_SUCCESS) { + // Pool exhausted unexpectedly, try creating a new pool. + // Move current to submitted. + if (submitted_last_) { + submitted_last_->next = writable_first_; + } else { + submitted_first_ = writable_first_; + } + submitted_last_ = writable_first_; + writable_first_ = writable_first_->next; + submitted_last_->next = nullptr; + if (!writable_first_) { + writable_last_ = nullptr; + } + + // Create new pool. + writable_first_ = CreatePool(); + if (!writable_first_) { + return VK_NULL_HANDLE; + } + writable_last_ = writable_first_; + + // Retry allocation. + alloc_info.descriptorPool = writable_first_->pool; + result = + dfn.vkAllocateDescriptorSets(vk_device, &alloc_info, &descriptor_set); + if (result != VK_SUCCESS) { + XELOGE("VulkanDescriptorPoolChain: Failed to allocate descriptor set"); + return VK_NULL_HANDLE; + } + } + + writable_first_->allocated_sets++; + writable_first_->last_submission_index = submission_index; + return descriptor_set; +} + +void VulkanDescriptorPoolChain::EndSubmission(uint64_t submission_index) { + // Mark current pool with submission index for reclaim tracking. + if (writable_first_) { + writable_first_->last_submission_index = submission_index; + } +} + +VulkanDescriptorPoolChain::Pool* VulkanDescriptorPoolChain::CreatePool() { + const VulkanDevice::Functions& dfn = device_->functions(); + VkDevice vk_device = device_->device(); + + VkDescriptorPoolCreateInfo create_info = {}; + create_info.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO; + create_info.flags = flags_; + create_info.maxSets = max_sets_per_pool_; + create_info.poolSizeCount = static_cast(pool_sizes_.size()); + create_info.pPoolSizes = pool_sizes_.data(); + + VkDescriptorPool vk_pool; + if (dfn.vkCreateDescriptorPool(vk_device, &create_info, nullptr, &vk_pool) != + VK_SUCCESS) { + XELOGE("VulkanDescriptorPoolChain: Failed to create descriptor pool"); + return nullptr; + } + + Pool* pool = new Pool; + pool->pool = vk_pool; + pool->last_submission_index = 0; + pool->allocated_sets = 0; + pool->next = nullptr; + return pool; +} + +void VulkanDescriptorPoolChain::DestroyPool(Pool* pool) { + const VulkanDevice::Functions& dfn = device_->functions(); + dfn.vkDestroyDescriptorPool(device_->device(), pool->pool, nullptr); + delete pool; +} + +} // namespace vulkan +} // namespace ui +} // namespace xe diff --git a/src/xenia/ui/vulkan/vulkan_descriptor_pool_chain.h b/src/xenia/ui/vulkan/vulkan_descriptor_pool_chain.h new file mode 100644 index 000000000..b0afa9865 --- /dev/null +++ b/src/xenia/ui/vulkan/vulkan_descriptor_pool_chain.h @@ -0,0 +1,81 @@ +/** + ****************************************************************************** + * Xenia : Xbox 360 Emulator Research Project * + ****************************************************************************** + * Copyright 2025 Ben Vanik. All rights reserved. * + * Released under the BSD license - see LICENSE in the root for more details. * + ****************************************************************************** + */ + +#ifndef XENIA_UI_VULKAN_VULKAN_DESCRIPTOR_POOL_CHAIN_H_ +#define XENIA_UI_VULKAN_VULKAN_DESCRIPTOR_POOL_CHAIN_H_ + +#include +#include + +#include "xenia/ui/vulkan/vulkan_device.h" + +namespace xe { +namespace ui { +namespace vulkan { + +// A chain of descriptor pools that rotates based on submission index, +// avoiding mid-frame GPU stalls when pools are exhausted. +// Similar to D3D12DescriptorHeapPool but for Vulkan's descriptor model. +class VulkanDescriptorPoolChain { + public: + VulkanDescriptorPoolChain(const VulkanDevice* device, + VkDescriptorPoolCreateFlags flags, + uint32_t max_sets_per_pool, + const VkDescriptorPoolSize* pool_sizes, + uint32_t pool_size_count, + VkDescriptorSetLayout set_layout); + ~VulkanDescriptorPoolChain(); + + // Reclaim pools that have been fully processed by the GPU. + // Call this when submissions complete (e.g., after fence signaled). + void Reclaim(uint64_t completed_submission_index); + + // Clear all pools. Must only be called when GPU is idle. + void ClearCache(); + + // Allocate a descriptor set from the chain. + // Returns VK_NULL_HANDLE on failure. + // The submission_index should be the current frame/submission being built. + VkDescriptorSet Allocate(uint64_t submission_index); + + // Mark current pool as submitted and ready for reclaim after GPU completes. + // Call this when submitting a command buffer. + void EndSubmission(uint64_t submission_index); + + private: + struct Pool { + VkDescriptorPool pool; + uint64_t last_submission_index; + uint32_t allocated_sets; + Pool* next; + }; + + Pool* CreatePool(); + void DestroyPool(Pool* pool); + + const VulkanDevice* device_; + VkDescriptorPoolCreateFlags flags_; + uint32_t max_sets_per_pool_; + std::vector pool_sizes_; + VkDescriptorSetLayout set_layout_; + + // Pools with free space, first is current. + Pool* writable_first_ = nullptr; + Pool* writable_last_ = nullptr; + + // Pools waiting for GPU to finish. + Pool* submitted_first_ = nullptr; + Pool* submitted_last_ = nullptr; +}; + +} // namespace vulkan +} // namespace ui +} // namespace xe + +#endif // XENIA_UI_VULKAN_VULKAN_DESCRIPTOR_POOL_CHAIN_H_