VK: synchronise the generated-frame acquire, and free framegen with the renderer

Two things frame generation was getting away with rather than doing.

The acquire for a generated frame passed VK_NULL_HANDLE for both the semaphore
and the fence. The specification forbids that outright, and the practical
consequence is that nothing made the blit wait for the presentation engine to
finish with the image -- the UNDEFINED old layout on the barrier discards
contents, it does not order anything, so the blit could overwrite an image
still being scanned out. It acquires with a semaphore now and the submit waits
on it.

A ring of eight rather than a single semaphore, because a binary semaphore may
not be waited again before it has been signalled again and up to three
generated frames are acquired per flip. Eight is more than two flips of slack,
so a slot is never revisited before the submit that waits on it has retired.

The other is a crash rather than a hazard. Frame generation's images are file
-scope globals created on the renderer's device, and nothing released them, so
stopping a game and starting another at the same resolution skipped the
rebuild -- capture_presented_frame only rebuilds when the dimensions change --
and recorded images belonging to a destroyed device into a live command
buffer. They are released in ~VKGSRender now, after the vkDeviceWaitIdle it
already does and before the swapchain goes.

That location is the point. An earlier attempt at the same teardown ran from
inside the capture hook, which executes while the frame's primary command
buffer is still being built, and it destroyed rendering outright. A
device-idle point the renderer owns is where this always belonged.

rpcs3/Emu/RSX/VK/VKGSRender.cpp

@@ -8,6 +8,7 @@
 #include "VKCommonPipelineLayout.h"
 #include "VKCompute.h"
 #include "VKGSRender.h"
+#include "VKFrameGen.h"
 #include "Emu/RSX/rsx_profiler.h"
 #include "vkutils/gpu_timer.h"
 #include "VKHelpers.h"
@@ -857,6 +858,20 @@ VKGSRender::~VKGSRender()
 	//Wait for device to finish up with resources
 	vkDeviceWaitIdle(*m_device);
 
+	// Frame generation holds images created on THIS device, and it is a set of file-scope globals
+	// that outlives the renderer. Nothing used to release them, so stopping a game and booting
+	// another at the same resolution skipped the rebuild -- capture_presented_frame only rebuilds
+	// when the dimensions change -- and recorded images belonging to a destroyed device into a
+	// live command buffer.
+	//
+	// Here specifically, and not from the present path: an earlier attempt tore this down inside
+	// the capture hook, which runs while the frame's primary command buffer is still being built,
+	// and it destroyed rendering outright. This is a device-idle point the renderer owns, after
+	// the wait above and before the swapchain goes, which is where the teardown always belonged.
+	vk::frame_gen::release_shared_images();
+	vk::frame_gen::shutdown();
+	destroy_framegen_acquire_semaphores();
+
 	// Globals. TODO: Refactor lifetime management
 	if (auto async_scheduler = g_fxo->try_get<vk::AsyncTaskScheduler>())
 	{

rpcs3/Emu/RSX/VK/VKGSRender.h

@@ -203,6 +203,16 @@ private:
 	vk::command_buffer_chunk* m_framegen_blit_cb[3] = {};
 	u32 m_framegen_blit_cb_count = 0;
 
+	// Acquire semaphores for generated-frame presents. A ring rather than one, because a binary
+	// semaphore may not be waited again until it has been signalled again, and up to three
+	// generated frames are acquired per flip. Eight gives more than two flips of slack, so a slot
+	// is never revisited before the submit that waits it has retired.
+	static constexpr u32 c_framegen_acquire_sem_count = 8;
+	VkSemaphore m_framegen_acquire_sem[c_framegen_acquire_sem_count] = {};
+	u32 m_framegen_acquire_sem_index = 0;
+
+	void destroy_framegen_acquire_semaphores();
+
 	VkViewport m_viewport {};
 	VkRect2D m_scissor {};
 

rpcs3/Emu/RSX/VK/VKPresent.cpp

@@ -215,6 +215,20 @@ bool VKGSRender::reinitialize_swapchain()
 // Best-effort throughout. A generated frame is an extra, so every failure path here simply
 // returns and lets the real frame present normally -- dropping an interpolated frame is invisible,
 // while stalling or presenting a torn one is not.
+void VKGSRender::destroy_framegen_acquire_semaphores()
+{
+	for (auto& sem : m_framegen_acquire_sem)
+	{
+		if (sem != VK_NULL_HANDLE)
+		{
+			vkDestroySemaphore(*m_device, sem, nullptr);
+			sem = VK_NULL_HANDLE;
+		}
+	}
+
+	m_framegen_acquire_sem_index = 0;
+}
+
 vk::command_buffer_chunk* VKGSRender::present_generated_frame(VkImage src)
 {
 	if (src == VK_NULL_HANDLE || swapchain_unavailable || m_swapchain->is_headless())
@@ -234,8 +248,33 @@ vk::command_buffer_chunk* VKGSRender::present_generated_frame(VkImage src)
 	// until the acquirable pool was empty, at which point the real frame's acquire in flip() blocks
 	// its full 100ms timeout every frame -- a hard lock at ten fps that only a swapchain rebuild
 	// clears, which is why it survived turning frame generation back off.
+	// Acquire with a semaphore, and make the blit wait on it.
+	//
+	// This passed VK_NULL_HANDLE for both the semaphore and the fence, which the specification
+	// forbids outright (VUID-vkAcquireNextImageKHR-semaphore-01780) and which left nothing making
+	// the blit wait for the presentation engine to be finished with the image. The UNDEFINED old
+	// layout on the barrier below is not that guarantee either -- it discards the contents, it
+	// does not order anything -- so the blit could overwrite an image still being scanned out.
+	VkSemaphore& acquire_sem = m_framegen_acquire_sem[m_framegen_acquire_sem_index];
+
+	if (acquire_sem == VK_NULL_HANDLE)
+	{
+		VkSemaphoreCreateInfo semaphore_info = {};
+		semaphore_info.sType = VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO;
+
+		if (vkCreateSemaphore(*m_device, &semaphore_info, nullptr, &acquire_sem) != VK_SUCCESS)
+		{
+			// Without one there is no correct way to do this, and presenting a torn generated
+			// frame is worse than presenting none.
+			return nullptr;
+		}
+	}
+
+	m_framegen_acquire_sem_index =
+		(m_framegen_acquire_sem_index + 1u) % c_framegen_acquire_sem_count;
+
 	const VkResult acquire_result =
-		m_swapchain->acquire_next_swapchain_image(VK_NULL_HANDLE, 0ull, &image);
+		m_swapchain->acquire_next_swapchain_image(acquire_sem, 0ull, &image);
 
 	if ((acquire_result != VK_SUCCESS && acquire_result != VK_SUBOPTIMAL_KHR) || image == umax)
 	{
@@ -318,6 +357,7 @@ vk::command_buffer_chunk* VKGSRender::present_generated_frame(VkImage src)
 	// of this swapchain image.
 	vk::queue_submit_t submit_info{};
 	submit_info.queue = m_device->get_graphics_queue();
+	submit_info.wait_on(acquire_sem, VK_PIPELINE_STAGE_TRANSFER_BIT);
 
 	// flush, because the present below runs on this thread while a deferred submit would not have
 	// happened yet under multithreaded RSX -- presenting a swapchain image before the blit that