From ad4339464a0c585def5ab9d5ef25e90c39f22cf5 Mon Sep 17 00:00:00 2001 From: GPUCode Date: Sat, 4 Mar 2023 22:22:12 +0200 Subject: [PATCH] renderer_vulkan: Submit present frames from the EmuThread * This commit also reworks scheduler synchronization somewhat to be more reliable. Flush is given an atomic_bool is signal when the submit is done --- .../renderer_vulkan/renderer_vulkan.cpp | 5 +- .../renderer_vulkan/vk_descriptor_manager.cpp | 3 +- .../renderer_vulkan/vk_renderpass_cache.cpp | 4 +- .../renderer_vulkan/vk_renderpass_cache.h | 4 +- .../renderer_vulkan/vk_scheduler.cpp | 105 ++++++++---------- src/video_core/renderer_vulkan/vk_scheduler.h | 10 +- .../renderer_vulkan/vk_texture_mailbox.cpp | 5 + .../renderer_vulkan/vk_texture_mailbox.h | 3 + .../renderer_vulkan/vk_texture_runtime.cpp | 4 +- 9 files changed, 70 insertions(+), 73 deletions(-) diff --git a/src/video_core/renderer_vulkan/renderer_vulkan.cpp b/src/video_core/renderer_vulkan/renderer_vulkan.cpp index 63727e9cf..bf8ac4223 100644 --- a/src/video_core/renderer_vulkan/renderer_vulkan.cpp +++ b/src/video_core/renderer_vulkan/renderer_vulkan.cpp @@ -217,9 +217,8 @@ void RendererVulkan::RenderToMailbox(const Layout::FramebufferLayout& layout, DrawScreens(frame, layout, flipped); - scheduler.Flush(frame->render_ready); - scheduler.Record([&mailbox, frame](vk::CommandBuffer) { mailbox->Present(frame); }); - scheduler.DispatchWork(); + scheduler.Flush(frame->render_ready, nullptr, &frame->is_submitted); + mailbox->Present(frame); } void RendererVulkan::BeginRendering(Frame* frame) { diff --git a/src/video_core/renderer_vulkan/vk_descriptor_manager.cpp b/src/video_core/renderer_vulkan/vk_descriptor_manager.cpp index 9a234c5d0..83770843c 100644 --- a/src/video_core/renderer_vulkan/vk_descriptor_manager.cpp +++ b/src/video_core/renderer_vulkan/vk_descriptor_manager.cpp @@ -188,7 +188,8 @@ void DescriptorManager::BuildLayouts() { pipeline_layout = device.createPipelineLayout(layout_info); } -std::vector DescriptorManager::AllocateSets(vk::DescriptorSetLayout layout, u32 num_sets) { +std::vector DescriptorManager::AllocateSets(vk::DescriptorSetLayout layout, + u32 num_sets) { static std::array layouts; layouts.fill(layout); diff --git a/src/video_core/renderer_vulkan/vk_renderpass_cache.cpp b/src/video_core/renderer_vulkan/vk_renderpass_cache.cpp index 28b2a35c8..d3050ed5a 100644 --- a/src/video_core/renderer_vulkan/vk_renderpass_cache.cpp +++ b/src/video_core/renderer_vulkan/vk_renderpass_cache.cpp @@ -44,12 +44,12 @@ void RenderpassCache::ClearFramebuffers() { } void RenderpassCache::BeginRendering(Surface* const color, Surface* const depth_stencil, - vk::Rect2D render_area, bool do_clear, vk::ClearValue clear) { + vk::Rect2D render_area, bool do_clear, vk::ClearValue clear) { return BeginRendering(Framebuffer{color, depth_stencil, render_area}, do_clear, clear); } void RenderpassCache::BeginRendering(const Framebuffer& framebuffer, bool do_clear, - vk::ClearValue clear) { + vk::ClearValue clear) { RenderingInfo new_info = { .color{ .aspect = vk::ImageAspectFlagBits::eColor, diff --git a/src/video_core/renderer_vulkan/vk_renderpass_cache.h b/src/video_core/renderer_vulkan/vk_renderpass_cache.h index 507eaf69a..dbaeb8442 100644 --- a/src/video_core/renderer_vulkan/vk_renderpass_cache.h +++ b/src/video_core/renderer_vulkan/vk_renderpass_cache.h @@ -55,9 +55,9 @@ public: /// Begins a new renderpass only when no other renderpass is currently active void BeginRendering(const Framebuffer& framebuffer, bool do_clear = false, - vk::ClearValue clear = {}); + vk::ClearValue clear = {}); void BeginRendering(Surface* const color, Surface* const depth_stencil, vk::Rect2D render_area, - bool do_clear = false, vk::ClearValue clear = {}); + bool do_clear = false, vk::ClearValue clear = {}); /// Exits from any currently active renderpass instance void EndRendering(); diff --git a/src/video_core/renderer_vulkan/vk_scheduler.cpp b/src/video_core/renderer_vulkan/vk_scheduler.cpp index 32944021d..d64a0ef2c 100644 --- a/src/video_core/renderer_vulkan/vk_scheduler.cpp +++ b/src/video_core/renderer_vulkan/vk_scheduler.cpp @@ -42,27 +42,16 @@ Scheduler::Scheduler(const Instance& instance, RenderpassCache& renderpass_cache Scheduler::~Scheduler() = default; -void Scheduler::Flush(vk::Semaphore signal, vk::Semaphore wait) { - SubmitExecution(signal, wait); -} - void Scheduler::Finish(vk::Semaphore signal, vk::Semaphore wait) { const u64 presubmit_tick = CurrentTick(); - SubmitExecution(signal, wait); - WaitWorker(); - Wait(presubmit_tick); -} + std::atomic_bool submit_done{false}; -void Scheduler::WaitWorker() { - if (!use_worker_thread) { - return; + Flush(signal, wait, &submit_done); + if (use_worker_thread) { + MICROPROFILE_SCOPE(Vulkan_WaitForWorker); + submit_done.wait(false); } - - MICROPROFILE_SCOPE(Vulkan_WaitForWorker); - DispatchWork(); - - std::unique_lock lock{work_mutex}; - wait_cv.wait(lock, [this] { return work_queue.empty(); }); + Wait(presubmit_tick); } void Scheduler::DispatchWork() { @@ -116,56 +105,60 @@ void Scheduler::AllocateWorkerCommandBuffers() { current_cmdbuf.begin(begin_info); } -void Scheduler::SubmitExecution(vk::Semaphore signal_semaphore, vk::Semaphore wait_semaphore) { +void Scheduler::Flush(vk::Semaphore signal, vk::Semaphore wait, std::atomic_bool* submit_done) { const vk::Semaphore handle = master_semaphore.Handle(); const u64 signal_value = master_semaphore.NextTick(); state = StateFlags::AllDirty; renderpass_cache.EndRendering(); - Record( - [signal_semaphore, wait_semaphore, handle, signal_value, this](vk::CommandBuffer cmdbuf) { - MICROPROFILE_SCOPE(Vulkan_Submit); - cmdbuf.end(); + Record([signal, wait, handle, signal_value, submit_done, this](vk::CommandBuffer cmdbuf) { + MICROPROFILE_SCOPE(Vulkan_Submit); + cmdbuf.end(); - const u32 num_signal_semaphores = signal_semaphore ? 2U : 1U; - const std::array signal_values{signal_value, u64(0)}; - const std::array signal_semaphores{handle, signal_semaphore}; + const u32 num_signal_semaphores = signal ? 2U : 1U; + const std::array signal_values{signal_value, u64(0)}; + const std::array signal_semaphores{handle, signal}; - const u32 num_wait_semaphores = wait_semaphore ? 2U : 1U; - const std::array wait_values{signal_value - 1, u64(1)}; - const std::array wait_semaphores{handle, wait_semaphore}; + const u32 num_wait_semaphores = wait ? 2U : 1U; + const std::array wait_values{signal_value - 1, u64(1)}; + const std::array wait_semaphores{handle, wait}; - static constexpr std::array wait_stage_masks = { - vk::PipelineStageFlagBits::eAllCommands, - vk::PipelineStageFlagBits::eColorAttachmentOutput, - }; + static constexpr std::array wait_stage_masks = { + vk::PipelineStageFlagBits::eAllCommands, + vk::PipelineStageFlagBits::eColorAttachmentOutput, + }; - const vk::TimelineSemaphoreSubmitInfoKHR timeline_si = { - .waitSemaphoreValueCount = num_wait_semaphores, - .pWaitSemaphoreValues = wait_values.data(), - .signalSemaphoreValueCount = num_signal_semaphores, - .pSignalSemaphoreValues = signal_values.data(), - }; + const vk::TimelineSemaphoreSubmitInfoKHR timeline_si = { + .waitSemaphoreValueCount = num_wait_semaphores, + .pWaitSemaphoreValues = wait_values.data(), + .signalSemaphoreValueCount = num_signal_semaphores, + .pSignalSemaphoreValues = signal_values.data(), + }; - const vk::SubmitInfo submit_info = { - .pNext = &timeline_si, - .waitSemaphoreCount = num_wait_semaphores, - .pWaitSemaphores = wait_semaphores.data(), - .pWaitDstStageMask = wait_stage_masks.data(), - .commandBufferCount = 1u, - .pCommandBuffers = &cmdbuf, - .signalSemaphoreCount = num_signal_semaphores, - .pSignalSemaphores = signal_semaphores.data(), - }; + const vk::SubmitInfo submit_info = { + .pNext = &timeline_si, + .waitSemaphoreCount = num_wait_semaphores, + .pWaitSemaphores = wait_semaphores.data(), + .pWaitDstStageMask = wait_stage_masks.data(), + .commandBufferCount = 1u, + .pCommandBuffers = &cmdbuf, + .signalSemaphoreCount = num_signal_semaphores, + .pSignalSemaphores = signal_semaphores.data(), + }; - try { - std::scoped_lock lock{queue_mutex}; - instance.GetGraphicsQueue().submit(submit_info); - } catch (vk::DeviceLostError& err) { - LOG_CRITICAL(Render_Vulkan, "Device lost during submit: {}", err.what()); - UNREACHABLE(); - } - }); + try { + std::scoped_lock lock{queue_mutex}; + instance.GetGraphicsQueue().submit(submit_info); + } catch (vk::DeviceLostError& err) { + LOG_CRITICAL(Render_Vulkan, "Device lost during submit: {}", err.what()); + UNREACHABLE(); + } + + if (submit_done) { + *submit_done = true; + submit_done->notify_one(); + } + }); if (!use_worker_thread) { AllocateWorkerCommandBuffers(); diff --git a/src/video_core/renderer_vulkan/vk_scheduler.h b/src/video_core/renderer_vulkan/vk_scheduler.h index 35e84964f..2c320b369 100644 --- a/src/video_core/renderer_vulkan/vk_scheduler.h +++ b/src/video_core/renderer_vulkan/vk_scheduler.h @@ -3,6 +3,7 @@ #pragma once +#include #include #include #include @@ -38,15 +39,12 @@ public: ~Scheduler(); /// Sends the current execution context to the GPU. - void Flush(vk::Semaphore signal = nullptr, vk::Semaphore wait = nullptr); + void Flush(vk::Semaphore signal = nullptr, vk::Semaphore wait = nullptr, + std::atomic_bool* submit_done = nullptr); /// Sends the current execution context to the GPU and waits for it to complete. void Finish(vk::Semaphore signal = nullptr, vk::Semaphore wait = nullptr); - /// Waits for the worker thread to finish executing everything. After this function returns it's - /// safe to touch worker resources. - void WaitWorker(); - /// Sends currently recorded work to the worker thread. void DispatchWork(); @@ -199,8 +197,6 @@ private: void AllocateWorkerCommandBuffers(); - void SubmitExecution(vk::Semaphore signal_semaphore, vk::Semaphore wait_semaphore); - void AcquireNewChunk(); private: diff --git a/src/video_core/renderer_vulkan/vk_texture_mailbox.cpp b/src/video_core/renderer_vulkan/vk_texture_mailbox.cpp index 150644242..7fbc360e7 100644 --- a/src/video_core/renderer_vulkan/vk_texture_mailbox.cpp +++ b/src/video_core/renderer_vulkan/vk_texture_mailbox.cpp @@ -41,6 +41,7 @@ PresentMailbox::PresentMailbox(const Instance& instance_, Swapchain& swapchain_, for (u32 i = 0; i < SWAP_CHAIN_SIZE; i++) { Frame& frame = swap_chain[i]; + frame.index = i; frame.cmdbuf = command_buffers[i]; frame.render_ready = device.createSemaphore({}); frame.present_done = device.createFence({.flags = vk::FenceCreateFlagBits::eSignaled}); @@ -162,6 +163,7 @@ Frame* PresentMailbox::GetRenderFrame() { } device.resetFences(frame->present_done); + frame->is_submitted = false; return frame; } @@ -326,6 +328,9 @@ void PresentMailbox::CopyToSwapchain(Frame* frame) { .pSignalSemaphores = &present_ready, }; + // Ensure we won't wait on a semaphore that has no way of being signaled + frame->is_submitted.wait(false); + try { std::scoped_lock lock{scheduler.QueueMutex(), frame->fence_mutex}; graphics_queue.submit(submit_info, frame->present_done); diff --git a/src/video_core/renderer_vulkan/vk_texture_mailbox.h b/src/video_core/renderer_vulkan/vk_texture_mailbox.h index 09f914286..b2ede0662 100644 --- a/src/video_core/renderer_vulkan/vk_texture_mailbox.h +++ b/src/video_core/renderer_vulkan/vk_texture_mailbox.h @@ -2,6 +2,7 @@ // Licensed under GPLv2 or any later version // Refer to the license.txt file included. +#include #include #include #include @@ -21,6 +22,7 @@ class RenderpassCache; struct Frame { u32 width{}; u32 height{}; + u32 index{}; VmaAllocation allocation{}; vk::Framebuffer framebuffer{}; vk::Image image{}; @@ -29,6 +31,7 @@ struct Frame { vk::Fence present_done{}; std::mutex fence_mutex{}; vk::CommandBuffer cmdbuf{}; + std::atomic_bool is_submitted{false}; }; class PresentMailbox final { diff --git a/src/video_core/renderer_vulkan/vk_texture_runtime.cpp b/src/video_core/renderer_vulkan/vk_texture_runtime.cpp index 06598f81f..f23d6a36d 100644 --- a/src/video_core/renderer_vulkan/vk_texture_runtime.cpp +++ b/src/video_core/renderer_vulkan/vk_texture_runtime.cpp @@ -446,7 +446,7 @@ void TextureRuntime::ClearTextureWithRenderpass(Surface& surface, }; renderpass_cache.BeginRendering(color_surface, depth_surface, render_area, true, - MakeClearValue(clear.value)); + MakeClearValue(clear.value)); renderpass_cache.EndRendering(); scheduler.Record([params, access_flag, pipeline_flags](vk::CommandBuffer cmdbuf) { @@ -1074,7 +1074,7 @@ vk::PipelineStageFlags Surface::PipelineStageFlags() const noexcept { return vk::PipelineStageFlagBits::eTransfer | vk::PipelineStageFlagBits::eFragmentShader | (alloc.is_framebuffer ? attachment_flags : vk::PipelineStageFlagBits::eNone) | (alloc.is_storage ? vk::PipelineStageFlagBits::eComputeShader - : vk::PipelineStageFlagBits::eNone); + : vk::PipelineStageFlagBits::eNone); } vk::ImageView Surface::DepthView() noexcept {