diff --git a/src/video_core/renderer_vulkan/vk_renderpass_cache.cpp b/src/video_core/renderer_vulkan/vk_renderpass_cache.cpp
index 981a411eb..167054088 100644
--- a/src/video_core/renderer_vulkan/vk_renderpass_cache.cpp
+++ b/src/video_core/renderer_vulkan/vk_renderpass_cache.cpp
@@ -9,60 +9,8 @@
 
 namespace Vulkan {
 
-VideoCore::PixelFormat ToFormatColor(u32 index) {
-    switch (index) {
-    case 0:
-        return VideoCore::PixelFormat::RGBA8;
-    case 1:
-        return VideoCore::PixelFormat::RGB8;
-    case 2:
-        return VideoCore::PixelFormat::RGB5A1;
-    case 3:
-        return VideoCore::PixelFormat::RGB565;
-    case 4:
-        return VideoCore::PixelFormat::RGBA4;
-    default:
-        return VideoCore::PixelFormat::Invalid;
-    }
-}
-
-VideoCore::PixelFormat ToFormatDepth(u32 index) {
-    switch (index) {
-    case 0:
-        return VideoCore::PixelFormat::D16;
-    case 2:
-        return VideoCore::PixelFormat::D24;
-    case 3:
-        return VideoCore::PixelFormat::D24S8;
-    default:
-        return VideoCore::PixelFormat::Invalid;
-    }
-}
-
 RenderpassCache::RenderpassCache(const Instance& instance, Scheduler& scheduler)
-    : instance{instance}, scheduler{scheduler} {
-
-    for (u32 color = 0; color <= MAX_COLOR_FORMATS; color++) {
-        for (u32 depth = 0; depth <= MAX_DEPTH_FORMATS; depth++) {
-            const VideoCore::PixelFormat color_format = ToFormatColor(color);
-            const VideoCore::PixelFormat depth_format = ToFormatDepth(depth);
-            if (color_format == VideoCore::PixelFormat::Invalid &&
-                depth_format == VideoCore::PixelFormat::Invalid) {
-                continue;
-            }
-
-            const FormatTraits& color_traits = instance.GetTraits(color_format);
-            const FormatTraits& depth_traits = instance.GetTraits(depth_format);
-
-            cached_renderpasses[color][depth][0] = CreateRenderPass(
-                color_traits.native, depth_traits.native, vk::AttachmentLoadOp::eLoad,
-                vk::ImageLayout::eGeneral, vk::ImageLayout::eGeneral);
-            cached_renderpasses[color][depth][1] = CreateRenderPass(
-                color_traits.native, depth_traits.native, vk::AttachmentLoadOp::eClear,
-                vk::ImageLayout::eGeneral, vk::ImageLayout::eGeneral);
-        }
-    }
-}
+    : instance{instance}, scheduler{scheduler} {}
 
 RenderpassCache::~RenderpassCache() {
     vk::Device device = instance.GetDevice();
@@ -136,15 +84,28 @@ void RenderpassCache::CreatePresentRenderpass(vk::Format format) {
 }
 
 vk::RenderPass RenderpassCache::GetRenderpass(VideoCore::PixelFormat color,
-                                              VideoCore::PixelFormat depth, bool is_clear) const {
+                                              VideoCore::PixelFormat depth, bool is_clear) {
     const u32 color_index =
         color == VideoCore::PixelFormat::Invalid ? MAX_COLOR_FORMATS : static_cast<u32>(color);
     const u32 depth_index = depth == VideoCore::PixelFormat::Invalid
                                 ? MAX_DEPTH_FORMATS
                                 : (static_cast<u32>(depth) - 14);
 
-    ASSERT(color_index <= MAX_COLOR_FORMATS && depth_index <= MAX_DEPTH_FORMATS);
-    return cached_renderpasses[color_index][depth_index][is_clear];
+    ASSERT_MSG(color_index <= MAX_COLOR_FORMATS && depth_index <= MAX_DEPTH_FORMATS &&
+                   (color_index != MAX_COLOR_FORMATS || depth_index != MAX_DEPTH_FORMATS),
+               "Invalid color index {} and/or depth_index {}", color_index, depth_index);
+
+    vk::RenderPass& renderpass = cached_renderpasses[color_index][depth_index][is_clear];
+    if (!renderpass) {
+        const vk::Format color_format = instance.GetTraits(color).native;
+        const vk::Format depth_format = instance.GetTraits(depth).native;
+        const vk::AttachmentLoadOp load_op =
+            is_clear ? vk::AttachmentLoadOp::eClear : vk::AttachmentLoadOp::eLoad;
+        renderpass = CreateRenderPass(color_format, depth_format, load_op,
+                                      vk::ImageLayout::eGeneral, vk::ImageLayout::eGeneral);
+    }
+
+    return renderpass;
 }
 
 vk::RenderPass RenderpassCache::CreateRenderPass(vk::Format color, vk::Format depth,
diff --git a/src/video_core/renderer_vulkan/vk_renderpass_cache.h b/src/video_core/renderer_vulkan/vk_renderpass_cache.h
index 68f007647..bb4831be5 100644
--- a/src/video_core/renderer_vulkan/vk_renderpass_cache.h
+++ b/src/video_core/renderer_vulkan/vk_renderpass_cache.h
@@ -43,7 +43,7 @@ public:
 
     /// Returns the renderpass associated with the color-depth format pair
     [[nodiscard]] vk::RenderPass GetRenderpass(VideoCore::PixelFormat color,
-                                               VideoCore::PixelFormat depth, bool is_clear) const;
+                                               VideoCore::PixelFormat depth, bool is_clear);
 
     /// Returns the swapchain clear renderpass
     [[nodiscard]] vk::RenderPass GetPresentRenderpass() const {