renderer_vulkan: Fix storage descriptor binding and respect color mask

* RGBA8 surfaces now expose an additional R32Uint view used for storage descriptors. The format is guaranteed by the spec to support atomic loads/stores. This requires the mutable flag which incurs a performance cost, but might be better than breaking the current renderpass each draw when rendering shadows, especially on mobile

* Color mask is also implemented which fixes Street Fighter and Monster Hunter Stories

src/video_core/rasterizer_cache/rasterizer_cache.h

@@ -886,8 +886,8 @@ void RasterizerCache<T>::ValidateSurface(const Surface& surface, PAddr addr, u32
             // If the region was created entirely on the GPU,
             // assume it was a developer mistake and skip flushing.
             if (boost::icl::contains(dirty_regions, interval)) {
-                LOG_INFO(HW_GPU, "Region created fully on GPU and reinterpretation is "
+                LOG_DEBUG(HW_GPU, "Region created fully on GPU and reinterpretation is "
-                                 "invalid. Skipping validation");
+                                  "invalid. Skipping validation");
                 validate_regions.erase(interval);
                 continue;
             }

src/video_core/renderer_vulkan/vk_pipeline_cache.cpp

@@ -550,8 +550,7 @@ vk::Pipeline PipelineCache::BuildPipeline(const PipelineInfo& info) {
         .srcAlphaBlendFactor = PicaToVK::BlendFunc(info.blending.src_alpha_blend_factor),
         .dstAlphaBlendFactor = PicaToVK::BlendFunc(info.blending.dst_alpha_blend_factor),
         .alphaBlendOp = PicaToVK::BlendEquation(info.blending.alpha_blend_eq),
-        .colorWriteMask = vk::ColorComponentFlagBits::eR | vk::ColorComponentFlagBits::eG |
+        .colorWriteMask = static_cast<vk::ColorComponentFlags>(info.blending.color_write_mask)};
-                          vk::ColorComponentFlagBits::eB | vk::ColorComponentFlagBits::eA};
 
     const vk::PipelineColorBlendStateCreateInfo color_blending = {
         .logicOpEnable = !info.blending.blend_enable.Value(),

src/video_core/renderer_vulkan/vk_rasterizer.cpp

@@ -124,7 +124,7 @@ RasterizerVulkan::RasterizerVulkan(Frontend::EmuWindow& emu_window, const Instan
       renderpass_cache{renderpass_cache}, res_cache{*this, runtime},
       pipeline_cache{instance, scheduler, renderpass_cache},
       null_surface{NULL_PARAMS, vk::Format::eR8G8B8A8Unorm, NULL_USAGE, runtime},
-      null_storage_surface{NULL_PARAMS, vk::Format::eR8G8B8A8Uint, NULL_STORAGE_USAGE, runtime},
+      null_storage_surface{NULL_PARAMS, vk::Format::eR32Uint, NULL_STORAGE_USAGE, runtime},
       vertex_buffer{
           instance, scheduler, VERTEX_BUFFER_SIZE, vk::BufferUsageFlagBits::eVertexBuffer, {}},
       uniform_buffer{
@@ -714,7 +714,8 @@ bool RasterizerVulkan::Draw(bool accelerate, bool is_indexed) {
                 case TextureType::Shadow2D: {
                     auto surface = res_cache.GetTextureSurface(texture);
                     if (surface) {
-                        pipeline_cache.BindStorageImage(0, surface->GetImageView());
+                        surface->Transition(vk::ImageLayout::eGeneral, 0, surface->alloc.levels);
+                        pipeline_cache.BindStorageImage(0, surface->GetStorageView());
                     } else {
                         pipeline_cache.BindStorageImage(0, null_storage_surface.GetImageView());
                     }

src/video_core/renderer_vulkan/vk_task_scheduler.cpp

@@ -30,11 +30,11 @@ TaskScheduler::TaskScheduler(const Instance& instance, RendererVulkan& renderer)
     }
 
     constexpr std::array pool_sizes = {
-        vk::DescriptorPoolSize{vk::DescriptorType::eUniformBuffer, 1024},
+        vk::DescriptorPoolSize{vk::DescriptorType::eUniformBuffer, 2048},
-        vk::DescriptorPoolSize{vk::DescriptorType::eUniformBufferDynamic, 1024},
+        vk::DescriptorPoolSize{vk::DescriptorType::eUniformBufferDynamic, 2048},
         vk::DescriptorPoolSize{vk::DescriptorType::eSampledImage, 2048},
-        vk::DescriptorPoolSize{vk::DescriptorType::eSampler, 2048},
+        vk::DescriptorPoolSize{vk::DescriptorType::eSampler, 4096},
-        vk::DescriptorPoolSize{vk::DescriptorType::eUniformTexelBuffer, 1024},
+        vk::DescriptorPoolSize{vk::DescriptorType::eUniformTexelBuffer, 2048},
         vk::DescriptorPoolSize{vk::DescriptorType::eStorageImage, 1024}};
 
     const vk::DescriptorPoolCreateInfo descriptor_pool_info = {

src/video_core/renderer_vulkan/vk_texture_runtime.cpp

@@ -69,6 +69,9 @@ TextureRuntime::~TextureRuntime() {
             device.destroyImageView(alloc.depth_view);
             device.destroyImageView(alloc.stencil_view);
         }
+        if (alloc.storage_view) {
+            device.destroyImageView(alloc.storage_view);
+        }
     }
 
     for (const auto& [key, framebuffer] : clear_framebuffers) {
@@ -114,11 +117,13 @@ ImageAlloc TextureRuntime::Allocate(u32 width, u32 height, VideoCore::PixelForma
     const vk::Format vk_format = is_suitable ? traits.native : traits.fallback;
     const vk::ImageUsageFlags vk_usage = is_suitable ? traits.usage : GetImageUsage(aspect);
 
-    return Allocate(width, height, type, vk_format, vk_usage);
+    return Allocate(width, height, type, vk_format, vk_usage,
+                    format == VideoCore::PixelFormat::RGBA8);
 }
 
 ImageAlloc TextureRuntime::Allocate(u32 width, u32 height, VideoCore::TextureType type,
-                                    vk::Format format, vk::ImageUsageFlags usage) {
+                                    vk::Format format, vk::ImageUsageFlags usage,
+                                    bool create_storage_view) {
     ImageAlloc alloc{};
     alloc.format = format;
     alloc.levels = std::bit_width(std::max(width, height));
@@ -134,9 +139,13 @@ ImageAlloc TextureRuntime::Allocate(u32 width, u32 height, VideoCore::TextureTyp
         return alloc;
     }
 
-    const vk::ImageCreateFlags flags = type == VideoCore::TextureType::CubeMap
+    vk::ImageCreateFlags flags;
-                                           ? vk::ImageCreateFlagBits::eCubeCompatible
+    if (type == VideoCore::TextureType::CubeMap) {
-                                           : vk::ImageCreateFlags{};
+        flags |= vk::ImageCreateFlagBits::eCubeCompatible;
+    }
+    if (create_storage_view) {
+        flags |= vk::ImageCreateFlagBits::eMutableFormat;
+    }
 
     const vk::ImageCreateInfo image_info = {.flags = flags,
                                             .imageType = vk::ImageType::e2D,
@@ -207,6 +216,19 @@ ImageAlloc TextureRuntime::Allocate(u32 width, u32 height, VideoCore::TextureTyp
         alloc.stencil_view = device.createImageView(view_info);
     }
 
+    if (create_storage_view) {
+        const vk::ImageViewCreateInfo storage_view_info = {
+            .image = alloc.image,
+            .viewType = view_type,
+            .format = vk::Format::eR32Uint,
+            .subresourceRange = {.aspectMask = alloc.aspect,
+                                 .baseMipLevel = 0,
+                                 .levelCount = alloc.levels,
+                                 .baseArrayLayer = 0,
+                                 .layerCount = alloc.layers}};
+        alloc.storage_view = device.createImageView(storage_view_info);
+    }
+
     return alloc;
 }
 

src/video_core/renderer_vulkan/vk_texture_runtime.h

@@ -39,6 +39,7 @@ struct ImageAlloc {
     vk::ImageView base_view;
     vk::ImageView depth_view;
     vk::ImageView stencil_view;
+    vk::ImageView storage_view;
     VmaAllocation allocation;
     vk::ImageUsageFlags usage;
     vk::Format format;
@@ -99,7 +100,8 @@ public:
 
     /// Allocates a vulkan image
     [[nodiscard]] ImageAlloc Allocate(u32 width, u32 height, VideoCore::TextureType type,
-                                      vk::Format format, vk::ImageUsageFlags usage);
+                                      vk::Format format, vk::ImageUsageFlags usage,
+                                      bool create_storage_view = false);
 
     /// Causes a GPU command flush
     void Finish();
@@ -204,6 +206,12 @@ public:
         return alloc.stencil_view;
     }
 
+    /// Returns the R32 image view used for atomic load/store
+    vk::ImageView GetStorageView() const {
+        ASSERT(alloc.storage_view);
+        return alloc.storage_view;
+    }
+
     /// Returns the internal format of the allocated texture
     vk::Format GetInternalFormat() const {
         return alloc.format;