rasterizer_cache: Add support for texture dumping

src/common/CMakeLists.txt

@@ -66,6 +66,7 @@ add_library(common STATIC
     detached_tasks.h
     bit_field.h
     bit_set.h
+    bit_util.h
     cityhash.cpp
     cityhash.h
     color.h
@@ -110,6 +111,7 @@ add_library(common STATIC
     scm_rev.cpp
     scm_rev.h
     scope_exit.h
+    scratch_buffer.h
     settings.cpp
     settings.h
     serialization/atomic.h
@@ -145,8 +147,8 @@ add_library(common STATIC
 
 create_target_directory_groups(common)
 
-target_link_libraries(common PUBLIC fmt::fmt microprofile Boost::boost Boost::serialization)
+target_link_libraries(common PUBLIC fmt::fmt microprofile Boost::boost Boost::serialization spng_static)
-target_link_libraries(common PRIVATE libzstd_static spng_static)
+target_link_libraries(common PRIVATE libzstd_static)
 set_target_properties(common PROPERTIES INTERPROCEDURAL_OPTIMIZATION ${ENABLE_LTO})
 
 if ("x86_64" IN_LIST ARCHITECTURE)

src/common/bit_util.h

@@ -0,0 +1,66 @@
+// SPDX-FileCopyrightText: Copyright 2018 yuzu Emulator Project
+// SPDX-License-Identifier: GPL-2.0-or-later
+
+#pragma once
+
+#include <bit>
+#include <climits>
+#include <cstddef>
+#include <type_traits>
+
+#include "common/common_types.h"
+
+namespace Common {
+
+/// Gets the size of a specified type T in bits.
+template <typename T>
+[[nodiscard]] constexpr std::size_t BitSize() {
+    return sizeof(T) * CHAR_BIT;
+}
+
+[[nodiscard]] constexpr u32 MostSignificantBit32(const u32 value) {
+    return 31U - static_cast<u32>(std::countl_zero(value));
+}
+
+[[nodiscard]] constexpr u32 MostSignificantBit64(const u64 value) {
+    return 63U - static_cast<u32>(std::countl_zero(value));
+}
+
+[[nodiscard]] constexpr u32 Log2Floor32(const u32 value) {
+    return MostSignificantBit32(value);
+}
+
+[[nodiscard]] constexpr u32 Log2Floor64(const u64 value) {
+    return MostSignificantBit64(value);
+}
+
+[[nodiscard]] constexpr u32 Log2Ceil32(const u32 value) {
+    const u32 log2_f = Log2Floor32(value);
+    return log2_f + static_cast<u32>((value ^ (1U << log2_f)) != 0U);
+}
+
+[[nodiscard]] constexpr u32 Log2Ceil64(const u64 value) {
+    const u64 log2_f = Log2Floor64(value);
+    return static_cast<u32>(log2_f + static_cast<u64>((value ^ (1ULL << log2_f)) != 0ULL));
+}
+
+template <typename T>
+    requires std::is_unsigned_v<T>
+[[nodiscard]] constexpr bool IsPow2(T value) {
+    return std::has_single_bit(value);
+}
+
+template <typename T>
+    requires std::is_integral_v<T>
+[[nodiscard]] T NextPow2(T value) {
+    return static_cast<T>(1ULL << ((8U * sizeof(T)) - std::countl_zero(value - 1U)));
+}
+
+template <size_t bit_index, typename T>
+    requires std::is_integral_v<T>
+[[nodiscard]] constexpr bool Bit(const T value) {
+    static_assert(bit_index < BitSize<T>(), "bit_index must be smaller than size of T");
+    return ((value >> bit_index) & T(1)) == T(1);
+}
+
+} // namespace Common

src/common/image_util.cpp

@@ -57,7 +57,7 @@ bool DecodePNG(std::span<const u8> in_data, std::vector<u8>& out_data, u32& widt
     return true;
 }
 
-bool EncodePNG(std::span<u8> in_data, const std::string& out_path, u32 width, u32 height,
+bool EncodePNG(std::span<const u8> in_data, const std::string& out_path, u32 width, u32 height,
                u32 stride, s32 level) {
     auto ctx = make_spng_ctx(SPNG_CTX_ENCODER);
     if (!ctx) [[unlikely]] {

src/common/image_util.h

@@ -19,7 +19,7 @@ namespace Common {
  */
 bool DecodePNG(std::span<const u8> in_data, std::vector<u8>& out_data, u32& width, u32& height);
 
-bool EncodePNG(std::span<u8> in_data, const std::string& out_path, u32 width, u32 height,
+bool EncodePNG(std::span<const u8> in_data, const std::string& out_path, u32 width, u32 height,
                u32 stride, s32 level);
 
 } // namespace Common

src/common/scratch_buffer.h

@@ -0,0 +1,40 @@
+// Copyright 2023 Citra Emulator Project
+// Licensed under GPLv2 or any later version
+// Refer to the license.txt file included.
+
+#pragma once
+
+#include <type_traits>
+#include <span>
+#include <memory>
+#include "common/common_types.h"
+
+namespace Common {
+
+template <typename T, u32 alignment = 0>
+class ScratchBuffer {
+    static_assert(std::is_trivial_v<T>, "Must use a POD type");
+
+public:
+    ScratchBuffer(std::size_t size_) : size{size_} {
+        buffer = std::unique_ptr<T>(new typename std::remove_extent<T>::type[size]);
+    }
+
+    [[nodiscard]] std::size_t Size() const noexcept {
+        return size;
+    }
+
+    [[nodiscard]] T* Data() const noexcept {
+        return buffer.get();
+    }
+
+    [[nodiscard]] std::span<const T> Span() const noexcept {
+        return std::span<const T>{buffer.get(), size};
+    }
+
+private:
+    std::unique_ptr<T> buffer;
+    std::size_t size;
+};
+
+} // namespace Common

src/video_core/CMakeLists.txt

@@ -28,10 +28,10 @@ add_library(video_core STATIC
     regs_texturing.h
     renderer_base.cpp
     renderer_base.h
-    rasterizer_cache/custom/custom_tex_manager.cpp
-    rasterizer_cache/custom/custom_tex_manager.h
     rasterizer_cache/framebuffer_base.cpp
     rasterizer_cache/framebuffer_base.h
+    rasterizer_cache/hires_replacer.cpp
+    rasterizer_cache/hires_replacer.h
     rasterizer_cache/pixel_format.cpp
     rasterizer_cache/pixel_format.h
     rasterizer_cache/rasterizer_cache.cpp

src/video_core/rasterizer_cache/custom/custom_tex_manager.cpp

@@ -1 +0,0 @@
-

src/video_core/rasterizer_cache/custom/custom_tex_manager.h

@@ -1,53 +0,0 @@
-// Copyright 2023 Citra Emulator Project
-// Licensed under GPLv2 or any later version
-// Refer to the license.txt file included.
-
-#pragma once
-
-#include <string>
-#include <unordered_map>
-#include <unordered_set>
-#include <vector>
-#include "common/common_types.h"
-
-namespace VideoCore {
-
-struct CustomTexture {
-    u32 width;
-    u32 height;
-    u32 levels;
-    std::vector<u8> pixels;
-};
-
-// This is to avoid parsing the filename multiple times
-struct CustomTexPathInfo {
-    std::string path;
-    u64 hash;
-};
-
-// TODO: think of a better name for this class...
-class CustomTexManager {
-public:
-    explicit CustomTexManager();
-    ~CustomTexManager();
-
-    bool IsTextureDumped(u64 hash) const;
-    void SetTextureDumped(u64 hash);
-
-    bool IsTextureCached(u64 hash) const;
-    const CustomTexture& LookupTexture(u64 hash) const;
-    void CacheTexture(u64 hash, const std::vector<u8>& tex, u32 width, u32 height);
-
-    void AddTexturePath(u64 hash, const std::string& path);
-    void FindCustomTextures(u64 program_id);
-    void PreloadTextures(Frontend::ImageInterface& image_interface);
-    bool CustomTextureExists(u64 hash) const;
-    const CustomTexPathInfo& LookupTexturePathInfo(u64 hash) const;
-    bool IsTexturePathMapEmpty() const;
-
-private:
-    std::unordered_set<u64> dumped_textures;
-    std::unordered_map<u64, CustomTexInfo> custom_textures;
-    std::unordered_map<u64, CustomTexPathInfo> custom_texture_paths;
-};
-} // namespace VideoCore

src/video_core/rasterizer_cache/hires_replacer.cpp

@@ -0,0 +1,66 @@
+// Copyright 2023 Citra Emulator Project
+// Licensed under GPLv2 or any later version
+// Refer to the license.txt file included.
+
+#include "common/bit_util.h"
+#include "common/file_util.h"
+#include "common/scratch_buffer.h"
+#include "common/image_util.h"
+#include "core/core.h"
+#include "video_core/rasterizer_cache/hires_replacer.h"
+#include "video_core/rasterizer_cache/surface_base.h"
+
+namespace VideoCore {
+
+HiresReplacer::HiresReplacer() :
+    workers{std::max(std::thread::hardware_concurrency(), 2U) - 1, "Hires processing"} {
+
+}
+
+void HiresReplacer::DumpSurface(const SurfaceBase& surface, std::span<const u8> data) {
+    const u32 data_hash = Common::ComputeHash64(data.data(), data.size());
+    const u32 width = surface.width;
+    const u32 height = surface.height;
+    const PixelFormat format = surface.pixel_format;
+
+    // Check if it's been dumped already
+    if (dumped_surfaces.contains(data_hash)) {
+        return;
+    }
+
+    // If this is a partial update do not dump it, it's probably not a texture
+    if (surface.BytesInPixels(width * height) != data.size()) {
+        LOG_WARNING(Render, "Not dumping {:016X} because it's a partial texture update");
+        return;
+    }
+
+    // Make sure the texture size is a power of 2.
+    // If not, the surface is probably a framebuffer
+    if (!Common::IsPow2(surface.width) || !Common::IsPow2(surface.height)) {
+        LOG_WARNING(Render, "Not dumping {:016X} because size isn't a power of 2 ({}x{})",
+                    data_hash, width, height);
+        return;
+    }
+
+    // Allocate a temporary buffer for the thread to use
+    Common::ScratchBuffer<u8> pixels(data.size());
+    std::memcpy(pixels.Data(), data.data(), data.size());
+
+    // Proceed with the dump. The texture should be already decoded
+    const u64 program_id = Core::System::GetInstance().Kernel().GetCurrentProcess()->codeset->program_id;
+    const auto dump = [width, height, data_hash, format, program_id, pixels = std::move(pixels)]() {
+        std::string dump_path =
+            fmt::format("{}textures/{:016X}/", FileUtil::GetUserPath(FileUtil::UserPath::DumpDir), program_id);
+        if (!FileUtil::CreateFullPath(dump_path)) {
+            LOG_ERROR(Render, "Unable to create {}", dump_path);
+            return;
+        }
+
+        dump_path += fmt::format("tex1_{}x{}_{:016X}_{}.png", width, height, data_hash, format);
+        Common::EncodePNG(pixels.Span(), dump_path, width, height, width, 0);
+    };
+
+    dump();
+}
+
+} // namespace VideoCore

src/video_core/rasterizer_cache/hires_replacer.h

@@ -0,0 +1,29 @@
+// Copyright 2023 Citra Emulator Project
+// Licensed under GPLv2 or any later version
+// Refer to the license.txt file included.
+
+#pragma once
+
+#include <string>
+#include <span>
+#include <unordered_map>
+#include <unordered_set>
+#include <vector>
+#include "common/thread_worker.h"
+
+namespace VideoCore {
+
+class SurfaceBase;
+
+class HiresReplacer {
+public:
+    HiresReplacer();
+
+    void DumpSurface(const SurfaceBase& surface, std::span<const u8> data);
+
+private:
+    Common::ThreadWorker workers;
+    std::unordered_set<u64> dumped_surfaces;
+};
+
+} // namespace VideoCore

src/video_core/rasterizer_cache/rasterizer_cache.h

@@ -28,7 +28,8 @@ inline auto RangeFromInterval(auto& map, const auto& interval) {
 template <class T>
 RasterizerCache<T>::RasterizerCache(Memory::MemorySystem& memory_, Runtime& runtime_)
     : memory{memory_}, runtime{runtime_}, resolution_scale_factor{
-                                              VideoCore::GetResolutionScaleFactor()} {
+                                              VideoCore::GetResolutionScaleFactor()},
+    dump_textures{Settings::values.dump_textures.GetValue()} {
     using TextureConfig = Pica::TexturingRegs::TextureConfig;
 
     // Create null handles for all cached resources
@@ -877,6 +878,10 @@ void RasterizerCache<T>::UploadSurface(const Surface& surface, SurfaceInterval i
     DecodeTexture(load_info, load_info.addr, load_info.end, upload_data, staging.mapped,
                   runtime.NeedsConvertion(surface->pixel_format));
 
+    if (dump_textures) {
+        replacer.DumpSurface(*surface, staging.mapped);
+    }
+
     const BufferTextureCopy upload = {
         .buffer_offset = 0,
         .buffer_size = staging.size,

src/video_core/rasterizer_cache/rasterizer_cache_base.h

@@ -9,6 +9,7 @@
 #include <unordered_map>
 #include <boost/icl/interval_map.hpp>
 #include <boost/range/iterator_range.hpp>
+#include "video_core/rasterizer_cache/hires_replacer.h"
 #include "video_core/rasterizer_cache/sampler_params.h"
 #include "video_core/rasterizer_cache/surface_params.h"
 #include "video_core/rasterizer_cache/utils.h"
@@ -184,6 +185,7 @@ private:
 private:
     Memory::MemorySystem& memory;
     Runtime& runtime;
+    HiresReplacer replacer;
     PageMap cached_pages;
     SurfaceMap dirty_regions;
     std::vector<Surface> remove_surfaces;
@@ -204,6 +206,7 @@ private:
     };
 
     RenderTargets render_targets;
+    const bool& dump_textures;
 };
 
 } // namespace VideoCore