custom_tex_manager: Fix dumping issues

* Use the proper hash for dumping

* Add the mipmap as a postfix to help pack creators

src/video_core/rasterizer_cache/custom_tex_manager.cpp

@@ -123,8 +123,23 @@ void CustomTexManager::FindCustomTextures() {
     textures_loaded = true;
 }
 
-void CustomTexManager::DumpTexture(const SurfaceParams& params, std::span<const u8> data) {
-    const u64 data_hash = ComputeHash64(data.data(), data.size());
+u64 CustomTexManager::ComputeHash(const SurfaceParams& params, std::span<u8> data) {
+    const u32 decoded_size = params.width * params.height * GetBytesPerPixel(params.pixel_format);
+    if (temp_buffer.size() < decoded_size) {
+        temp_buffer.resize(decoded_size);
+    }
+
+    // This is suboptimal as we could just hash the 3DS data instead.
+    // However in the interest of compatibility with old texture packs
+    // this must be done...
+    const auto decoded = std::span{temp_buffer.data(), decoded_size};
+    DecodeTexture(params, params.addr, params.end, data, decoded);
+
+    return ComputeHash64(decoded.data(), decoded_size);
+}
+
+void CustomTexManager::DumpTexture(const SurfaceParams& params, u32 level, std::span<u8> data) {
+    const u64 data_hash = ComputeHash(params, data);
     const u32 data_size = static_cast<u32>(data.size());
     const u32 width = params.width;
     const u32 height = params.height;
@@ -150,7 +165,7 @@ void CustomTexManager::DumpTexture(const SurfaceParams& params, std::span<const
 
     // Proceed with the dump.
     const u64 program_id = system.Kernel().GetCurrentProcess()->codeset->program_id;
-    auto dump = [width, height, params, data_hash, format, data_size, program_id,
+    auto dump = [width, height, params, level, data_hash, format, data_size, program_id,
                  pixels = std::move(pixels)]() mutable {
         // Decode and convert to RGBA8
         const std::span encoded = pixels.Span().first(data_size);
@@ -165,7 +180,8 @@ void CustomTexManager::DumpTexture(const SurfaceParams& params, std::span<const
             return;
         }
 
-        dump_path += fmt::format("tex1_{}x{}_{:016X}_{}.png", width, height, data_hash, format);
+        dump_path +=
+            fmt::format("tex1_{}x{}_{:016X}_{}_mip{}.png", width, height, data_hash, format, level);
         EncodePNG(dump_path, decoded, width, height);
     };
 
@@ -173,30 +189,14 @@ void CustomTexManager::DumpTexture(const SurfaceParams& params, std::span<const
     dumped_textures.insert(data_hash);
 }
 
-const Texture& CustomTexManager::GetTexture(const SurfaceParams& params, std::span<u8> data) {
-    u64 data_hash;
-    if (compatibility_mode) {
-        const u32 decoded_size =
-            params.width * params.height * GetBytesPerPixel(params.pixel_format);
-        ScratchBuffer<u8> decoded(decoded_size);
-        DecodeTexture(params, params.addr, params.end, data, decoded.Span());
-        data_hash = ComputeHash64(decoded.Data(), decoded_size);
-    } else {
-        data_hash = ComputeHash64(data.data(), data.size());
-    }
-
+const Texture& CustomTexManager::GetTexture(u64 data_hash) {
     auto it = custom_textures.find(data_hash);
     if (it == custom_textures.end()) {
-        LOG_WARNING(
-            Render, "Unable to find replacement for {}x{} {} surface upload with hash {:016X}",
-            params.width, params.height, PixelFormatAsString(params.pixel_format), data_hash);
+        LOG_WARNING(Render, "Unable to find replacement for surface with hash {:016X}", data_hash);
         return dummy_texture;
     }
 
-    LOG_DEBUG(Render, "Assigning {} to {}x{} {} surface with address {:#x} and hash {:016X}",
-              it->second.path, params.width, params.height,
-              PixelFormatAsString(params.pixel_format), params.addr, data_hash);
-
+    LOG_DEBUG(Render, "Assigning {} to surface with hash {:016X}", it->second.path, data_hash);
     return it->second;
 }
 

src/video_core/rasterizer_cache/custom_tex_manager.h

@@ -49,11 +49,14 @@ public:
     /// Searches the load directory assigned to program_id for any custom textures and loads them
     void FindCustomTextures();
 
+    /// Returns a unique indentifier for a 3DS texture
+    u64 ComputeHash(const SurfaceParams& params, std::span<u8> data);
+
     /// Saves the provided pixel data described by params to disk as png
-    void DumpTexture(const SurfaceParams& params, std::span<const u8> data);
+    void DumpTexture(const SurfaceParams& params, u32 level, std::span<u8> data);
 
     /// Returns the custom texture handle assigned to the provided data hash
-    const Texture& GetTexture(const SurfaceParams& params, std::span<u8> data);
+    const Texture& GetTexture(u64 data_hash);
 
     /// Decodes the data in texture to a consumable format
     void DecodeToStaging(const Texture& texture, const StagingData& staging);
@@ -71,6 +74,7 @@ private:
     Common::ThreadWorker workers;
     std::unordered_set<u64> dumped_textures;
     std::unordered_map<u64, Texture> custom_textures;
+    std::vector<u8> temp_buffer;
     Texture dummy_texture{};
     bool textures_loaded{};
     bool compatibility_mode{true};

src/video_core/rasterizer_cache/rasterizer_cache.h

@@ -883,8 +883,10 @@ void RasterizerCache<T>::UploadSurface(const Surface& surface, SurfaceInterval i
     }
 
     const auto upload_data = source_ptr.GetWriteBytes(load_info.end - load_info.addr);
+
+    // Check if we need to dump the texture
     if (dump_textures) {
-        custom_tex_manager.DumpTexture(load_info, upload_data);
+        custom_tex_manager.DumpTexture(load_info, surface->LevelOf(load_info.addr), upload_data);
     }
 
     // Check if we need to replace the texture
@@ -914,22 +916,15 @@ bool RasterizerCache<T>::UploadCustomSurface(const Surface& surface, const Surfa
                                              std::span<u8> upload_data) {
     const u32 level = surface->LevelOf(load_info.addr);
     const bool is_base_level = level == 0;
-    const Texture& texture = custom_tex_manager.GetTexture(load_info, upload_data);
+    const u64 hash = custom_tex_manager.ComputeHash(load_info, upload_data);
+    const Texture& texture = custom_tex_manager.GetTexture(hash);
 
-    if (custom_tex_manager.CompatibilityMode() && !is_base_level) {
-        // Pack provides mipmap but not the base level. Fallback to normal upload
-        if (!surface->IsCustom() && texture) {
-            return false;
-        }
-
-        // Pack provides base level but not any mipmaps.
-        // We can't fallback to normal upload so ignore it.
-        // The base level should already have generated mips for us.
-        if (surface->IsCustom() && !texture) {
-            return true;
-        }
+    // The old texture pack system did not support mipmaps so older packs might do
+    // wonky things. For example Henriko's pack has mipmaps larger than the base
+    // level. To avoid crashes just don't upload mipmaps for custom surfaces
+    if (custom_tex_manager.CompatibilityMode() && surface->IsCustom() && !is_base_level) {
+        return true;
     }
-
     if (!texture) {
         return false;
     }