From aa79505ddd64ac8bd48eb0e1a26f662080cb3b22 Mon Sep 17 00:00:00 2001 From: GPUCode Date: Mon, 13 Nov 2023 10:05:08 +0200 Subject: [PATCH] custom_tex_manager: Implement hot-reloading --- src/common/file_util.cpp | 1 - src/common/file_watcher.cpp | 36 ++++++++------ src/common/file_watcher.h | 7 +-- .../custom_textures/custom_tex_manager.cpp | 47 +++++++++++++++---- .../custom_textures/custom_tex_manager.h | 33 +++++++++---- src/video_core/custom_textures/material.h | 3 ++ .../rasterizer_cache/rasterizer_cache.h | 2 +- .../rasterizer_cache/surface_base.cpp | 13 ++++- .../rasterizer_cache/surface_base.h | 3 +- .../renderer_opengl/gl_texture_runtime.cpp | 5 +- .../renderer_vulkan/renderer_vulkan.cpp | 3 +- .../renderer_vulkan/vk_texture_runtime.cpp | 5 +- 12 files changed, 109 insertions(+), 49 deletions(-) diff --git a/src/common/file_util.cpp b/src/common/file_util.cpp index 1d8b2c509..09592a707 100644 --- a/src/common/file_util.cpp +++ b/src/common/file_util.cpp @@ -534,7 +534,6 @@ u64 ScanDirectoryTree(const std::string& directory, FSTEntry& parent_entry, } void GetAllFilesFromNestedEntries(FSTEntry& directory, std::vector& output) { - std::vector files; for (auto& entry : directory.children) { if (entry.isDirectory) { GetAllFilesFromNestedEntries(entry, output); diff --git a/src/common/file_watcher.cpp b/src/common/file_watcher.cpp index 6bdac82f9..58c2e1812 100644 --- a/src/common/file_watcher.cpp +++ b/src/common/file_watcher.cpp @@ -19,8 +19,9 @@ static FileAction Win32ActionToFileAction(DWORD action) { case FILE_ACTION_MODIFIED: return FileAction::Modified; case FILE_ACTION_RENAMED_OLD_NAME: + return FileAction::RenamedOldName; case FILE_ACTION_RENAMED_NEW_NAME: - return FileAction::Renamed; + return FileAction::RenamedNewName; default: UNREACHABLE_MSG("Unknown action {}", action); return FileAction::Invalid; @@ -28,11 +29,11 @@ static FileAction Win32ActionToFileAction(DWORD action) { } struct FileWatcher::Impl { - explicit Impl(const std::wstring& path, FileWatcher::Callback&& callback_) + explicit Impl(const std::string& path, FileWatcher::Callback&& callback_) : callback{callback_} { // Create file handle for the directory we are watching. dir_handle = - CreateFileW(path.c_str(), FILE_LIST_DIRECTORY | GENERIC_READ, + CreateFile(path.c_str(), FILE_LIST_DIRECTORY | GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, NULL, OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OVERLAPPED, NULL); ASSERT_MSG(dir_handle != INVALID_HANDLE_VALUE, "Unable to create watch file"); @@ -70,6 +71,8 @@ struct FileWatcher::Impl { while (is_running) { bool result = ReadDirectoryChangesW(dir_handle, buffer.data(), buffer.size(), TRUE, + FILE_NOTIFY_CHANGE_FILE_NAME | + FILE_NOTIFY_CHANGE_DIR_NAME | FILE_NOTIFY_CHANGE_LAST_WRITE, NULL, &overlapped, NULL); ASSERT_MSG(result, "Unable to read directory changes: {}", GetLastErrorMsg()); @@ -101,25 +104,28 @@ struct FileWatcher::Impl { return; } - size_t next_entry_offset{}; - FILE_NOTIFY_INFORMATION fni; - do { + u32 next_entry_offset{}; + while (true) { // Retrieve file notify information. - std::memcpy(&fni, buffer.data() + next_entry_offset, sizeof(fni)); - next_entry_offset += fni.NextEntryOffset; + auto fni = reinterpret_cast(buffer.data() + next_entry_offset); // Call the callback function informing about the change. - if (fni.Action != 0) { - const std::wstring file_name{fni.FileName, fni.FileNameLength}; - const FileAction action = Win32ActionToFileAction(fni.Action); + if (fni->Action != 0) { + std::string file_name(fni->FileNameLength / sizeof(WCHAR), ' '); + WideCharToMultiByte(CP_UTF8, 0, fni->FileName, + fni->FileNameLength / sizeof(WCHAR), file_name.data(), file_name.size(), NULL, NULL); + const FileAction action = Win32ActionToFileAction(fni->Action); callback(file_name, action); } // If this was the last action, break. - if (fni.NextEntryOffset == 0) { + if (fni->NextEntryOffset == 0) { break; } - } while (true); + + // Move to next fni structure. + next_entry_offset += fni->NextEntryOffset; + } } private: @@ -134,7 +140,9 @@ private: std::thread watch_thread; }; -FileWatcher::FileWatcher(const std::wstring& log_dir, Callback&& callback) +FileWatcher::FileWatcher(const std::string& log_dir, Callback&& callback) : impl{std::make_unique(log_dir, std::move(callback))} {} +FileWatcher::~FileWatcher() = default; + } // namespace Common diff --git a/src/common/file_watcher.h b/src/common/file_watcher.h index 7acb57888..369194000 100644 --- a/src/common/file_watcher.h +++ b/src/common/file_watcher.h @@ -13,15 +13,16 @@ enum class FileAction : u8 { Added, Removed, Modified, - Renamed, + RenamedOldName, + RenamedNewName, Invalid = std::numeric_limits::max(), }; class FileWatcher { - using Callback = std::function; + using Callback = std::function; public: - explicit FileWatcher(const std::wstring& log_dir, Callback&& callback); + explicit FileWatcher(const std::string& log_dir, Callback&& callback); ~FileWatcher(); private: diff --git a/src/video_core/custom_textures/custom_tex_manager.cpp b/src/video_core/custom_textures/custom_tex_manager.cpp index 18d69e129..4a39afac7 100644 --- a/src/video_core/custom_textures/custom_tex_manager.cpp +++ b/src/video_core/custom_textures/custom_tex_manager.cpp @@ -1,9 +1,10 @@ // Copyright 2023 Citra Emulator Project // Licensed under GPLv2 or any later version // Refer to the license.txt file included. - +#pragma clang optimize off #include #include "common/file_util.h" +#include "common/file_watcher.h" #include "common/memory_detect.h" #include "common/microprofile.h" #include "common/settings.h" @@ -16,6 +17,7 @@ #include "video_core/custom_textures/custom_tex_manager.h" #include "video_core/rasterizer_cache/surface_params.h" #include "video_core/rasterizer_cache/utils.h" +#include "video_core/renderer_base.h" namespace VideoCore { @@ -63,17 +65,17 @@ void CustomTexManager::TickFrame() { return; } std::size_t num_uploads = 0; - for (auto it = async_uploads.begin(); it != async_uploads.end();) { + for (auto it = async_actions.begin(); it != async_actions.end();) { if (num_uploads >= MAX_UPLOADS_PER_TICK) { return; } switch (it->material->state) { case DecodeState::Decoded: - it->func(); + it->func(it->material); num_uploads++; [[fallthrough]]; case DecodeState::Failed: - it = async_uploads.erase(it); + it = async_actions.erase(it); continue; default: it++; @@ -102,7 +104,7 @@ void CustomTexManager::FindCustomTextures() { if (file.isDirectory) { continue; } - custom_textures.push_back(std::make_unique(image_interface)); + custom_textures.emplace_back(std::make_unique(image_interface)); CustomTexture* const texture{custom_textures.back().get()}; if (!ParseFilename(file, texture)) { continue; @@ -292,16 +294,17 @@ Material* CustomTexManager::GetMaterial(u64 data_hash) { return it->second.get(); } -bool CustomTexManager::Decode(Material* material, std::function&& upload) { +bool CustomTexManager::Decode(Material* material, AsyncFunc&& upload) { if (!async_custom_loading) { material->LoadFromDisk(flip_png_files); - return upload(); + return upload(material); } if (material->IsUnloaded()) { material->state = DecodeState::Pending; workers->QueueWork([material, this] { material->LoadFromDisk(flip_png_files); }); } - async_uploads.push_back({ + std::scoped_lock lock{async_actions_mutex}; + async_actions.push_back({ .material = material, .func = std::move(upload), }); @@ -374,6 +377,14 @@ std::vector CustomTexManager::GetTextures(u64 title_id) { FileUtil::CreateFullPath(load_path); } + const auto callback = [this](const std::string& file, Common::FileAction action) { + OnFileAction(file, action); + }; + + // Create a file watcher to monitor any changes to the textures directory for hot-reloading. + file_watcher = std::make_unique(load_path, callback); + + // Retrieve all texture files. FileUtil::FSTEntry texture_dir; std::vector textures; FileUtil::ScanDirectoryTree(load_path, texture_dir, 64); @@ -386,4 +397,24 @@ void CustomTexManager::CreateWorkers() { workers = std::make_unique(num_workers, "Custom textures"); } +void CustomTexManager::OnFileAction(const std::string& file, Common::FileAction action) { + const auto invalidate = [this](const Material* material) -> bool { + for (const SurfaceParams* params : material->loaded_to) { + system.Renderer().Rasterizer()->InvalidateRegion(params->addr, params->size); + } + return true; + }; + + const std::string filename{FileUtil::GetFilename(file)}; + const auto& hashes = path_to_hash_map[filename]; + std::scoped_lock lock{async_actions_mutex}; + + for (const Hash hash : hashes) { + async_actions.push_back({ + .material = material_map[hash].get(), + .func = std::move(invalidate), + }); + } +} + } // namespace VideoCore diff --git a/src/video_core/custom_textures/custom_tex_manager.h b/src/video_core/custom_textures/custom_tex_manager.h index ba8a6b5dd..1a0c641cc 100644 --- a/src/video_core/custom_textures/custom_tex_manager.h +++ b/src/video_core/custom_textures/custom_tex_manager.h @@ -12,6 +12,11 @@ #include "video_core/custom_textures/material.h" #include "video_core/rasterizer_interface.h" +namespace Common { +class FileWatcher; +enum class FileAction : u8; +} // namespace Common + namespace Core { class System; } @@ -24,12 +29,10 @@ namespace VideoCore { class SurfaceParams; -struct AsyncUpload { - const Material* material; - std::function func; -}; - class CustomTexManager { + using Hash = u64; + using AsyncFunc = std::function; + public: explicit CustomTexManager(Core::System& system); ~CustomTexManager(); @@ -57,7 +60,7 @@ public: Material* GetMaterial(u64 data_hash); /// Decodes the textures in material to a consumable format and uploads it. - bool Decode(Material* material, std::function&& upload); + bool Decode(Material* material, AsyncFunc&& func); /// True when mipmap uploads should be skipped (legacy packs only) bool SkipMipmaps() const noexcept { @@ -79,15 +82,25 @@ private: /// Creates the thread workers. void CreateWorkers(); + /// Callback for when a custom texture file is modified. + void OnFileAction(const std::string& file, Common::FileAction action); + private: + struct AsyncAction { + const Material* material; + AsyncFunc func; + }; + Core::System& system; Frontend::ImageInterface& image_interface; - std::unordered_set dumped_textures; - std::unordered_map> material_map; - std::unordered_map> path_to_hash_map; + std::unordered_set dumped_textures; + std::unordered_map> material_map; + std::unordered_map> path_to_hash_map; std::vector> custom_textures; - std::list async_uploads; + std::mutex async_actions_mutex; + std::list async_actions; std::unique_ptr workers; + std::unique_ptr file_watcher; bool textures_loaded{false}; bool async_custom_loading{true}; bool skip_mipmap{false}; diff --git a/src/video_core/custom_textures/material.h b/src/video_core/custom_textures/material.h index 69d6a838c..8bd3316bb 100644 --- a/src/video_core/custom_textures/material.h +++ b/src/video_core/custom_textures/material.h @@ -18,6 +18,8 @@ class ImageInterface; namespace VideoCore { +class SurfaceParams; + enum class MapType : u32 { Color = 0, Normal = 1, @@ -72,6 +74,7 @@ struct Material { u64 hash; CustomPixelFormat format; std::array textures; + mutable std::vector loaded_to; std::atomic state{}; void LoadFromDisk(bool flip_png) noexcept; diff --git a/src/video_core/rasterizer_cache/rasterizer_cache.h b/src/video_core/rasterizer_cache/rasterizer_cache.h index efb3eeb58..e6bc7280c 100644 --- a/src/video_core/rasterizer_cache/rasterizer_cache.h +++ b/src/video_core/rasterizer_cache/rasterizer_cache.h @@ -1043,7 +1043,7 @@ bool RasterizerCache::UploadCustomSurface(SurfaceId surface_id, SurfaceInterv surface.flags |= SurfaceFlagBits::Custom; - const auto upload = [this, level, surface_id, material]() -> bool { + const auto upload = [this, level, surface_id](const Material* material) -> bool { ASSERT_MSG(True(slot_surfaces[surface_id].flags & SurfaceFlagBits::Custom), "Surface is not suitable for custom upload, aborting!"); if (!slot_surfaces[surface_id].IsCustom()) { diff --git a/src/video_core/rasterizer_cache/surface_base.cpp b/src/video_core/rasterizer_cache/surface_base.cpp index 8d310dfe4..e33204a24 100644 --- a/src/video_core/rasterizer_cache/surface_base.cpp +++ b/src/video_core/rasterizer_cache/surface_base.cpp @@ -11,7 +11,18 @@ namespace VideoCore { SurfaceBase::SurfaceBase(const SurfaceParams& params) : SurfaceParams{params} {} -SurfaceBase::~SurfaceBase() = default; +SurfaceBase::SurfaceBase(const SurfaceParams& params, const Material* mat) + : SurfaceParams{params}, material{mat} { + custom_format = material->format; + material->loaded_to.push_back(this); +} + +SurfaceBase::~SurfaceBase() { + if (!material) { + return; + } + std::erase_if(material->loaded_to, [this](SurfaceParams* params) { return params == this; }); +} bool SurfaceBase::CanFill(const SurfaceParams& dest_surface, SurfaceInterval fill_interval) const { if (type == SurfaceType::Fill && IsRegionValid(fill_interval) && diff --git a/src/video_core/rasterizer_cache/surface_base.h b/src/video_core/rasterizer_cache/surface_base.h index e786a82b0..8b5b9ddee 100644 --- a/src/video_core/rasterizer_cache/surface_base.h +++ b/src/video_core/rasterizer_cache/surface_base.h @@ -26,7 +26,8 @@ DECLARE_ENUM_FLAG_OPERATORS(SurfaceFlagBits); class SurfaceBase : public SurfaceParams { public: - SurfaceBase(const SurfaceParams& params); + explicit SurfaceBase(const SurfaceParams& params); + explicit SurfaceBase(const SurfaceParams& params, const Material* mat); ~SurfaceBase(); /// Returns true when this surface can be used to fill the fill_interval of dest_surface diff --git a/src/video_core/renderer_opengl/gl_texture_runtime.cpp b/src/video_core/renderer_opengl/gl_texture_runtime.cpp index f24fe10ac..e8ee07060 100644 --- a/src/video_core/renderer_opengl/gl_texture_runtime.cpp +++ b/src/video_core/renderer_opengl/gl_texture_runtime.cpp @@ -333,7 +333,7 @@ Surface::Surface(TextureRuntime& runtime_, const VideoCore::SurfaceParams& param Surface::Surface(TextureRuntime& runtime, const VideoCore::SurfaceBase& surface, const VideoCore::Material* mat) - : SurfaceBase{surface}, tuple{runtime.GetFormatTuple(mat->format)} { + : SurfaceBase{surface, mat}, tuple{runtime.GetFormatTuple(mat->format)} { if (mat && !driver->IsCustomFormatSupported(mat->format)) { return; } @@ -342,9 +342,6 @@ Surface::Surface(TextureRuntime& runtime, const VideoCore::SurfaceBase& surface, const GLenum target = texture_type == VideoCore::TextureType::CubeMap ? GL_TEXTURE_CUBE_MAP : GL_TEXTURE_2D; - custom_format = mat->format; - material = mat; - textures[0] = MakeHandle(target, mat->width, mat->height, levels, tuple, DebugName(false)); if (res_scale != 1) { textures[1] = MakeHandle(target, mat->width, mat->height, levels, DEFAULT_TUPLE, diff --git a/src/video_core/renderer_vulkan/renderer_vulkan.cpp b/src/video_core/renderer_vulkan/renderer_vulkan.cpp index 5986b3b7e..becb766c2 100644 --- a/src/video_core/renderer_vulkan/renderer_vulkan.cpp +++ b/src/video_core/renderer_vulkan/renderer_vulkan.cpp @@ -3,10 +3,10 @@ // Refer to the license.txt file included. #include "common/assert.h" +#include "common/file_watcher.h" #include "common/logging/log.h" #include "common/microprofile.h" #include "common/settings.h" -#include "common/texture.h" #include "core/core.h" #include "core/frontend/emu_window.h" #include "core/hw/gpu.h" @@ -19,7 +19,6 @@ #include "video_core/host_shaders/vulkan_present_frag_spv.h" #include "video_core/host_shaders/vulkan_present_interlaced_frag_spv.h" #include "video_core/host_shaders/vulkan_present_vert_spv.h" -#include "vulkan/vulkan_format_traits.hpp" #include diff --git a/src/video_core/renderer_vulkan/vk_texture_runtime.cpp b/src/video_core/renderer_vulkan/vk_texture_runtime.cpp index bff700969..8a37e87ed 100644 --- a/src/video_core/renderer_vulkan/vk_texture_runtime.cpp +++ b/src/video_core/renderer_vulkan/vk_texture_runtime.cpp @@ -750,7 +750,7 @@ Surface::Surface(TextureRuntime& runtime_, const VideoCore::SurfaceParams& param Surface::Surface(TextureRuntime& runtime_, const VideoCore::SurfaceBase& surface, const VideoCore::Material* mat) - : SurfaceBase{surface}, runtime{&runtime_}, instance{&runtime_.GetInstance()}, + : SurfaceBase{surface, mat}, runtime{&runtime_}, instance{&runtime_.GetInstance()}, scheduler{&runtime_.GetScheduler()}, traits{instance->GetTraits(mat->format)} { if (!traits.transfer_support) { return; @@ -791,9 +791,6 @@ Surface::Surface(TextureRuntime& runtime_, const VideoCore::SurfaceBase& surface vk::PipelineStageFlagBits::eTopOfPipe, vk::DependencyFlagBits::eByRegion, {}, {}, barriers); }); - - custom_format = mat->format; - material = mat; } Surface::~Surface() {