From 67c20e04b42a325076628de548fe4ca47b9271ef Mon Sep 17 00:00:00 2001 From: emufan4568 Date: Wed, 10 Aug 2022 00:03:26 +0300 Subject: [PATCH] shader_disk_cache: Remove precompiled cache * Serves little purpose at this point and complicates things. Maybe readd later? --- src/video_core/common/backend.h | 6 +- src/video_core/common/pipeline_cache.cpp | 58 ++++---- src/video_core/common/rasterizer.h | 4 +- src/video_core/common/shader_disk_cache.cpp | 153 +++----------------- src/video_core/common/shader_disk_cache.h | 27 +--- src/video_core/regs.h | 4 +- 6 files changed, 55 insertions(+), 197 deletions(-) diff --git a/src/video_core/common/backend.h b/src/video_core/common/backend.h index 3f7415db9..e6758c460 100644 --- a/src/video_core/common/backend.h +++ b/src/video_core/common/backend.h @@ -4,7 +4,6 @@ #pragma once -#include "common/object_pool.h" #include "common/vector_math.h" #include "video_core/common/pipeline.h" #include "video_core/common/framebuffer.h" @@ -15,7 +14,9 @@ class EmuWindow; namespace VideoCore { -/// Common interface of a video backend +class ShaderDiskCache; + +// Common interface of a video backend class BackendBase { public: BackendBase(Frontend::EmuWindow& window) : window(window) {} @@ -59,7 +60,6 @@ public: // Executes a compute shader virtual void DispatchCompute(PipelineHandle pipeline, Common::Vec3 groupsize, Common::Vec3 groups) = 0; - private: Frontend::EmuWindow& window; }; diff --git a/src/video_core/common/pipeline_cache.cpp b/src/video_core/common/pipeline_cache.cpp index 51dd9d0f8..1d263c892 100644 --- a/src/video_core/common/pipeline_cache.cpp +++ b/src/video_core/common/pipeline_cache.cpp @@ -126,18 +126,11 @@ void PipelineCache::LoadDiskCache(const std::atomic_bool& stop_loading, const Di // Load uncompressed precompiled file for non-separable shaders. // Precompiled file for separable shaders is compressed. - auto [decompiled, dumps] = disk_cache.LoadPrecompiled(true); - + std::optional decompiled = disk_cache.LoadPrecompiled(); if (stop_loading) { return; } - std::set supported_formats = GetSupportedFormats(); - - // Track if precompiled cache was altered during loading to know if we have to serialize the - // virtual precompiled cache file back to the hard drive - bool precompiled_cache_altered = false; - std::mutex mutex; std::atomic_bool compilation_failed = false; if (callback) { @@ -147,7 +140,7 @@ void PipelineCache::LoadDiskCache(const std::atomic_bool& stop_loading, const Di std::vector load_raws_index; for (u64 i = 0; i < raws.size(); i++) { if (stop_loading || compilation_failed) { - return; + break; } const ShaderDiskCacheRaw& raw = raws[i]; @@ -161,35 +154,42 @@ void PipelineCache::LoadDiskCache(const std::atomic_bool& stop_loading, const Di raw.GetUniqueIdentifier(), calculated_hash); disk_cache.InvalidateAll(); - return; + break; } - const auto dump = dumps.find(unique_identifier); - const auto decomp = decompiled.find(unique_identifier); + const auto iter = decompiled->find(unique_identifier); ShaderHandle shader{}; - if (dump != dumps.end() && decomp != decompiled.end()) { + if (iter != decompiled->end()) { // Only load the vertex shader if its sanitize_mul setting matches + ShaderDiskCacheDecompiled& decomp = iter->second; if (raw.GetProgramType() == ProgramType::VertexShader && - decomp->second.sanitize_mul != VideoCore::g_hw_shader_accurate_mul) { - continue; + decomp.sanitize_mul != VideoCore::g_hw_shader_accurate_mul) { + break; } - // If the shader is dumped, attempt to load it - shader = GeneratePrecompiledProgram(dump->second, supported_formats); - if (!shader.IsValid()) { - // If any shader failed, stop trying to compile, delete the cache, and start - // loading from raws - compilation_failed = true; - return; + ShaderStage stage; + switch (raw.GetProgramType()) { + case ProgramType::VertexShader: + stage = ShaderStage::Vertex; + break; + case ProgramType::GeometryShader: + stage = ShaderStage::Geometry; + break; + case ProgramType::FragmentShader: + stage = ShaderStage::Fragment; + break; } + // Create shader from GLSL source + shader = backend->CreateShader(stage, "Precompiled shader", decomp.result); + // We have both the binary shader and the decompiled, so inject it into the // cache if (raw.GetProgramType() == ProgramType::VertexShader) { auto [conf, setup] = BuildVSConfigFromRaw(raw); std::scoped_lock lock(mutex); - pica_vertex_shaders.Inject(conf, decomp->second.result, std::move(shader)); + pica_vertex_shaders.Inject(conf, decomp.result, std::move(shader)); } else if (raw.GetProgramType() == ProgramType::FragmentShader) { const PicaFSConfig conf{raw.GetRawShaderConfig()}; @@ -200,7 +200,7 @@ void PipelineCache::LoadDiskCache(const std::atomic_bool& stop_loading, const Di // Unsupported shader type got stored somehow so nuke the cache LOG_CRITICAL(Frontend, "failed to load raw ProgramType {}", raw.GetProgramType()); compilation_failed = true; - return; + break; } } else { // Since precompiled didn't have the dump, we'll load them in the next phase @@ -218,8 +218,6 @@ void PipelineCache::LoadDiskCache(const std::atomic_bool& stop_loading, const Di bool load_all_raws = false; if (compilation_failed) { disk_cache.InvalidatePrecompiled(); - dumps.clear(); - precompiled_cache_altered = true; load_all_raws = true; } @@ -292,8 +290,6 @@ void PipelineCache::LoadDiskCache(const std::atomic_bool& stop_loading, const Di // If this is a new separable shader, add it the precompiled cache if (result) { disk_cache.SaveDecompiled(unique_identifier, *result, sanitize_mul); - disk_cache.SaveDump(unique_identifier, shader); - precompiled_cache_altered = true; } if (callback) { @@ -324,10 +320,6 @@ void PipelineCache::LoadDiskCache(const std::atomic_bool& stop_loading, const Di if (compilation_failed) { disk_cache.InvalidateAll(); } - - if (precompiled_cache_altered) { - disk_cache.SaveVirtualPrecompiledFile(); - } } -} // namespace OpenGL +} // namespace VideoCore diff --git a/src/video_core/common/rasterizer.h b/src/video_core/common/rasterizer.h index e9aaa4294..38c327edd 100644 --- a/src/video_core/common/rasterizer.h +++ b/src/video_core/common/rasterizer.h @@ -16,8 +16,8 @@ class EmuWindow; namespace VideoCore { class PipelineCache; -class Callback; -using DiskLoadCallback = Callback; +enum class LoadCallbackStage : u8; +using DiskLoadCallback = std::function; // Structure that the hardware rendered vertices are composed of struct HardwareVertex { diff --git a/src/video_core/common/shader_disk_cache.cpp b/src/video_core/common/shader_disk_cache.cpp index 13a93723e..08b28aaef 100644 --- a/src/video_core/common/shader_disk_cache.cpp +++ b/src/video_core/common/shader_disk_cache.cpp @@ -4,15 +4,14 @@ #include #include -#include "common/assert.h" #include "common/common_paths.h" #include "common/file_util.h" #include "common/logging/log.h" #include "common/scm_rev.h" #include "common/zstd_compression.h" #include "core/core.h" -#include "core/hle/kernel/process.h" #include "core/settings.h" +#include "video_core/common/backend.h" #include "video_core/common/shader_disk_cache.h" namespace VideoCore { @@ -165,10 +164,10 @@ std::optional> ShaderDiskCache::LoadTransferable return {std::move(raws)}; } -std::pair, ShaderDumpsMap> -ShaderDiskCache::LoadPrecompiled(bool compressed) { - if (!IsUsable()) - return {}; +std::optional ShaderDiskCache::LoadPrecompiled() { + if (!IsUsable()) { + return std::nullopt; + } FileUtil::IOFile file(GetPrecompiledPath(), "rb"); if (!file.IsOpen()) { @@ -177,99 +176,62 @@ ShaderDiskCache::LoadPrecompiled(bool compressed) { return {}; } - const std::optional result = LoadPrecompiledFile(file, compressed); - if (!result.has_value()) { - LOG_INFO(Render_OpenGL, - "Failed to load precompiled cache for game with title id={} - removing", + auto Invalidate = [&file, this]() -> std::nullopt_t { + LOG_INFO(Render_OpenGL, "Failed to load precompiled cache for game with title id={} - removing", GetTitleID()); file.Close(); InvalidatePrecompiled(); - return {}; - } - return result.value(); -} + return std::nullopt; + }; -std::optional, ShaderDumpsMap>> -ShaderDiskCache::LoadPrecompiledFile(FileUtil::IOFile& file, bool compressed) { // Read compressed file from disk and decompress to virtual precompiled cache file std::vector precompiled_file(file.GetSize()); file.ReadBytes(precompiled_file.data(), precompiled_file.size()); - if (compressed) { - const auto decompressed = Common::Compression::DecompressDataZSTD(precompiled_file); - SaveArrayToPrecompiled(decompressed.data(), decompressed.size()); - } else { - SaveArrayToPrecompiled(precompiled_file.data(), precompiled_file.size()); - } - + const auto decompressed = Common::Compression::DecompressDataZSTD(precompiled_file); + SaveArrayToPrecompiled(decompressed.data(), decompressed.size()); decompressed_precompiled_cache_offset = 0; ShaderCacheVersionHash file_hash{}; if (!LoadArrayFromPrecompiled(file_hash.data(), file_hash.size())) { return std::nullopt; } + if (GetShaderCacheVersionHash() != file_hash) { LOG_INFO(Render_OpenGL, "Precompiled cache is from another version of the emulator"); return std::nullopt; } - std::unordered_map decompiled; - ShaderDumpsMap dumps; + ShaderDecompiledMap decompiled{}; while (decompressed_precompiled_cache_offset < decompressed_precompiled_cache.size()) { PrecompiledEntryKind kind{}; if (!LoadObjectFromPrecompiled(kind)) { - return std::nullopt; + return Invalidate(); } switch (kind) { case PrecompiledEntryKind::Decompiled: { u64 unique_identifier{}; if (!LoadObjectFromPrecompiled(unique_identifier)) { - return std::nullopt; + return Invalidate(); } std::optional entry = LoadDecompiledEntry(); if (!entry) { - return std::nullopt; + return Invalidate(); } decompiled.insert({unique_identifier, std::move(*entry)}); break; } - case PrecompiledEntryKind::Dump: { - u64 unique_identifier; - if (!LoadObjectFromPrecompiled(unique_identifier)) { - return std::nullopt; - } - - ShaderDiskCacheDump dump; - if (!LoadObjectFromPrecompiled(dump.binary_format)) { - return std::nullopt; - } - - u32 binary_length{}; - if (!LoadObjectFromPrecompiled(binary_length)) { - return std::nullopt; - } - - dump.binary.resize(binary_length); - if (!LoadArrayFromPrecompiled(dump.binary.data(), dump.binary.size())) { - return std::nullopt; - } - - dumps.insert({unique_identifier, dump}); - break; - } default: - return std::nullopt; + return Invalidate(); } } - LOG_INFO(Render_OpenGL, - "Found a precompiled disk cache with {} decompiled entries and {} binary entries", - decompiled.size(), dumps.size()); - return {{decompiled, dumps}}; + LOG_INFO(Render_OpenGL, "Found a precompiled disk cache with {} decompiled entries", decompiled.size()); + return decompiled; } std::optional ShaderDiskCache::LoadDecompiledEntry() { @@ -354,8 +316,10 @@ void ShaderDiskCache::SaveRaw(const ShaderDiskCacheRaw& entry) { } FileUtil::IOFile file = AppendTransferableFile(); - if (!file.IsOpen()) + if (!file.IsOpen()) { return; + } + if (file.WriteObject(TransferableEntryKind::Raw) != 1 || !entry.Save(file)) { LOG_ERROR(Render_OpenGL, "Failed to save raw transferable cache entry - removing"); file.Close(); @@ -381,60 +345,6 @@ void ShaderDiskCache::SaveDecompiled(u64 unique_identifier, const std::string& c } } -void ShaderDiskCache::SaveDump(u64 unique_identifier, ShaderHandle shader) { - if (!IsUsable()) - return; - - GLint binary_length{}; - glGetProgramiv(program, GL_PROGRAM_BINARY_LENGTH, &binary_length); - - GLenum binary_format{}; - std::vector binary(binary_length); - glGetProgramBinary(program, binary_length, nullptr, &binary_format, binary.data()); - - if (!SaveObjectToPrecompiled(static_cast(PrecompiledEntryKind::Dump)) || - !SaveObjectToPrecompiled(unique_identifier) || - !SaveObjectToPrecompiled(static_cast(binary_format)) || - !SaveObjectToPrecompiled(static_cast(binary_length)) || - !SaveArrayToPrecompiled(binary.data(), binary.size())) { - LOG_ERROR(Render_OpenGL, "Failed to save binary program file in shader={:016x} - removing", - unique_identifier); - InvalidatePrecompiled(); - return; - } -} - -void ShaderDiskCache::SaveDumpToFile(u64 unique_identifier, ShaderHandle shader, bool sanitize_mul) { - if (!IsUsable()) - return; - - FileUtil::IOFile file = AppendPrecompiledFile(); - if (!file.IsOpen()) - return; - - GLint binary_length{}; - glGetProgramiv(program, GL_PROGRAM_BINARY_LENGTH, &binary_length); - - GLenum binary_format{}; - std::vector binary(binary_length); - glGetProgramBinary(program, binary_length, nullptr, &binary_format, binary.data()); - - if (file.WriteObject(static_cast(PrecompiledEntryKind::Dump)) != 1 || - file.WriteObject(unique_identifier) != 1 || - file.WriteObject(static_cast(binary_format)) != 1 || - file.WriteObject(static_cast(binary_length)) != 1 || - file.WriteArray(binary.data(), binary.size()) != binary.size()) { - LOG_ERROR(Render_OpenGL, "Failed to save binary program file in shader={:016x} - removing", - unique_identifier); - InvalidatePrecompiled(); - return; - } - - // SaveDecompiled is used only to store the accurate multiplication setting, a better way is to - // probably change the header in SaveDump - SaveDecompiledToFile(file, unique_identifier, {}, sanitize_mul); -} - bool ShaderDiskCache::IsUsable() const { return tried_to_load && Settings::values.use_disk_shader_cache; } @@ -494,25 +404,6 @@ void ShaderDiskCache::SavePrecompiledHeaderToVirtualPrecompiledCache() { } } -void ShaderDiskCache::SaveVirtualPrecompiledFile() { - decompressed_precompiled_cache_offset = 0; - const std::vector& compressed = Common::Compression::CompressDataZSTDDefault( - decompressed_precompiled_cache.data(), decompressed_precompiled_cache.size()); - - const auto precompiled_path{GetPrecompiledPath()}; - FileUtil::IOFile file(precompiled_path, "wb"); - - if (!file.IsOpen()) { - LOG_ERROR(Render_OpenGL, "Failed to open precompiled cache in path={}", precompiled_path); - return; - } - if (file.WriteBytes(compressed.data(), compressed.size()) != compressed.size()) { - LOG_ERROR(Render_OpenGL, "Failed to write precompiled cache version in path={}", - precompiled_path); - return; - } -} - bool ShaderDiskCache::EnsureDirectories() const { const auto CreateDir = [](const std::string& dir) { if (!FileUtil::CreateDir(dir)) { diff --git a/src/video_core/common/shader_disk_cache.h b/src/video_core/common/shader_disk_cache.h index b9cecf9e2..44b4fd96d 100644 --- a/src/video_core/common/shader_disk_cache.h +++ b/src/video_core/common/shader_disk_cache.h @@ -5,11 +5,8 @@ #pragma once #include -#include #include -#include #include -#include #include "video_core/regs.h" #include "video_core/common/shader.h" @@ -74,14 +71,7 @@ struct ShaderDiskCacheDecompiled { bool sanitize_mul; }; -// Contains an OpenGL dumped binary program -struct ShaderDiskCacheDump { - //GLenum binary_format; - std::vector binary; -}; - using ShaderDecompiledMap = std::unordered_map; -using ShaderDumpsMap = std::unordered_map; class BackendBase; @@ -93,9 +83,6 @@ public: /// Loads transferable cache. If file has a old version or on failure, it deletes the file. std::optional> LoadTransferable(); - /// Loads current game's precompiled cache. Invalidates on failure. - std::pair LoadPrecompiled(bool compressed); - /// Removes the transferable (and precompiled) cache file. void InvalidateAll(); @@ -108,20 +95,10 @@ public: /// Saves a decompiled entry to the precompiled file. Does not check for collisions. void SaveDecompiled(u64 unique_identifier, const std::string& code, bool sanitize_mul); - /// Saves a dump entry to the precompiled file. Does not check for collisions. - void SaveDump(u64 unique_identifier, ShaderHandle shader); - - /// Saves a dump entry to the precompiled file. Does not check for collisions. - void SaveDumpToFile(u64 unique_identifier, ShaderHandle shader, bool sanitize_mul); - - /// Serializes virtual precompiled shader cache file to real file - void SaveVirtualPrecompiledFile(); + /// Loads the transferable cache. Returns empty on failure. + std::optional LoadPrecompiled(); private: - /// Loads the transferable cache. Returns empty on failure. - std::optional> LoadPrecompiledFile( - FileUtil::IOFile& file, bool compressed); - /// Loads a decompiled cache entry from m_precompiled_cache_virtual_file. /// Returns empty on failure. std::optional LoadDecompiledEntry(); diff --git a/src/video_core/regs.h b/src/video_core/regs.h index 53dbfcd6e..307665b75 100644 --- a/src/video_core/regs.h +++ b/src/video_core/regs.h @@ -1,4 +1,4 @@ -// Copyright 2017 Citra Emulator Project +// Copyright 2022 Citra Emulator Project // Licensed under GPLv2 or any later version // Refer to the license.txt file included. @@ -6,8 +6,6 @@ #include #include -#include - #include "common/common_funcs.h" #include "common/common_types.h" #include "video_core/regs_framebuffer.h"