shader_disk_cache: Remove precompiled cache

* Serves little purpose at this point and complicates things. Maybe readd later?

src/video_core/common/backend.h

@@ -4,7 +4,6 @@
 
 #pragma once
 
-#include "common/object_pool.h"
 #include "common/vector_math.h"
 #include "video_core/common/pipeline.h"
 #include "video_core/common/framebuffer.h"
@@ -15,7 +14,9 @@ class EmuWindow;
 
 namespace VideoCore {
 
-/// Common interface of a video backend
+class ShaderDiskCache;
+
+// Common interface of a video backend
 class BackendBase {
 public:
     BackendBase(Frontend::EmuWindow& window) : window(window) {}
@@ -59,7 +60,6 @@ public:
     // Executes a compute shader
     virtual void DispatchCompute(PipelineHandle pipeline, Common::Vec3<u32> groupsize,
                                  Common::Vec3<u32> groups) = 0;
-
 private:
     Frontend::EmuWindow& window;
 };

src/video_core/common/pipeline_cache.cpp

@@ -126,18 +126,11 @@ void PipelineCache::LoadDiskCache(const std::atomic_bool& stop_loading, const Di
 
     // Load uncompressed precompiled file for non-separable shaders.
     // Precompiled file for separable shaders is compressed.
-    auto [decompiled, dumps] = disk_cache.LoadPrecompiled(true);
+    std::optional decompiled = disk_cache.LoadPrecompiled();
-
     if (stop_loading) {
         return;
     }
 
-    std::set<GLenum> supported_formats = GetSupportedFormats();
-
-    // Track if precompiled cache was altered during loading to know if we have to serialize the
-    // virtual precompiled cache file back to the hard drive
-    bool precompiled_cache_altered = false;
-
     std::mutex mutex;
     std::atomic_bool compilation_failed = false;
     if (callback) {
@@ -147,7 +140,7 @@ void PipelineCache::LoadDiskCache(const std::atomic_bool& stop_loading, const Di
     std::vector<std::size_t> load_raws_index;
     for (u64 i = 0; i < raws.size(); i++) {
         if (stop_loading || compilation_failed) {
-            return;
+            break;
         }
 
         const ShaderDiskCacheRaw& raw = raws[i];
@@ -161,35 +154,42 @@ void PipelineCache::LoadDiskCache(const std::atomic_bool& stop_loading, const Di
                                      raw.GetUniqueIdentifier(), calculated_hash);
 
             disk_cache.InvalidateAll();
-            return;
+            break;
         }
 
-        const auto dump = dumps.find(unique_identifier);
+        const auto iter = decompiled->find(unique_identifier);
-        const auto decomp = decompiled.find(unique_identifier);
 
         ShaderHandle shader{};
-        if (dump != dumps.end() && decomp != decompiled.end()) {
+        if (iter != decompiled->end()) {
             // Only load the vertex shader if its sanitize_mul setting matches
+            ShaderDiskCacheDecompiled& decomp = iter->second;
             if (raw.GetProgramType() == ProgramType::VertexShader &&
-                decomp->second.sanitize_mul != VideoCore::g_hw_shader_accurate_mul) {
+                decomp.sanitize_mul != VideoCore::g_hw_shader_accurate_mul) {
-                continue;
+                break;
             }
 
-            // If the shader is dumped, attempt to load it
+            ShaderStage stage;
-            shader = GeneratePrecompiledProgram(dump->second, supported_formats);
+            switch (raw.GetProgramType()) {
-            if (!shader.IsValid()) {
+            case ProgramType::VertexShader:
-                // If any shader failed, stop trying to compile, delete the cache, and start
+                stage = ShaderStage::Vertex;
-                // loading from raws
+                break;
-                compilation_failed = true;
+            case ProgramType::GeometryShader:
-                return;
+                stage = ShaderStage::Geometry;
+                break;
+            case ProgramType::FragmentShader:
+                stage = ShaderStage::Fragment;
+                break;
             }
 
+            // Create shader from GLSL source
+            shader = backend->CreateShader(stage, "Precompiled shader", decomp.result);
+
             // We have both the binary shader and the decompiled, so inject it into the
             // cache
             if (raw.GetProgramType() == ProgramType::VertexShader) {
                 auto [conf, setup] = BuildVSConfigFromRaw(raw);
                 std::scoped_lock lock(mutex);
-                pica_vertex_shaders.Inject(conf, decomp->second.result, std::move(shader));
+                pica_vertex_shaders.Inject(conf, decomp.result, std::move(shader));
 
             } else if (raw.GetProgramType() == ProgramType::FragmentShader) {
                 const PicaFSConfig conf{raw.GetRawShaderConfig()};
@@ -200,7 +200,7 @@ void PipelineCache::LoadDiskCache(const std::atomic_bool& stop_loading, const Di
                 // Unsupported shader type got stored somehow so nuke the cache
                 LOG_CRITICAL(Frontend, "failed to load raw ProgramType {}", raw.GetProgramType());
                 compilation_failed = true;
-                return;
+                break;
             }
         } else {
             // Since precompiled didn't have the dump, we'll load them in the next phase
@@ -218,8 +218,6 @@ void PipelineCache::LoadDiskCache(const std::atomic_bool& stop_loading, const Di
     bool load_all_raws = false;
     if (compilation_failed) {
         disk_cache.InvalidatePrecompiled();
-        dumps.clear();
-        precompiled_cache_altered = true;
         load_all_raws = true;
     }
 
@@ -292,8 +290,6 @@ void PipelineCache::LoadDiskCache(const std::atomic_bool& stop_loading, const Di
             // If this is a new separable shader, add it the precompiled cache
             if (result) {
                 disk_cache.SaveDecompiled(unique_identifier, *result, sanitize_mul);
-                disk_cache.SaveDump(unique_identifier, shader);
-                precompiled_cache_altered = true;
             }
 
             if (callback) {
@@ -324,10 +320,6 @@ void PipelineCache::LoadDiskCache(const std::atomic_bool& stop_loading, const Di
     if (compilation_failed) {
         disk_cache.InvalidateAll();
     }
-
-    if (precompiled_cache_altered) {
-        disk_cache.SaveVirtualPrecompiledFile();
-    }
 }
 
-} // namespace OpenGL
+} // namespace VideoCore

src/video_core/common/rasterizer.h

@@ -16,8 +16,8 @@ class EmuWindow;
 namespace VideoCore {
 class PipelineCache;
 
-class Callback;
+enum class LoadCallbackStage : u8;
-using DiskLoadCallback = Callback;
+using DiskLoadCallback = std::function<void(LoadCallbackStage, std::size_t, std::size_t)>;
 
 // Structure that the hardware rendered vertices are composed of
 struct HardwareVertex {

src/video_core/common/shader_disk_cache.cpp

@@ -4,15 +4,14 @@
 
 #include <cstring>
 #include <fmt/format.h>
-#include "common/assert.h"
 #include "common/common_paths.h"
 #include "common/file_util.h"
 #include "common/logging/log.h"
 #include "common/scm_rev.h"
 #include "common/zstd_compression.h"
 #include "core/core.h"
-#include "core/hle/kernel/process.h"
 #include "core/settings.h"
+#include "video_core/common/backend.h"
 #include "video_core/common/shader_disk_cache.h"
 
 namespace VideoCore {
@@ -165,10 +164,10 @@ std::optional<std::vector<ShaderDiskCacheRaw>> ShaderDiskCache::LoadTransferable
     return {std::move(raws)};
 }
 
-std::pair<std::unordered_map<u64, ShaderDiskCacheDecompiled>, ShaderDumpsMap>
+std::optional<ShaderDecompiledMap> ShaderDiskCache::LoadPrecompiled() {
-ShaderDiskCache::LoadPrecompiled(bool compressed) {
+    if (!IsUsable()) {
-    if (!IsUsable())
+        return std::nullopt;
-        return {};
+    }
 
     FileUtil::IOFile file(GetPrecompiledPath(), "rb");
     if (!file.IsOpen()) {
@@ -177,99 +176,62 @@ ShaderDiskCache::LoadPrecompiled(bool compressed) {
         return {};
     }
 
-    const std::optional result = LoadPrecompiledFile(file, compressed);
+    auto Invalidate = [&file, this]() -> std::nullopt_t {
-    if (!result.has_value()) {
+        LOG_INFO(Render_OpenGL, "Failed to load precompiled cache for game with title id={} - removing",
-        LOG_INFO(Render_OpenGL,
-                 "Failed to load precompiled cache for game with title id={} - removing",
                  GetTitleID());
         file.Close();
         InvalidatePrecompiled();
-        return {};
-    }
 
-    return result.value();
+        return std::nullopt;
-}
+    };
 
-std::optional<std::pair<std::unordered_map<u64, ShaderDiskCacheDecompiled>, ShaderDumpsMap>>
-ShaderDiskCache::LoadPrecompiledFile(FileUtil::IOFile& file, bool compressed) {
     // Read compressed file from disk and decompress to virtual precompiled cache file
     std::vector<u8> precompiled_file(file.GetSize());
     file.ReadBytes(precompiled_file.data(), precompiled_file.size());
 
-    if (compressed) {
+    const auto decompressed = Common::Compression::DecompressDataZSTD(precompiled_file);
-        const auto decompressed = Common::Compression::DecompressDataZSTD(precompiled_file);
+    SaveArrayToPrecompiled(decompressed.data(), decompressed.size());
-        SaveArrayToPrecompiled(decompressed.data(), decompressed.size());
-    } else {
-        SaveArrayToPrecompiled(precompiled_file.data(), precompiled_file.size());
-    }
-
     decompressed_precompiled_cache_offset = 0;
 
     ShaderCacheVersionHash file_hash{};
     if (!LoadArrayFromPrecompiled(file_hash.data(), file_hash.size())) {
         return std::nullopt;
     }
+
     if (GetShaderCacheVersionHash() != file_hash) {
         LOG_INFO(Render_OpenGL, "Precompiled cache is from another version of the emulator");
         return std::nullopt;
     }
 
-    std::unordered_map<u64, ShaderDiskCacheDecompiled> decompiled;
+    ShaderDecompiledMap decompiled{};
-    ShaderDumpsMap dumps;
     while (decompressed_precompiled_cache_offset < decompressed_precompiled_cache.size()) {
         PrecompiledEntryKind kind{};
         if (!LoadObjectFromPrecompiled(kind)) {
-            return std::nullopt;
+            return Invalidate();
         }
 
         switch (kind) {
         case PrecompiledEntryKind::Decompiled: {
             u64 unique_identifier{};
             if (!LoadObjectFromPrecompiled(unique_identifier)) {
-                return std::nullopt;
+                return Invalidate();
             }
 
             std::optional entry = LoadDecompiledEntry();
             if (!entry) {
-                return std::nullopt;
+                return Invalidate();
             }
 
             decompiled.insert({unique_identifier, std::move(*entry)});
             break;
         }
-        case PrecompiledEntryKind::Dump: {
-            u64 unique_identifier;
-            if (!LoadObjectFromPrecompiled(unique_identifier)) {
-                return std::nullopt;
-            }
-
-            ShaderDiskCacheDump dump;
-            if (!LoadObjectFromPrecompiled(dump.binary_format)) {
-                return std::nullopt;
-            }
-
-            u32 binary_length{};
-            if (!LoadObjectFromPrecompiled(binary_length)) {
-                return std::nullopt;
-            }
-
-            dump.binary.resize(binary_length);
-            if (!LoadArrayFromPrecompiled(dump.binary.data(), dump.binary.size())) {
-                return std::nullopt;
-            }
-
-            dumps.insert({unique_identifier, dump});
-            break;
-        }
         default:
-            return std::nullopt;
+            return Invalidate();
         }
     }
 
-    LOG_INFO(Render_OpenGL,
+    LOG_INFO(Render_OpenGL, "Found a precompiled disk cache with {} decompiled entries", decompiled.size());
-             "Found a precompiled disk cache with {} decompiled entries and {} binary entries",
+    return decompiled;
-             decompiled.size(), dumps.size());
-    return {{decompiled, dumps}};
 }
 
 std::optional<ShaderDiskCacheDecompiled> ShaderDiskCache::LoadDecompiledEntry() {
@@ -354,8 +316,10 @@ void ShaderDiskCache::SaveRaw(const ShaderDiskCacheRaw& entry) {
     }
 
     FileUtil::IOFile file = AppendTransferableFile();
-    if (!file.IsOpen())
+    if (!file.IsOpen()) {
         return;
+    }
+
     if (file.WriteObject(TransferableEntryKind::Raw) != 1 || !entry.Save(file)) {
         LOG_ERROR(Render_OpenGL, "Failed to save raw transferable cache entry - removing");
         file.Close();
@@ -381,60 +345,6 @@ void ShaderDiskCache::SaveDecompiled(u64 unique_identifier, const std::string& c
     }
 }
 
-void ShaderDiskCache::SaveDump(u64 unique_identifier, ShaderHandle shader) {
-    if (!IsUsable())
-        return;
-
-    GLint binary_length{};
-    glGetProgramiv(program, GL_PROGRAM_BINARY_LENGTH, &binary_length);
-
-    GLenum binary_format{};
-    std::vector<u8> binary(binary_length);
-    glGetProgramBinary(program, binary_length, nullptr, &binary_format, binary.data());
-
-    if (!SaveObjectToPrecompiled(static_cast<u32>(PrecompiledEntryKind::Dump)) ||
-        !SaveObjectToPrecompiled(unique_identifier) ||
-        !SaveObjectToPrecompiled(static_cast<u32>(binary_format)) ||
-        !SaveObjectToPrecompiled(static_cast<u32>(binary_length)) ||
-        !SaveArrayToPrecompiled(binary.data(), binary.size())) {
-        LOG_ERROR(Render_OpenGL, "Failed to save binary program file in shader={:016x} - removing",
-                  unique_identifier);
-        InvalidatePrecompiled();
-        return;
-    }
-}
-
-void ShaderDiskCache::SaveDumpToFile(u64 unique_identifier, ShaderHandle shader, bool sanitize_mul) {
-    if (!IsUsable())
-        return;
-
-    FileUtil::IOFile file = AppendPrecompiledFile();
-    if (!file.IsOpen())
-        return;
-
-    GLint binary_length{};
-    glGetProgramiv(program, GL_PROGRAM_BINARY_LENGTH, &binary_length);
-
-    GLenum binary_format{};
-    std::vector<u8> binary(binary_length);
-    glGetProgramBinary(program, binary_length, nullptr, &binary_format, binary.data());
-
-    if (file.WriteObject(static_cast<u32>(PrecompiledEntryKind::Dump)) != 1 ||
-        file.WriteObject(unique_identifier) != 1 ||
-        file.WriteObject(static_cast<u32>(binary_format)) != 1 ||
-        file.WriteObject(static_cast<u32>(binary_length)) != 1 ||
-        file.WriteArray(binary.data(), binary.size()) != binary.size()) {
-        LOG_ERROR(Render_OpenGL, "Failed to save binary program file in shader={:016x} - removing",
-                  unique_identifier);
-        InvalidatePrecompiled();
-        return;
-    }
-
-    // SaveDecompiled is used only to store the accurate multiplication setting, a better way is to
-    // probably change the header in SaveDump
-    SaveDecompiledToFile(file, unique_identifier, {}, sanitize_mul);
-}
-
 bool ShaderDiskCache::IsUsable() const {
     return tried_to_load && Settings::values.use_disk_shader_cache;
 }
@@ -494,25 +404,6 @@ void ShaderDiskCache::SavePrecompiledHeaderToVirtualPrecompiledCache() {
     }
 }
 
-void ShaderDiskCache::SaveVirtualPrecompiledFile() {
-    decompressed_precompiled_cache_offset = 0;
-    const std::vector<u8>& compressed = Common::Compression::CompressDataZSTDDefault(
-        decompressed_precompiled_cache.data(), decompressed_precompiled_cache.size());
-
-    const auto precompiled_path{GetPrecompiledPath()};
-    FileUtil::IOFile file(precompiled_path, "wb");
-
-    if (!file.IsOpen()) {
-        LOG_ERROR(Render_OpenGL, "Failed to open precompiled cache in path={}", precompiled_path);
-        return;
-    }
-    if (file.WriteBytes(compressed.data(), compressed.size()) != compressed.size()) {
-        LOG_ERROR(Render_OpenGL, "Failed to write precompiled cache version in path={}",
-                  precompiled_path);
-        return;
-    }
-}
-
 bool ShaderDiskCache::EnsureDirectories() const {
     const auto CreateDir = [](const std::string& dir) {
         if (!FileUtil::CreateDir(dir)) {

src/video_core/common/shader_disk_cache.h

@@ -5,11 +5,8 @@
 #pragma once
 
 #include <optional>
-#include <span>
 #include <memory>
-#include <string_view>
 #include <unordered_map>
-#include <vector>
 #include "video_core/regs.h"
 #include "video_core/common/shader.h"
 
@@ -74,14 +71,7 @@ struct ShaderDiskCacheDecompiled {
     bool sanitize_mul;
 };
 
-// Contains an OpenGL dumped binary program
-struct ShaderDiskCacheDump {
-    //GLenum binary_format;
-    std::vector<u8> binary;
-};
-
 using ShaderDecompiledMap = std::unordered_map<u64, ShaderDiskCacheDecompiled>;
-using ShaderDumpsMap = std::unordered_map<u64, ShaderDiskCacheDump>;
 
 class BackendBase;
 
@@ -93,9 +83,6 @@ public:
     /// Loads transferable cache. If file has a old version or on failure, it deletes the file.
     std::optional<std::vector<ShaderDiskCacheRaw>> LoadTransferable();
 
-    /// Loads current game's precompiled cache. Invalidates on failure.
-    std::pair<ShaderDecompiledMap, ShaderDumpsMap> LoadPrecompiled(bool compressed);
-
     /// Removes the transferable (and precompiled) cache file.
     void InvalidateAll();
 
@@ -108,20 +95,10 @@ public:
     /// Saves a decompiled entry to the precompiled file. Does not check for collisions.
     void SaveDecompiled(u64 unique_identifier, const std::string& code, bool sanitize_mul);
 
-    /// Saves a dump entry to the precompiled file. Does not check for collisions.
+    /// Loads the transferable cache. Returns empty on failure.
-    void SaveDump(u64 unique_identifier, ShaderHandle shader);
+    std::optional<ShaderDecompiledMap> LoadPrecompiled();
-
-    /// Saves a dump entry to the precompiled file. Does not check for collisions.
-    void SaveDumpToFile(u64 unique_identifier, ShaderHandle shader, bool sanitize_mul);
-
-    /// Serializes virtual precompiled shader cache file to real file
-    void SaveVirtualPrecompiledFile();
 
 private:
-    /// Loads the transferable cache. Returns empty on failure.
-    std::optional<std::pair<ShaderDecompiledMap, ShaderDumpsMap>> LoadPrecompiledFile(
-        FileUtil::IOFile& file, bool compressed);
-
     /// Loads a decompiled cache entry from m_precompiled_cache_virtual_file.
     /// Returns empty on failure.
     std::optional<ShaderDiskCacheDecompiled> LoadDecompiledEntry();

src/video_core/regs.h

@@ -1,4 +1,4 @@
-// Copyright 2017 Citra Emulator Project
+// Copyright 2022 Citra Emulator Project
 // Licensed under GPLv2 or any later version
 // Refer to the license.txt file included.
 
@@ -6,8 +6,6 @@
 
 #include <array>
 #include <cstddef>
-#include <string>
-
 #include "common/common_funcs.h"
 #include "common/common_types.h"
 #include "video_core/regs_framebuffer.h"