octo/citra
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

shader_disk_cache: Remove precompiled cache

Browse Source 
* Serves little purpose at this point and complicates things. Maybe readd later?
This commit is contained in:
emufan4568
2022-08-10 00:03:26 +03:00
parent 609e32e317
commit 67c20e04b4
6 changed files with 55 additions and 197 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
src/video_core/common/backend.h
View File

@ -4,7 +4,6 @@
#pragma once
#include "common/object_pool.h"
#include "common/vector_math.h"
#include "video_core/common/pipeline.h"
#include "video_core/common/framebuffer.h"
@ -15,7 +14,9 @@ class EmuWindow;
namespace VideoCore {
/// Common interface of a video backend
class ShaderDiskCache;
// Common interface of a video backend
class BackendBase {
public:
BackendBase(Frontend::EmuWindow& window) : window(window) {}
@ -59,7 +60,6 @@ public:
// Executes a compute shader
virtual void DispatchCompute(PipelineHandle pipeline, Common::Vec3<u32> groupsize,
Common::Vec3<u32> groups) = 0;
private:
Frontend::EmuWindow& window;
};

58
src/video_core/common/pipeline_cache.cpp
View File

@ -126,18 +126,11 @@ void PipelineCache::LoadDiskCache(const std::atomic_bool& stop_loading, const Di
// Load uncompressed precompiled file for non-separable shaders.
// Precompiled file for separable shaders is compressed.
auto [decompiled, dumps] = disk_cache.LoadPrecompiled(true);
std::optional decompiled = disk_cache.LoadPrecompiled();
if (stop_loading) {
return;
}
std::set<GLenum> supported_formats = GetSupportedFormats();
// Track if precompiled cache was altered during loading to know if we have to serialize the
// virtual precompiled cache file back to the hard drive
bool precompiled_cache_altered = false;
std::mutex mutex;
std::atomic_bool compilation_failed = false;
if (callback) {
@ -147,7 +140,7 @@ void PipelineCache::LoadDiskCache(const std::atomic_bool& stop_loading, const Di
std::vector<std::size_t> load_raws_index;
for (u64 i = 0; i < raws.size(); i++) {
if (stop_loading || compilation_failed) {
return;
break;
}
const ShaderDiskCacheRaw& raw = raws[i];
@ -161,35 +154,42 @@ void PipelineCache::LoadDiskCache(const std::atomic_bool& stop_loading, const Di
raw.GetUniqueIdentifier(), calculated_hash);
disk_cache.InvalidateAll();
return;
break;
}
const auto dump = dumps.find(unique_identifier);
const auto decomp = decompiled.find(unique_identifier);
const auto iter = decompiled->find(unique_identifier);
ShaderHandle shader{};
if (dump != dumps.end() && decomp != decompiled.end()) {
if (iter != decompiled->end()) {
// Only load the vertex shader if its sanitize_mul setting matches
ShaderDiskCacheDecompiled& decomp = iter->second;
if (raw.GetProgramType() == ProgramType::VertexShader &&
decomp->second.sanitize_mul != VideoCore::g_hw_shader_accurate_mul) {
continue;
decomp.sanitize_mul != VideoCore::g_hw_shader_accurate_mul) {
break;
}
// If the shader is dumped, attempt to load it
shader = GeneratePrecompiledProgram(dump->second, supported_formats);
if (!shader.IsValid()) {
// If any shader failed, stop trying to compile, delete the cache, and start
// loading from raws
compilation_failed = true;
return;
ShaderStage stage;
switch (raw.GetProgramType()) {
case ProgramType::VertexShader:
stage = ShaderStage::Vertex;
break;
case ProgramType::GeometryShader:
stage = ShaderStage::Geometry;
break;
case ProgramType::FragmentShader:
stage = ShaderStage::Fragment;
break;
}
// Create shader from GLSL source
shader = backend->CreateShader(stage, "Precompiled shader", decomp.result);
// We have both the binary shader and the decompiled, so inject it into the
// cache
if (raw.GetProgramType() == ProgramType::VertexShader) {
auto [conf, setup] = BuildVSConfigFromRaw(raw);
std::scoped_lock lock(mutex);
pica_vertex_shaders.Inject(conf, decomp->second.result, std::move(shader));
pica_vertex_shaders.Inject(conf, decomp.result, std::move(shader));
} else if (raw.GetProgramType() == ProgramType::FragmentShader) {
const PicaFSConfig conf{raw.GetRawShaderConfig()};
@ -200,7 +200,7 @@ void PipelineCache::LoadDiskCache(const std::atomic_bool& stop_loading, const Di
// Unsupported shader type got stored somehow so nuke the cache
LOG_CRITICAL(Frontend, "failed to load raw ProgramType {}", raw.GetProgramType());
compilation_failed = true;
return;
break;
}
} else {
// Since precompiled didn't have the dump, we'll load them in the next phase
@ -218,8 +218,6 @@ void PipelineCache::LoadDiskCache(const std::atomic_bool& stop_loading, const Di
bool load_all_raws = false;
if (compilation_failed) {
disk_cache.InvalidatePrecompiled();
dumps.clear();
precompiled_cache_altered = true;
load_all_raws = true;
}
@ -292,8 +290,6 @@ void PipelineCache::LoadDiskCache(const std::atomic_bool& stop_loading, const Di
// If this is a new separable shader, add it the precompiled cache
if (result) {
disk_cache.SaveDecompiled(unique_identifier, *result, sanitize_mul);
disk_cache.SaveDump(unique_identifier, shader);
precompiled_cache_altered = true;
}
if (callback) {
@ -324,10 +320,6 @@ void PipelineCache::LoadDiskCache(const std::atomic_bool& stop_loading, const Di
if (compilation_failed) {
disk_cache.InvalidateAll();
}
if (precompiled_cache_altered) {
disk_cache.SaveVirtualPrecompiledFile();
}
}
} // namespace OpenGL
} // namespace VideoCore

4
src/video_core/common/rasterizer.h
View File

@ -16,8 +16,8 @@ class EmuWindow;
namespace VideoCore {
class PipelineCache;
class Callback;
using DiskLoadCallback = Callback;
enum class LoadCallbackStage : u8;
using DiskLoadCallback = std::function<void(LoadCallbackStage, std::size_t, std::size_t)>;
// Structure that the hardware rendered vertices are composed of
struct HardwareVertex {

153
src/video_core/common/shader_disk_cache.cpp
View File

@ -4,15 +4,14 @@
#include <cstring>
#include <fmt/format.h>
#include "common/assert.h"
#include "common/common_paths.h"
#include "common/file_util.h"
#include "common/logging/log.h"
#include "common/scm_rev.h"
#include "common/zstd_compression.h"
#include "core/core.h"
#include "core/hle/kernel/process.h"
#include "core/settings.h"
#include "video_core/common/backend.h"
#include "video_core/common/shader_disk_cache.h"
namespace VideoCore {
@ -165,10 +164,10 @@ std::optional<std::vector<ShaderDiskCacheRaw>> ShaderDiskCache::LoadTransferable
return {std::move(raws)};
}
std::pair<std::unordered_map<u64, ShaderDiskCacheDecompiled>, ShaderDumpsMap>
ShaderDiskCache::LoadPrecompiled(bool compressed) {
if (!IsUsable())
return {};
std::optional<ShaderDecompiledMap> ShaderDiskCache::LoadPrecompiled() {
if (!IsUsable()) {
return std::nullopt;
}
FileUtil::IOFile file(GetPrecompiledPath(), "rb");
if (!file.IsOpen()) {
@ -177,99 +176,62 @@ ShaderDiskCache::LoadPrecompiled(bool compressed) {
return {};
}
const std::optional result = LoadPrecompiledFile(file, compressed);
if (!result.has_value()) {
LOG_INFO(Render_OpenGL,
"Failed to load precompiled cache for game with title id={} - removing",
auto Invalidate = [&file, this]() -> std::nullopt_t {
LOG_INFO(Render_OpenGL, "Failed to load precompiled cache for game with title id={} - removing",
GetTitleID());
file.Close();
InvalidatePrecompiled();
return {};
}
return result.value();
}
return std::nullopt;
};
std::optional<std::pair<std::unordered_map<u64, ShaderDiskCacheDecompiled>, ShaderDumpsMap>>
ShaderDiskCache::LoadPrecompiledFile(FileUtil::IOFile& file, bool compressed) {
// Read compressed file from disk and decompress to virtual precompiled cache file
std::vector<u8> precompiled_file(file.GetSize());
file.ReadBytes(precompiled_file.data(), precompiled_file.size());
if (compressed) {
const auto decompressed = Common::Compression::DecompressDataZSTD(precompiled_file);
SaveArrayToPrecompiled(decompressed.data(), decompressed.size());
} else {
SaveArrayToPrecompiled(precompiled_file.data(), precompiled_file.size());
}
const auto decompressed = Common::Compression::DecompressDataZSTD(precompiled_file);
SaveArrayToPrecompiled(decompressed.data(), decompressed.size());
decompressed_precompiled_cache_offset = 0;
ShaderCacheVersionHash file_hash{};
if (!LoadArrayFromPrecompiled(file_hash.data(), file_hash.size())) {
return std::nullopt;
}
if (GetShaderCacheVersionHash() != file_hash) {
LOG_INFO(Render_OpenGL, "Precompiled cache is from another version of the emulator");
return std::nullopt;
}
std::unordered_map<u64, ShaderDiskCacheDecompiled> decompiled;
ShaderDumpsMap dumps;
ShaderDecompiledMap decompiled{};
while (decompressed_precompiled_cache_offset < decompressed_precompiled_cache.size()) {
PrecompiledEntryKind kind{};
if (!LoadObjectFromPrecompiled(kind)) {
return std::nullopt;
return Invalidate();
}
switch (kind) {
case PrecompiledEntryKind::Decompiled: {
u64 unique_identifier{};
if (!LoadObjectFromPrecompiled(unique_identifier)) {
return std::nullopt;
return Invalidate();
}
std::optional entry = LoadDecompiledEntry();
if (!entry) {
return std::nullopt;
return Invalidate();
}
decompiled.insert({unique_identifier, std::move(*entry)});
break;
}
case PrecompiledEntryKind::Dump: {
u64 unique_identifier;
if (!LoadObjectFromPrecompiled(unique_identifier)) {
return std::nullopt;
}
ShaderDiskCacheDump dump;
if (!LoadObjectFromPrecompiled(dump.binary_format)) {
return std::nullopt;
}
u32 binary_length{};
if (!LoadObjectFromPrecompiled(binary_length)) {
return std::nullopt;
}
dump.binary.resize(binary_length);
if (!LoadArrayFromPrecompiled(dump.binary.data(), dump.binary.size())) {
return std::nullopt;
}
dumps.insert({unique_identifier, dump});
break;
}
default:
return std::nullopt;
return Invalidate();
}
}
LOG_INFO(Render_OpenGL,
"Found a precompiled disk cache with {} decompiled entries and {} binary entries",
decompiled.size(), dumps.size());
return {{decompiled, dumps}};
LOG_INFO(Render_OpenGL, "Found a precompiled disk cache with {} decompiled entries", decompiled.size());
return decompiled;
}
std::optional<ShaderDiskCacheDecompiled> ShaderDiskCache::LoadDecompiledEntry() {
@ -354,8 +316,10 @@ void ShaderDiskCache::SaveRaw(const ShaderDiskCacheRaw& entry) {
}
FileUtil::IOFile file = AppendTransferableFile();
if (!file.IsOpen())
if (!file.IsOpen()) {
return;
}
if (file.WriteObject(TransferableEntryKind::Raw) != 1 || !entry.Save(file)) {
LOG_ERROR(Render_OpenGL, "Failed to save raw transferable cache entry - removing");
file.Close();
@ -381,60 +345,6 @@ void ShaderDiskCache::SaveDecompiled(u64 unique_identifier, const std::string& c
}
}
void ShaderDiskCache::SaveDump(u64 unique_identifier, ShaderHandle shader) {
if (!IsUsable())
return;
GLint binary_length{};
glGetProgramiv(program, GL_PROGRAM_BINARY_LENGTH, &binary_length);
GLenum binary_format{};
std::vector<u8> binary(binary_length);
glGetProgramBinary(program, binary_length, nullptr, &binary_format, binary.data());
if (!SaveObjectToPrecompiled(static_cast<u32>(PrecompiledEntryKind::Dump)) ||
!SaveObjectToPrecompiled(unique_identifier) ||
!SaveObjectToPrecompiled(static_cast<u32>(binary_format)) ||
!SaveObjectToPrecompiled(static_cast<u32>(binary_length)) ||
!SaveArrayToPrecompiled(binary.data(), binary.size())) {
LOG_ERROR(Render_OpenGL, "Failed to save binary program file in shader={:016x} - removing",
unique_identifier);
InvalidatePrecompiled();
return;
}
}
void ShaderDiskCache::SaveDumpToFile(u64 unique_identifier, ShaderHandle shader, bool sanitize_mul) {
if (!IsUsable())
return;
FileUtil::IOFile file = AppendPrecompiledFile();
if (!file.IsOpen())
return;
GLint binary_length{};
glGetProgramiv(program, GL_PROGRAM_BINARY_LENGTH, &binary_length);
GLenum binary_format{};
std::vector<u8> binary(binary_length);
glGetProgramBinary(program, binary_length, nullptr, &binary_format, binary.data());
if (file.WriteObject(static_cast<u32>(PrecompiledEntryKind::Dump)) != 1 ||
file.WriteObject(unique_identifier) != 1 ||
file.WriteObject(static_cast<u32>(binary_format)) != 1 ||
file.WriteObject(static_cast<u32>(binary_length)) != 1 ||
file.WriteArray(binary.data(), binary.size()) != binary.size()) {
LOG_ERROR(Render_OpenGL, "Failed to save binary program file in shader={:016x} - removing",
unique_identifier);
InvalidatePrecompiled();
return;
}
// SaveDecompiled is used only to store the accurate multiplication setting, a better way is to
// probably change the header in SaveDump
SaveDecompiledToFile(file, unique_identifier, {}, sanitize_mul);
}
bool ShaderDiskCache::IsUsable() const {
return tried_to_load && Settings::values.use_disk_shader_cache;
}
@ -494,25 +404,6 @@ void ShaderDiskCache::SavePrecompiledHeaderToVirtualPrecompiledCache() {
}
}
void ShaderDiskCache::SaveVirtualPrecompiledFile() {
decompressed_precompiled_cache_offset = 0;
const std::vector<u8>& compressed = Common::Compression::CompressDataZSTDDefault(
decompressed_precompiled_cache.data(), decompressed_precompiled_cache.size());
const auto precompiled_path{GetPrecompiledPath()};
FileUtil::IOFile file(precompiled_path, "wb");
if (!file.IsOpen()) {
LOG_ERROR(Render_OpenGL, "Failed to open precompiled cache in path={}", precompiled_path);
return;
}
if (file.WriteBytes(compressed.data(), compressed.size()) != compressed.size()) {
LOG_ERROR(Render_OpenGL, "Failed to write precompiled cache version in path={}",
precompiled_path);
return;
}
}
bool ShaderDiskCache::EnsureDirectories() const {
const auto CreateDir = [](const std::string& dir) {
if (!FileUtil::CreateDir(dir)) {

27
src/video_core/common/shader_disk_cache.h
View File

@ -5,11 +5,8 @@
#pragma once
#include <optional>
#include <span>
#include <memory>
#include <string_view>
#include <unordered_map>
#include <vector>
#include "video_core/regs.h"
#include "video_core/common/shader.h"
@ -74,14 +71,7 @@ struct ShaderDiskCacheDecompiled {
bool sanitize_mul;
};
// Contains an OpenGL dumped binary program
struct ShaderDiskCacheDump {
//GLenum binary_format;
std::vector<u8> binary;
};
using ShaderDecompiledMap = std::unordered_map<u64, ShaderDiskCacheDecompiled>;
using ShaderDumpsMap = std::unordered_map<u64, ShaderDiskCacheDump>;
class BackendBase;
@ -93,9 +83,6 @@ public:
/// Loads transferable cache. If file has a old version or on failure, it deletes the file.
std::optional<std::vector<ShaderDiskCacheRaw>> LoadTransferable();
/// Loads current game's precompiled cache. Invalidates on failure.
std::pair<ShaderDecompiledMap, ShaderDumpsMap> LoadPrecompiled(bool compressed);
/// Removes the transferable (and precompiled) cache file.
void InvalidateAll();
@ -108,20 +95,10 @@ public:
/// Saves a decompiled entry to the precompiled file. Does not check for collisions.
void SaveDecompiled(u64 unique_identifier, const std::string& code, bool sanitize_mul);
/// Saves a dump entry to the precompiled file. Does not check for collisions.
void SaveDump(u64 unique_identifier, ShaderHandle shader);
/// Saves a dump entry to the precompiled file. Does not check for collisions.
void SaveDumpToFile(u64 unique_identifier, ShaderHandle shader, bool sanitize_mul);
/// Serializes virtual precompiled shader cache file to real file
void SaveVirtualPrecompiledFile();
/// Loads the transferable cache. Returns empty on failure.
std::optional<ShaderDecompiledMap> LoadPrecompiled();
private:
/// Loads the transferable cache. Returns empty on failure.
std::optional<std::pair<ShaderDecompiledMap, ShaderDumpsMap>> LoadPrecompiledFile(
FileUtil::IOFile& file, bool compressed);
/// Loads a decompiled cache entry from m_precompiled_cache_virtual_file.
/// Returns empty on failure.
std::optional<ShaderDiskCacheDecompiled> LoadDecompiledEntry();

4
src/video_core/regs.h
View File

@ -1,4 +1,4 @@
// Copyright 2017 Citra Emulator Project
// Copyright 2022 Citra Emulator Project
// Licensed under GPLv2 or any later version
// Refer to the license.txt file included.
@ -6,8 +6,6 @@
#include <array>
#include <cstddef>
#include <string>
#include "common/common_funcs.h"
#include "common/common_types.h"
#include "video_core/regs_framebuffer.h"