Compare commits

...

10 Commits

Author SHA1 Message Date
f3bbcb31c9 renderer_opengl: Remove non-separable path 2023-11-04 12:59:20 +02:00
b10f3d96f5 command_processor: Fix out-of-bounds float-uniform access (#7111)
Addresses:
https://github.com/citra-emu/citra/issues/6696
https://github.com/citra-emu/citra/issues/6871
2023-11-03 03:35:52 -07:00
b5d744bcae ci: Work around macOS GitHub runner pip install failures. (#7110) 2023-11-03 03:35:32 -07:00
89d5d4a2b6 externals: allow user to use system cubeb (#7107) 2023-11-02 17:33:40 -07:00
4284893044 Implement RomFS cache and async reads. (#7089)
* Implement RomFS cache and async reads.

* Suggestions and fix compilation.

* Apply suggestions
2023-11-02 17:19:00 -07:00
79ea06b226 qt: Update to 6.6.0 (#7099) 2023-11-01 17:58:02 -07:00
8d811913a5 externals: allow user to use system cryptopp (#7105) 2023-11-01 17:57:10 -07:00
ac9d72a95c vk_texture_runtime: Fix debug scope label lambda-capture (#7102)
`DebugScope` was capturing a `string_view` in a lambda which is only
valid during the scope of this ctor. When the lambda gets invoked at a
later time, it will read undefined garbage. The lambda needs to make a
deep copy of this `string_view` into a `string` so that it is valid by
the time the scheduler invokes this lambda.
2023-11-01 21:30:54 +01:00
d3ce43782d externals: allow users to use system libenet (#7100) 2023-10-31 14:01:50 -07:00
597a2e8ead Add missing FS:USER functions (#7051) 2023-10-31 14:01:25 -07:00
40 changed files with 1005 additions and 431 deletions

View File

@ -1,5 +1,10 @@
#!/bin/bash -ex
# TODO: Work around pip install issues with Python 3.12 in the GitHub runner image.
# See: https://github.com/actions/runner-images/issues/8709
PYTHON_PATH=$(brew --prefix python@3.11)
export PATH=$PYTHON_PATH/bin:$PYTHON_PATH/libexec/bin:$PATH
mkdir build && cd build
cmake .. -GNinja \
-DCMAKE_BUILD_TYPE=Release \

View File

@ -1,5 +1,10 @@
#!/bin/bash -ex
# TODO: Work around pip install issues with Python 3.12 in the GitHub runner image.
# See: https://github.com/actions/runner-images/issues/8709
PYTHON_PATH=$(brew --prefix python@3.11)
export PATH=$PYTHON_PATH/bin:$PYTHON_PATH/libexec/bin:$PATH
mkdir build && cd build
cmake .. -GNinja \
-DCMAKE_BUILD_TYPE=Release \

View File

@ -59,7 +59,7 @@ jobs:
name: ${{ env.OS }}-${{ env.TARGET }}
path: artifacts/
macos:
runs-on: macos-latest
runs-on: macos-13
strategy:
matrix:
target: ["x86_64", "arm64"]
@ -92,7 +92,7 @@ jobs:
path: ${{ env.OS }}-${{ env.TARGET }}
key: ${{ runner.os }}-${{ matrix.target }}-${{ github.sha }}-${{ github.run_id }}-${{ github.run_attempt }}
macos-universal:
runs-on: macos-latest
runs-on: macos-13
needs: macos
env:
OS: macos
@ -234,7 +234,7 @@ jobs:
name: ${{ env.OS }}-${{ env.TARGET }}
path: src/android/app/artifacts/
ios:
runs-on: macos-latest
runs-on: macos-13
if: ${{ !startsWith(github.ref, 'refs/tags/') }}
env:
CCACHE_DIR: ${{ github.workspace }}/.ccache

View File

@ -236,7 +236,7 @@ find_package(Threads REQUIRED)
if (ENABLE_QT)
if (NOT USE_SYSTEM_QT)
download_qt(6.5.1)
download_qt(6.6.0)
endif()
find_package(Qt6 REQUIRED COMPONENTS Widgets Multimedia Concurrent)

View File

@ -2,7 +2,13 @@
Name=colorful_dark
Comment=Colorful theme (Dark style)
Inherits=default
Directories=16x16
Directories=16x16,48x48,256x256
[16x16]
Size=16
[48x48]
Size=48
[256x256]
Size=256

View File

@ -2,7 +2,13 @@
Name=colorful_midnight_blue
Comment=Colorful theme (Midnight Blue style)
Inherits=default
Directories=16x16
Directories=16x16,48x48,256x256
[16x16]
Size=16
[48x48]
Size=48
[256x256]
Size=256

View File

@ -46,11 +46,17 @@ set(CATCH_INSTALL_EXTRAS OFF CACHE BOOL "")
add_subdirectory(catch2)
# Crypto++
set(CRYPTOPP_BUILD_DOCUMENTATION OFF CACHE BOOL "")
set(CRYPTOPP_BUILD_TESTING OFF CACHE BOOL "")
set(CRYPTOPP_INSTALL OFF CACHE BOOL "")
set(CRYPTOPP_SOURCES "${CMAKE_SOURCE_DIR}/externals/cryptopp" CACHE STRING "")
add_subdirectory(cryptopp-cmake)
if(USE_SYSTEM_CRYPTOPP)
find_package(cryptopp REQUIRED)
add_library(cryptopp INTERFACE)
target_link_libraries(cryptopp INTERFACE cryptopp::cryptopp)
else()
set(CRYPTOPP_BUILD_DOCUMENTATION OFF CACHE BOOL "")
set(CRYPTOPP_BUILD_TESTING OFF CACHE BOOL "")
set(CRYPTOPP_INSTALL OFF CACHE BOOL "")
set(CRYPTOPP_SOURCES "${CMAKE_SOURCE_DIR}/externals/cryptopp" CACHE STRING "")
add_subdirectory(cryptopp-cmake)
endif()
# dds-ktx
add_library(dds-ktx INTERFACE)
@ -228,15 +234,30 @@ else()
endif()
# ENet
add_subdirectory(enet)
target_include_directories(enet INTERFACE ./enet/include)
if(USE_SYSTEM_ENET)
find_package(libenet REQUIRED)
add_library(enet INTERFACE)
target_link_libraries(enet INTERFACE libenet::libenet)
else()
add_subdirectory(enet)
target_include_directories(enet INTERFACE ./enet/include)
endif()
# Cubeb
if (ENABLE_CUBEB)
set(BUILD_TESTS OFF CACHE BOOL "")
set(BUILD_TOOLS OFF CACHE BOOL "")
set(BUNDLE_SPEEX ON CACHE BOOL "")
add_subdirectory(cubeb EXCLUDE_FROM_ALL)
if(USE_SYSTEM_CUBEB)
find_package(cubeb REQUIRED)
add_library(cubeb INTERFACE)
target_link_libraries(cubeb INTERFACE cubeb::cubeb)
if(TARGET cubeb::cubeb)
message(STATUS "Found system cubeb")
endif()
else()
set(BUILD_TESTS OFF CACHE BOOL "")
set(BUILD_TOOLS OFF CACHE BOOL "")
set(BUNDLE_SPEEX ON CACHE BOOL "")
add_subdirectory(cubeb EXCLUDE_FROM_ALL)
endif()
endif()
# DiscordRPC

View File

@ -19,6 +19,9 @@ option(USE_SYSTEM_FDK_AAC_HEADERS "Use the system fdk-aac headers (instead of th
option(USE_SYSTEM_FFMPEG_HEADERS "Use the system FFmpeg headers (instead of the bundled one)" OFF)
option(USE_SYSTEM_GLSLANG "Use the system glslang and SPIR-V libraries (instead of the bundled ones)" OFF)
option(USE_SYSTEM_ZSTD "Use the system Zstandard library (instead of the bundled one)" OFF)
option(USE_SYSTEM_ENET "Use the system libenet (instead of the bundled one)" OFF)
option(USE_SYSTEM_CRYPTOPP "Use the system cryptopp (instead of the bundled one)" OFF)
option(USE_SYSTEM_CUBEB "Use the system cubeb (instead of the bundled one)" OFF)
# Qt and MoltenVK are handled separately
CMAKE_DEPENDENT_OPTION(DISABLE_SYSTEM_SDL2 "Disable system SDL2" OFF "USE_SYSTEM_LIBS" OFF)
@ -37,6 +40,9 @@ CMAKE_DEPENDENT_OPTION(DISABLE_SYSTEM_FDK_AAC_HEADERS "Disable system fdk_aac" O
CMAKE_DEPENDENT_OPTION(DISABLE_SYSTEM_FFMPEG_HEADERS "Disable system ffmpeg" OFF "USE_SYSTEM_LIBS" OFF)
CMAKE_DEPENDENT_OPTION(DISABLE_SYSTEM_GLSLANG "Disable system glslang" OFF "USE_SYSTEM_LIBS" OFF)
CMAKE_DEPENDENT_OPTION(DISABLE_SYSTEM_ZSTD "Disable system Zstandard" OFF "USE_SYSTEM_LIBS" OFF)
CMAKE_DEPENDENT_OPTION(DISABLE_SYSTEM_ENET "Disable system libenet" OFF "USE_SYSTEM_LIBS" OFF)
CMAKE_DEPENDENT_OPTION(DISABLE_SYSTEM_CRYPTOPP "Disable system cryptopp" OFF "USE_SYSTEM_LIBS" OFF)
CMAKE_DEPENDENT_OPTION(DISABLE_SYSTEM_CUBEB "Disable system cubeb" OFF "USE_SYSTEM_LIBS" OFF)
set(LIB_VAR_LIST
SDL2
@ -55,6 +61,9 @@ set(LIB_VAR_LIST
FFMPEG_HEADERS
GLSLANG
ZSTD
ENET
CRYPTOPP
CUBEB
)
# First, check that USE_SYSTEM_XXX is not used with USE_SYSTEM_LIBS

View File

@ -0,0 +1,36 @@
if(NOT CRYPTOPP_FOUND)
pkg_check_modules(CRYPTOPP_TMP libcrypto++)
find_path(CRYPTOPP_INCLUDE_DIRS NAMES cryptlib.h
PATHS
${CRYPTOPP_TMP_INCLUDE_DIRS}
/usr/include
/usr/include/crypto++
/usr/local/include
/usr/local/include/crypto++
)
find_library(CRYPTOPP_LIBRARY_DIRS NAMES crypto++
PATHS
${CRYPTOPP_TMP_LIBRARY_DIRS}
/usr/lib
/usr/locallib
)
if(CRYPTOPP_INCLUDE_DIRS AND CRYPTOPP_LIBRARY_DIRS)
set(CRYPTOPP_FOUND TRUE CACHE INTERNAL "Found cryptopp")
message(STATUS "Found cryptopp: ${CRYPTOPP_LIBRARY_DIRS}, ${CRYPTOPP_INCLUDE_DIRS}")
else()
set(CRYPTOPP_FOUND FALSE CACHE INTERNAL "Found cryptopp")
message(STATUS "Cryptopp not found.")
endif()
endif()
if(CRYPTOPP_FOUND AND NOT TARGET cryptopp::cryptopp)
add_library(cryptopp::cryptopp UNKNOWN IMPORTED)
set_target_properties(cryptopp::cryptopp PROPERTIES
INCLUDE_DIRECTORIES ${CRYPTOPP_INCLUDE_DIRS}
INTERFACE_LINK_LIBRARIES ${CRYPTOPP_LIBRARY_DIRS}
IMPORTED_LOCATION ${CRYPTOPP_LIBRARY_DIRS}
)
endif()

View File

@ -0,0 +1,34 @@
if(NOT libenet_FOUND)
pkg_check_modules(ENET_TMP libenet)
find_path(libenet_INCLUDE_DIRS NAMES enet.h PATH_SUFFIXES enet
PATHS
${ENET_TMP_INCLUDE_DIRS}
/usr/include
/usr/local/include
)
find_library(libenet_LIBRARY_DIRS NAMES enet
PATHS
${ENET_TMP_LIBRARY_DIRS}
/usr/lib
/usr/local/lib
)
if(libenet_INCLUDE_DIRS AND libenet_LIBRARY_DIRS)
set(libenet_FOUND TRUE CACHE INTERNAL "Found libenet")
message(STATUS "Found libenet ${libenet_LIBRARY_DIRS}, ${libenet_INCLUDE_DIRS}")
else()
set(libenet_FOUND FALSE CACHE INTERNAL "Found libenet")
message(STATUS "Libenet not found.")
endif()
endif()
if(libenet_FOUND AND NOT TARGET libenet::libenet)
add_library(libenet::libenet UNKNOWN IMPORTED)
set_target_properties(libenet::libenet PROPERTIES
INCLUDE_DIRECTORIES ${libenet_INCLUDE_DIRS}
INTERFACE_LINK_LIBRARIES ${libenet_LIBRARY_DIRS}
IMPORTED_LOCATION ${libenet_LIBRARY_DIRS}
)
endif()

View File

@ -258,13 +258,15 @@ void GameList::OnUpdateThemedIcons() {
for (int i = 0; i < item_model->invisibleRootItem()->rowCount(); i++) {
QStandardItem* child = item_model->invisibleRootItem()->child(i);
const int icon_size = IconSizes.at(UISettings::values.game_list_icon_size.GetValue());
switch (child->data(GameListItem::TypeRole).value<GameListItemType>()) {
case GameListItemType::InstalledDir:
child->setData(QIcon::fromTheme(QStringLiteral("sd_card")).pixmap(48),
child->setData(QIcon::fromTheme(QStringLiteral("sd_card")).pixmap(icon_size),
Qt::DecorationRole);
break;
case GameListItemType::SystemDir:
child->setData(QIcon::fromTheme(QStringLiteral("chip")).pixmap(48), Qt::DecorationRole);
child->setData(QIcon::fromTheme(QStringLiteral("chip")).pixmap(icon_size),
Qt::DecorationRole);
break;
case GameListItemType::CustomDir: {
const UISettings::GameDir& game_dir =
@ -272,11 +274,12 @@ void GameList::OnUpdateThemedIcons() {
const QString icon_name = QFileInfo::exists(game_dir.path)
? QStringLiteral("folder")
: QStringLiteral("bad_folder");
child->setData(QIcon::fromTheme(icon_name).pixmap(48), Qt::DecorationRole);
child->setData(QIcon::fromTheme(icon_name).pixmap(icon_size), Qt::DecorationRole);
break;
}
case GameListItemType::AddDir:
child->setData(QIcon::fromTheme(QStringLiteral("plus")).pixmap(48), Qt::DecorationRole);
child->setData(QIcon::fromTheme(QStringLiteral("plus")).pixmap(icon_size),
Qt::DecorationRole);
break;
default:
break;

View File

@ -2741,7 +2741,10 @@ void GMainWindow::filterBarSetChecked(bool state) {
}
void GMainWindow::UpdateUITheme() {
const QString default_icons = QStringLiteral(":/icons/default");
const QString icons_base_path = QStringLiteral(":/icons/");
const QString default_theme = QStringLiteral("default");
const QString default_theme_path = icons_base_path + default_theme;
const QString& current_theme = UISettings::values.theme;
const bool is_default_theme = current_theme == QString::fromUtf8(UISettings::themes[0].second);
QStringList theme_paths(default_theme_paths);
@ -2759,8 +2762,8 @@ void GMainWindow::UpdateUITheme() {
qApp->setStyleSheet({});
setStyleSheet({});
}
theme_paths.append(default_icons);
QIcon::setThemeName(default_icons);
theme_paths.append(default_theme_path);
QIcon::setThemeName(default_theme);
} else {
const QString theme_uri(QLatin1Char{':'} + current_theme + QStringLiteral("/style.qss"));
QFile f(theme_uri);
@ -2772,9 +2775,9 @@ void GMainWindow::UpdateUITheme() {
LOG_ERROR(Frontend, "Unable to set style, stylesheet file not found");
}
const QString theme_name = QStringLiteral(":/icons/") + current_theme;
theme_paths.append({default_icons, theme_name});
QIcon::setThemeName(theme_name);
const QString current_theme_path = icons_base_path + current_theme;
theme_paths.append({default_theme_path, current_theme_path});
QIcon::setThemeName(current_theme);
}
QIcon::setThemeSearchPaths(theme_paths);

View File

@ -17,7 +17,7 @@
#include "common/file_util.h"
#include "common/logging/log.h"
#ifdef Q_OS_OSX
#ifdef Q_OS_MACOS
#define DEFAULT_TOOL_PATH QStringLiteral("../../../../maintenancetool")
#else
#define DEFAULT_TOOL_PATH QStringLiteral("../maintenancetool")
@ -102,7 +102,7 @@ QString UpdaterPrivate::ToSystemExe(QString base_path) {
return base_path + QStringLiteral(".exe");
else
return base_path;
#elif defined(Q_OS_OSX)
#elif defined(Q_OS_MACOS)
if (base_path.endsWith(QStringLiteral(".app")))
base_path.truncate(base_path.lastIndexOf(QStringLiteral(".")));
return base_path + QStringLiteral(".app/Contents/MacOS/") + QFileInfo(base_path).fileName();
@ -112,7 +112,7 @@ QString UpdaterPrivate::ToSystemExe(QString base_path) {
}
QFileInfo UpdaterPrivate::GetMaintenanceTool() const {
#if defined(Q_OS_UNIX) && !defined(Q_OS_OSX)
#if defined(Q_OS_UNIX) && !defined(Q_OS_MACOS)
const auto appimage_path = QProcessEnvironment::systemEnvironment()
.value(QStringLiteral("APPIMAGE"), {})
.toStdString();

View File

@ -124,6 +124,7 @@ add_library(citra_common STATIC
serialization/boost_flat_set.h
serialization/boost_small_vector.hpp
serialization/boost_vector.hpp
static_lru_cache.h
string_literal.h
string_util.cpp
string_util.h

View File

@ -1155,6 +1155,43 @@ std::size_t IOFile::ReadImpl(void* data, std::size_t length, std::size_t data_si
return std::fread(data, data_size, length, m_file);
}
#ifdef _WIN32
static std::size_t pread(int fd, void* buf, size_t count, uint64_t offset) {
long unsigned int read_bytes = 0;
OVERLAPPED overlapped = {0};
HANDLE file = reinterpret_cast<HANDLE>(_get_osfhandle(fd));
overlapped.OffsetHigh = static_cast<uint32_t>(offset >> 32);
overlapped.Offset = static_cast<uint32_t>(offset & 0xFFFF'FFFFLL);
SetLastError(0);
bool ret = ReadFile(file, buf, static_cast<uint32_t>(count), &read_bytes, &overlapped);
if (!ret && GetLastError() != ERROR_HANDLE_EOF) {
errno = GetLastError();
return std::numeric_limits<std::size_t>::max();
}
return read_bytes;
}
#else
#define pread ::pread
#endif
std::size_t IOFile::ReadAtImpl(void* data, std::size_t length, std::size_t data_size,
std::size_t offset) {
if (!IsOpen()) {
m_good = false;
return std::numeric_limits<std::size_t>::max();
}
if (length == 0) {
return 0;
}
DEBUG_ASSERT(data != nullptr);
return pread(fileno(m_file), data, data_size * length, offset);
}
std::size_t IOFile::WriteImpl(const void* data, std::size_t length, std::size_t data_size) {
if (!IsOpen()) {
m_good = false;

View File

@ -294,6 +294,18 @@ public:
return items_read;
}
template <typename T>
std::size_t ReadAtArray(T* data, std::size_t length, std::size_t offset) {
static_assert(std::is_trivially_copyable_v<T>,
"Given array does not consist of trivially copyable objects");
std::size_t items_read = ReadAtImpl(data, length, sizeof(T), offset);
if (items_read != length)
m_good = false;
return items_read;
}
template <typename T>
std::size_t WriteArray(const T* data, std::size_t length) {
static_assert(std::is_trivially_copyable_v<T>,
@ -312,6 +324,12 @@ public:
return ReadArray(reinterpret_cast<char*>(data), length);
}
template <typename T>
std::size_t ReadAtBytes(T* data, std::size_t length, std::size_t offset) {
static_assert(std::is_trivially_copyable_v<T>, "T must be trivially copyable");
return ReadAtArray(reinterpret_cast<char*>(data), length, offset);
}
template <typename T>
std::size_t WriteBytes(const T* data, std::size_t length) {
static_assert(std::is_trivially_copyable_v<T>, "T must be trivially copyable");
@ -363,6 +381,8 @@ public:
private:
std::size_t ReadImpl(void* data, std::size_t length, std::size_t data_size);
std::size_t ReadAtImpl(void* data, std::size_t length, std::size_t data_size,
std::size_t offset);
std::size_t WriteImpl(const void* data, std::size_t length, std::size_t data_size);
bool Open();

View File

@ -0,0 +1,113 @@
// Modified version of: https://www.boost.org/doc/libs/1_79_0/boost/compute/detail/lru_cache.hpp
// Most important change is the use of an array instead of a map, so that elements are
// statically allocated. The insert and get methods have been merged into the request method.
// Original license:
//
//---------------------------------------------------------------------------//
// Copyright (c) 2013 Kyle Lutz <kyle.r.lutz@gmail.com>
//
// Distributed under the Boost Software License, Version 1.0
// See accompanying file LICENSE_1_0.txt or copy at
// http://www.boost.org/LICENSE_1_0.txt
//
// See http://boostorg.github.com/compute for more information.
//---------------------------------------------------------------------------//
#pragma once
#include <array>
#include <list>
#include <tuple>
#include <utility>
namespace Common {
// a cache which evicts the least recently used item when it is full
// the cache elements are statically allocated.
template <class Key, class Value, size_t Size>
class StaticLRUCache {
public:
using key_type = Key;
using value_type = Value;
using list_type = std::list<std::pair<Key, size_t>>;
using array_type = std::array<Value, Size>;
StaticLRUCache() = default;
~StaticLRUCache() = default;
size_t size() const {
return m_list.size();
}
constexpr size_t capacity() const {
return m_array.size();
}
bool empty() const {
return m_list.empty();
}
bool contains(const key_type& key) const {
return find(key) != m_list.end();
}
// Requests an element from the cache. If it is not found,
// the element is inserted using its key.
// Returns whether the element was present in the cache
// and a reference to the element itself.
std::pair<bool, value_type&> request(const key_type& key) {
// lookup value in the cache
auto i = find(key);
if (i == m_list.cend()) {
size_t next_index = size();
// insert item into the cache, but first check if it is full
if (next_index >= capacity()) {
// cache is full, evict the least recently used item
next_index = evict();
}
// insert the new item
m_list.push_front(std::make_pair(key, next_index));
return std::pair<bool, value_type&>(false, m_array[next_index]);
}
// return the value, but first update its place in the most
// recently used list
if (i != m_list.cbegin()) {
// move item to the front of the most recently used list
auto backup = *i;
m_list.erase(i);
m_list.push_front(backup);
// return the value
return std::pair<bool, value_type&>(true, m_array[backup.second]);
} else {
// the item is already at the front of the most recently
// used list so just return it
return std::pair<bool, value_type&>(true, m_array[i->second]);
}
}
void clear() {
m_list.clear();
}
private:
typename list_type::const_iterator find(const key_type& key) const {
return std::find_if(m_list.cbegin(), m_list.cend(),
[&key](const auto& el) { return el.first == key; });
}
size_t evict() {
// evict item from the end of most recently used list
typename list_type::iterator i = --m_list.end();
size_t evicted_index = i->second;
m_list.erase(i);
return evicted_index;
}
private:
array_type m_array;
list_type m_list;
};
} // namespace Common

View File

@ -86,6 +86,20 @@ public:
*/
virtual void Flush() const = 0;
/**
* Whether the backend supports cached reads.
*/
virtual bool AllowsCachedReads() const {
return false;
}
/**
* Whether the cache is ready for a specified offset and length.
*/
virtual bool CacheReady(std::size_t file_offset, std::size_t length) {
return false;
}
protected:
std::unique_ptr<DelayGenerator> delay_generator;

View File

@ -131,6 +131,14 @@ public:
}
void Flush() const override {}
bool AllowsCachedReads() const override {
return romfs_file->AllowsCachedReads();
}
bool CacheReady(std::size_t file_offset, std::size_t length) override {
return romfs_file->CacheReady(file_offset, length);
}
private:
std::shared_ptr<RomFSReader> romfs_file;

View File

@ -53,6 +53,14 @@ public:
bool DumpRomFS(const std::string& target_path);
bool AllowsCachedReads() const override {
return false;
}
bool CacheReady(std::size_t file_offset, std::size_t length) override {
return false;
}
private:
struct File;
struct Directory {

View File

@ -1,4 +1,5 @@
#include <algorithm>
#include <vector>
#include <cryptopp/aes.h>
#include <cryptopp/modes.h>
#include "common/archives.h"
@ -9,17 +10,102 @@ SERIALIZE_EXPORT_IMPL(FileSys::DirectRomFSReader)
namespace FileSys {
std::size_t DirectRomFSReader::ReadFile(std::size_t offset, std::size_t length, u8* buffer) {
length = std::min(length, static_cast<std::size_t>(data_size) - offset);
if (length == 0)
return 0; // Crypto++ does not like zero size buffer
file.Seek(file_offset + offset, SEEK_SET);
std::size_t read_length = std::min(length, static_cast<std::size_t>(data_size) - offset);
read_length = file.ReadBytes(buffer, read_length);
if (is_encrypted) {
CryptoPP::CTR_Mode<CryptoPP::AES>::Decryption d(key.data(), key.size(), ctr.data());
d.Seek(crypto_offset + offset);
d.ProcessData(buffer, buffer, read_length);
const auto segments = BreakupRead(offset, length);
size_t read_progress = 0;
// Skip cache if the read is too big
if (segments.size() == 1 && segments[0].second > cache_line_size) {
length = file.ReadAtBytes(buffer, length, file_offset + offset);
if (is_encrypted) {
CryptoPP::CTR_Mode<CryptoPP::AES>::Decryption d(key.data(), key.size(), ctr.data());
d.Seek(crypto_offset + offset);
d.ProcessData(buffer, buffer, length);
}
// LOG_INFO(Service_FS, "Cache SKIP: offset={}, length={}", offset, length);
return length;
}
return read_length;
// TODO(PabloMK7): Make cache thread safe, read the comment in CacheReady function.
// std::unique_lock<std::shared_mutex> read_guard(cache_mutex);
for (const auto& seg : segments) {
size_t read_size = cache_line_size;
size_t page = OffsetToPage(seg.first);
// Check if segment is in cache
auto cache_entry = cache.request(page);
if (!cache_entry.first) {
// If not found, read from disk and cache the data
read_size = file.ReadAtBytes(cache_entry.second.data(), read_size, file_offset + page);
if (is_encrypted && read_size) {
CryptoPP::CTR_Mode<CryptoPP::AES>::Decryption d(key.data(), key.size(), ctr.data());
d.Seek(crypto_offset + page);
d.ProcessData(cache_entry.second.data(), cache_entry.second.data(), read_size);
}
// LOG_INFO(Service_FS, "Cache MISS: page={}, length={}, into={}", page, seg.second,
// (seg.first - page));
} else {
// LOG_INFO(Service_FS, "Cache HIT: page={}, length={}, into={}", page, seg.second,
// (seg.first - page));
}
size_t copy_amount =
(read_size > (seg.first - page))
? std::min((seg.first - page) + seg.second, read_size) - (seg.first - page)
: 0;
std::memcpy(buffer + read_progress, cache_entry.second.data() + (seg.first - page),
copy_amount);
read_progress += copy_amount;
}
return read_progress;
}
bool DirectRomFSReader::AllowsCachedReads() const {
return true;
}
bool DirectRomFSReader::CacheReady(std::size_t file_offset, std::size_t length) {
auto segments = BreakupRead(file_offset, length);
if (segments.size() == 1 && segments[0].second > cache_line_size) {
return false;
} else {
// TODO(PabloMK7): Since the LRU cache is not thread safe, a lock must be used.
// However, this completely breaks the point of using a cache, because
// smaller reads may be blocked by bigger reads. For now, always return
// data being in cache to prevent the need of a lock, and only read data
// asynchronously if it is too big to use the cache.
/*
std::shared_lock<std::shared_mutex> read_guard(cache_mutex);
for (auto it = segments.begin(); it != segments.end(); it++) {
if (!cache.contains(OffsetToPage(it->first)))
return false;
}
*/
return true;
}
}
std::vector<std::pair<std::size_t, std::size_t>> DirectRomFSReader::BreakupRead(
std::size_t offset, std::size_t length) {
std::vector<std::pair<std::size_t, std::size_t>> ret;
// Reads bigger than the cache line size will probably never hit again
if (length > cache_line_size) {
ret.push_back(std::make_pair(offset, length));
return ret;
}
size_t curr_offset = offset;
while (length) {
size_t next_page = OffsetToPage(curr_offset + cache_line_size);
size_t curr_page_len = std::min(length, next_page - curr_offset);
ret.push_back(std::make_pair(curr_offset, curr_page_len));
curr_offset = next_page;
length -= curr_page_len;
}
return ret;
}
} // namespace FileSys

View File

@ -1,11 +1,14 @@
#pragma once
#include <array>
#include <shared_mutex>
#include <boost/serialization/array.hpp>
#include <boost/serialization/base_object.hpp>
#include <boost/serialization/export.hpp>
#include "common/alignment.h"
#include "common/common_types.h"
#include "common/file_util.h"
#include "common/static_lru_cache.h"
namespace FileSys {
@ -18,6 +21,8 @@ public:
virtual std::size_t GetSize() const = 0;
virtual std::size_t ReadFile(std::size_t offset, std::size_t length, u8* buffer) = 0;
virtual bool AllowsCachedReads() const = 0;
virtual bool CacheReady(std::size_t file_offset, std::size_t length) = 0;
private:
template <class Archive>
@ -48,6 +53,10 @@ public:
std::size_t ReadFile(std::size_t offset, std::size_t length, u8* buffer) override;
bool AllowsCachedReads() const override;
bool CacheReady(std::size_t file_offset, std::size_t length) override;
private:
bool is_encrypted;
FileUtil::IOFile file;
@ -57,8 +66,23 @@ private:
u64 crypto_offset;
u64 data_size;
// Total cache size: 128KB
static constexpr size_t cache_line_size = (1 << 13); // About 8KB
static constexpr size_t cache_line_count = 16;
Common::StaticLRUCache<std::size_t, std::array<u8, cache_line_size>, cache_line_count> cache;
// TODO(PabloMK7): Make cache thread safe, read the comment in CacheReady function.
// std::shared_mutex cache_mutex;
DirectRomFSReader() = default;
std::size_t OffsetToPage(std::size_t offset) {
return Common::AlignDown<std::size_t>(offset, cache_line_size);
}
std::vector<std::pair<std::size_t, std::size_t>> BreakupRead(std::size_t offset,
std::size_t length);
template <class Archive>
void serialize(Archive& ar, const unsigned int) {
ar& boost::serialization::base_object<RomFSReader>(*this);

View File

@ -221,6 +221,13 @@ public:
return session;
}
/**
* Returns the client thread that made the service request.
*/
std::shared_ptr<Thread> ClientThread() const {
return thread;
}
class WakeupCallback {
public:
virtual ~WakeupCallback() = default;

View File

@ -57,7 +57,6 @@ void File::Read(Kernel::HLERequestContext& ctx) {
IPC::RequestParser rp(ctx);
u64 offset = rp.Pop<u64>();
u32 length = rp.Pop<u32>();
auto& buffer = rp.PopMappedBuffer();
LOG_TRACE(Service_FS, "Read {}: offset=0x{:x} length=0x{:08X}", GetName(), offset, length);
const FileSessionSlot* file = GetSessionData(ctx.Session());
@ -76,22 +75,94 @@ void File::Read(Kernel::HLERequestContext& ctx) {
offset, length, backend->GetSize());
}
IPC::RequestBuilder rb = rp.MakeBuilder(2, 2);
// Conventional reading if the backend does not support cache.
if (!backend->AllowsCachedReads()) {
auto& buffer = rp.PopMappedBuffer();
IPC::RequestBuilder rb = rp.MakeBuilder(2, 2);
std::unique_ptr<u8*> data = std::make_unique<u8*>(static_cast<u8*>(operator new(length)));
const auto read = backend->Read(offset, length, *data);
if (read.Failed()) {
rb.Push(read.Code());
rb.Push<u32>(0);
} else {
buffer.Write(*data, 0, *read);
rb.Push(RESULT_SUCCESS);
rb.Push<u32>(static_cast<u32>(*read));
}
rb.PushMappedBuffer(buffer);
std::vector<u8> data(length);
ResultVal<std::size_t> read = backend->Read(offset, data.size(), data.data());
if (read.Failed()) {
rb.Push(read.Code());
rb.Push<u32>(0);
} else {
buffer.Write(data.data(), 0, *read);
rb.Push(RESULT_SUCCESS);
rb.Push<u32>(static_cast<u32>(*read));
std::chrono::nanoseconds read_timeout_ns{backend->GetReadDelayNs(length)};
ctx.SleepClientThread("file::read", read_timeout_ns, nullptr);
return;
}
rb.PushMappedBuffer(buffer);
std::chrono::nanoseconds read_timeout_ns{backend->GetReadDelayNs(length)};
ctx.SleepClientThread("file::read", read_timeout_ns, nullptr);
struct AsyncData {
// Input
u32 length;
u64 offset;
std::chrono::steady_clock::time_point pre_timer;
bool cache_ready;
// Output
ResultCode ret{0};
Kernel::MappedBuffer* buffer;
std::unique_ptr<u8*> data;
size_t read_size;
};
auto async_data = std::make_shared<AsyncData>();
async_data->buffer = &rp.PopMappedBuffer();
async_data->length = length;
async_data->offset = offset;
async_data->cache_ready = backend->CacheReady(offset, length);
if (!async_data->cache_ready) {
async_data->pre_timer = std::chrono::steady_clock::now();
}
// LOG_DEBUG(Service_FS, "cache={}, offset={}, length={}", cache_ready, offset, length);
ctx.RunAsync(
[this, async_data](Kernel::HLERequestContext& ctx) {
async_data->data =
std::make_unique<u8*>(static_cast<u8*>(operator new(async_data->length)));
const auto read =
backend->Read(async_data->offset, async_data->length, *async_data->data);
if (read.Failed()) {
async_data->ret = read.Code();
async_data->read_size = 0;
} else {
async_data->ret = RESULT_SUCCESS;
async_data->read_size = *read;
}
const auto read_delay = static_cast<s64>(backend->GetReadDelayNs(async_data->length));
if (!async_data->cache_ready) {
const auto time_took = std::chrono::duration_cast<std::chrono::nanoseconds>(
std::chrono::steady_clock::now() - async_data->pre_timer)
.count();
/*
if (time_took > read_delay) {
LOG_DEBUG(Service_FS, "Took longer! length={}, time_took={}, read_delay={}",
async_data->length, time_took, read_delay);
}
*/
return static_cast<s64>((read_delay > time_took) ? (read_delay - time_took) : 0);
} else {
return static_cast<s64>(read_delay);
}
},
[async_data](Kernel::HLERequestContext& ctx) {
IPC::RequestBuilder rb(ctx, 0x0802, 2, 2);
if (async_data->ret.IsError()) {
rb.Push(async_data->ret);
rb.Push<u32>(0);
} else {
async_data->buffer->Write(*async_data->data, 0, async_data->read_size);
rb.Push(RESULT_SUCCESS);
rb.Push<u32>(static_cast<u32>(async_data->read_size));
}
rb.PushMappedBuffer(*async_data->buffer);
},
!async_data->cache_ready);
}
void File::Write(Kernel::HLERequestContext& ctx) {

View File

@ -670,6 +670,26 @@ void FS_USER::GetFormatInfo(Kernel::HLERequestContext& ctx) {
rb.Push<bool>(format_info->duplicate_data != 0);
}
void FS_USER::GetProductInfo(Kernel::HLERequestContext& ctx) {
IPC::RequestParser rp(ctx);
u32 process_id = rp.Pop<u32>();
LOG_DEBUG(Service_FS, "called, process_id={}", process_id);
IPC::RequestBuilder rb = rp.MakeBuilder(6, 0);
const auto product_info = GetProductInfo(process_id);
if (!product_info.has_value()) {
rb.Push(ResultCode(FileSys::ErrCodes::ArchiveNotMounted, ErrorModule::FS,
ErrorSummary::NotFound, ErrorLevel::Status));
return;
}
rb.Push(RESULT_SUCCESS);
rb.PushRaw<ProductInfo>(product_info.value());
}
void FS_USER::GetProgramLaunchInfo(Kernel::HLERequestContext& ctx) {
IPC::RequestParser rp(ctx);
const auto process_id = rp.Pop<u32>();
@ -687,8 +707,20 @@ void FS_USER::GetProgramLaunchInfo(Kernel::HLERequestContext& ctx) {
return;
}
ProgramInfo program_info = program_info_result.Unwrap();
// Always report the launched program mediatype is SD if the friends module is requesting this
// information and the media type is game card. Otherwise, friends will append a "romid" field
// to the NASC request with a cartridge unique identifier. Using a dump of a game card and the
// game card itself at the same time online is known to have caused issues in the past.
auto process = ctx.ClientThread()->owner_process.lock();
if (process && process->codeset->name == "friends" &&
program_info.media_type == MediaType::GameCard) {
program_info.media_type = MediaType::SDMC;
}
rb.Push(RESULT_SUCCESS);
rb.PushRaw(program_info_result.Unwrap());
rb.PushRaw<ProgramInfo>(program_info);
}
void FS_USER::ObsoletedCreateExtSaveData(Kernel::HLERequestContext& ctx) {
@ -775,12 +807,12 @@ void FS_USER::AddSeed(Kernel::HLERequestContext& ctx) {
rb.Push(RESULT_SUCCESS);
}
void FS_USER::SetSaveDataSecureValue(Kernel::HLERequestContext& ctx) {
void FS_USER::ObsoletedSetSaveDataSecureValue(Kernel::HLERequestContext& ctx) {
IPC::RequestParser rp(ctx);
u64 value = rp.Pop<u64>();
u32 secure_value_slot = rp.Pop<u32>();
u32 unique_id = rp.Pop<u32>();
u8 title_variation = rp.Pop<u8>();
const u64 value = rp.Pop<u64>();
const u32 secure_value_slot = rp.Pop<u32>();
const u32 unique_id = rp.Pop<u32>();
const u8 title_variation = rp.Pop<u8>();
// TODO: Generate and Save the Secure Value
@ -794,12 +826,11 @@ void FS_USER::SetSaveDataSecureValue(Kernel::HLERequestContext& ctx) {
rb.Push(RESULT_SUCCESS);
}
void FS_USER::GetSaveDataSecureValue(Kernel::HLERequestContext& ctx) {
void FS_USER::ObsoletedGetSaveDataSecureValue(Kernel::HLERequestContext& ctx) {
IPC::RequestParser rp(ctx);
u32 secure_value_slot = rp.Pop<u32>();
u32 unique_id = rp.Pop<u32>();
u8 title_variation = rp.Pop<u8>();
const u32 secure_value_slot = rp.Pop<u32>();
const u32 unique_id = rp.Pop<u32>();
const u8 title_variation = rp.Pop<u8>();
LOG_WARNING(
Service_FS,
@ -816,7 +847,77 @@ void FS_USER::GetSaveDataSecureValue(Kernel::HLERequestContext& ctx) {
rb.Push<u64>(0); // the secure value
}
void FS_USER::Register(u32 process_id, u64 program_id, const std::string& filepath) {
void FS_USER::SetThisSaveDataSecureValue(Kernel::HLERequestContext& ctx) {
IPC::RequestParser rp(ctx);
const u32 secure_value_slot = rp.Pop<u32>();
const u64 value = rp.Pop<u64>();
// TODO: Generate and Save the Secure Value
LOG_WARNING(Service_FS, "(STUBBED) called, value=0x{:016x} secure_value_slot=0x{:08X}", value,
secure_value_slot);
IPC::RequestBuilder rb = rp.MakeBuilder(1, 0);
rb.Push(RESULT_SUCCESS);
}
void FS_USER::GetThisSaveDataSecureValue(Kernel::HLERequestContext& ctx) {
IPC::RequestParser rp(ctx);
const u32 secure_value_slot = rp.Pop<u32>();
LOG_WARNING(Service_FS, "(STUBBED) called secure_value_slot=0x{:08X}", secure_value_slot);
IPC::RequestBuilder rb = rp.MakeBuilder(5, 0);
rb.Push(RESULT_SUCCESS);
// TODO: Implement Secure Value Lookup & Generation
rb.Push<bool>(false); // indicates that the secure value doesn't exist
rb.Push<bool>(false); // looks like a boolean value, purpose unknown
rb.Push<u64>(0); // the secure value
}
void FS_USER::SetSaveDataSecureValue(Kernel::HLERequestContext& ctx) {
IPC::RequestParser rp(ctx);
const auto archive_handle = rp.PopRaw<ArchiveHandle>();
const u32 secure_value_slot = rp.Pop<u32>();
const u64 value = rp.Pop<u64>();
const bool flush = rp.Pop<bool>();
// TODO: Generate and Save the Secure Value
LOG_WARNING(Service_FS,
"(STUBBED) called, value=0x{:016x} secure_value_slot=0x{:04X} "
"archive_handle=0x{:08X} flush={}",
value, secure_value_slot, archive_handle, flush);
IPC::RequestBuilder rb = rp.MakeBuilder(1, 0);
rb.Push(RESULT_SUCCESS);
}
void FS_USER::GetSaveDataSecureValue(Kernel::HLERequestContext& ctx) {
IPC::RequestParser rp(ctx);
const auto archive_handle = rp.PopRaw<ArchiveHandle>();
const u32 secure_value_slot = rp.Pop<u32>();
LOG_WARNING(Service_FS, "(STUBBED) called secure_value_slot=0x{:08X} archive_handle=0x{:08X}",
secure_value_slot, archive_handle);
IPC::RequestBuilder rb = rp.MakeBuilder(5, 0);
rb.Push(RESULT_SUCCESS);
// TODO: Implement Secure Value Lookup & Generation
rb.Push<bool>(false); // indicates that the secure value doesn't exist
rb.Push<bool>(false); // looks like a boolean value, purpose unknown
rb.Push<u64>(0); // the secure value
}
void FS_USER::RegisterProgramInfo(u32 process_id, u64 program_id, const std::string& filepath) {
const MediaType media_type = GetMediaTypeFromPath(filepath);
program_info_map.insert_or_assign(process_id, ProgramInfo{program_id, media_type});
if (media_type == MediaType::GameCard) {
@ -828,6 +929,19 @@ std::string FS_USER::GetCurrentGamecardPath() const {
return current_gamecard_path;
}
void FS_USER::RegisterProductInfo(u32 process_id, const ProductInfo& product_info) {
product_info_map.insert_or_assign(process_id, product_info);
}
std::optional<FS_USER::ProductInfo> FS_USER::GetProductInfo(u32 process_id) {
auto it = product_info_map.find(process_id);
if (it != product_info_map.end()) {
return it->second;
} else {
return {};
}
}
ResultVal<u16> FS_USER::GetSpecialContentIndexFromGameCard(u64 title_id, SpecialContentType type) {
// TODO(B3N30) check if on real 3DS NCSD is checked if partition exists
@ -929,7 +1043,7 @@ FS_USER::FS_USER(Core::System& system)
{0x082B, nullptr, "CardNorDirectRead_4xIO"},
{0x082C, nullptr, "CardNorDirectCpuWriteWithoutVerify"},
{0x082D, nullptr, "CardNorDirectSectorEraseWithoutVerify"},
{0x082E, nullptr, "GetProductInfo"},
{0x082E, &FS_USER::GetProductInfo, "GetProductInfo"},
{0x082F, &FS_USER::GetProgramLaunchInfo, "GetProgramLaunchInfo"},
{0x0830, &FS_USER::ObsoletedCreateExtSaveData, "Obsoleted_3_0_CreateExtSaveData"},
{0x0831, nullptr, "CreateSharedExtSaveData"},
@ -984,12 +1098,16 @@ FS_USER::FS_USER(Core::System& system)
{0x0862, &FS_USER::SetPriority, "SetPriority"},
{0x0863, &FS_USER::GetPriority, "GetPriority"},
{0x0864, nullptr, "GetNandInfo"},
{0x0865, &FS_USER::SetSaveDataSecureValue, "SetSaveDataSecureValue"},
{0x0866, &FS_USER::GetSaveDataSecureValue, "GetSaveDataSecureValue"},
{0x0865, &FS_USER::ObsoletedSetSaveDataSecureValue, "SetSaveDataSecureValue"},
{0x0866, &FS_USER::ObsoletedGetSaveDataSecureValue, "GetSaveDataSecureValue"},
{0x0867, nullptr, "ControlSecureSave"},
{0x0868, nullptr, "GetMediaType"},
{0x0869, nullptr, "GetNandEraseCount"},
{0x086A, nullptr, "ReadNandReport"},
{0x086E, &FS_USER::SetThisSaveDataSecureValue, "SetThisSaveDataSecureValue" },
{0x086F, &FS_USER::GetThisSaveDataSecureValue, "GetThisSaveDataSecureValue" },
{0x0875, &FS_USER::SetSaveDataSecureValue, "SetSaveDataSecureValue" },
{0x0876, &FS_USER::GetSaveDataSecureValue, "GetSaveDataSecureValue" },
{0x087A, &FS_USER::AddSeed, "AddSeed"},
{0x087D, &FS_USER::GetNumSeeds, "GetNumSeeds"},
{0x0886, nullptr, "CheckUpdatedDat"},

View File

@ -4,10 +4,12 @@
#pragma once
#include <optional>
#include <unordered_map>
#include <boost/serialization/base_object.hpp>
#include "common/common_types.h"
#include "core/file_sys/errors.h"
#include "core/hle/service/fs/archive.h"
#include "core/hle/service/service.h"
namespace Core {
@ -47,12 +49,23 @@ class FS_USER final : public ServiceFramework<FS_USER, ClientSlot> {
public:
explicit FS_USER(Core::System& system);
// On real HW this is part of FS:Reg. But since that module is only used by loader and pm, which
// we HLEed, we can just directly use it here
void Register(u32 process_id, u64 program_id, const std::string& filepath);
// On real HW this is part of FSReg (FSReg:Register). But since that module is only used by
// loader and pm, which we HLEed, we can just directly use it here
void RegisterProgramInfo(u32 process_id, u64 program_id, const std::string& filepath);
std::string GetCurrentGamecardPath() const;
struct ProductInfo {
std::array<u8, 0x10> product_code;
u16_le maker_code;
u16_le remaster_version;
};
static_assert(sizeof(ProductInfo) == 0x14);
void RegisterProductInfo(u32 process_id, const ProductInfo& product_info);
std::optional<ProductInfo> GetProductInfo(u32 process_id);
/// Gets the registered program info of a process.
ResultVal<ProgramInfo> GetProgramLaunchInfo(u32 process_id) const {
auto info = program_info_map.find(process_id);
@ -509,6 +522,17 @@ private:
*/
void GetFormatInfo(Kernel::HLERequestContext& ctx);
/**
* FS_User::GetProductInfo service function.
* Inputs:
* 0 : 0x082E0040
* 1 : Process ID
* Outputs:
* 1 : Result of function, 0 on success, otherwise error code
* 2-6 : Product info
*/
void GetProductInfo(Kernel::HLERequestContext& ctx);
/**
* FS_User::GetProgramLaunchInfo service function.
* Inputs:
@ -600,7 +624,7 @@ private:
* 0 : 0x08650140
* 1 : Result of function, 0 on success, otherwise error code
*/
void SetSaveDataSecureValue(Kernel::HLERequestContext& ctx);
void ObsoletedSetSaveDataSecureValue(Kernel::HLERequestContext& ctx);
/**
* FS_User::GetSaveDataSecureValue service function.
@ -615,6 +639,57 @@ private:
* 2 : If Secure Value doesn't exist, 0, if it exists, 1
* 3-4 : Secure Value
*/
void ObsoletedGetSaveDataSecureValue(Kernel::HLERequestContext& ctx);
/**
* FS_User::SetThisSaveDataSecureValue service function.
* Inputs:
* 1 : Secure Value Slot
* 2-3 : Secure Value
* Outputs:
* 1 : Result of function, 0 on success, otherwise error code
*/
void SetThisSaveDataSecureValue(Kernel::HLERequestContext& ctx);
/**
* FS_User::GetSaveDataSecureValue service function.
* Inputs:
* 1 : Secure Value Slot
* Outputs:
* 1 : Result of function, 0 on success, otherwise error code
* 2 : If Secure Value doesn't exist, 0, if it exists, 1
* 3 : Unknown
* 4-5 : Secure Value
*/
void GetThisSaveDataSecureValue(Kernel::HLERequestContext& ctx);
/**
* FS_User::SetSaveDataSecureValue service function.
* Inputs:
* 0 : 0x08750180
* 1-2 : Archive
* 3 : Secure Value Slot
* 4 : value
* 5 : flush
* Outputs:
* 0 : header
* 1 : Result of function, 0 on success, otherwise error code
*/
void SetSaveDataSecureValue(Kernel::HLERequestContext& ctx);
/**
* FS_User::GetSaveDataSecureValue service function.
* Inputs:
* 0 : 0x087600C0
* 1-2 : Archive
* 2 : Secure Value slot
* Outputs:
* 0 : Header
* 1 : Result of function, 0 on success, otherwise error code
* 2 : If Secure Value doesn't exist, 0, if it exists, 1
* 3 : unknown
* 4-5 : Secure Value
*/
void GetSaveDataSecureValue(Kernel::HLERequestContext& ctx);
static ResultVal<u16> GetSpecialContentIndexFromGameCard(u64 title_id, SpecialContentType type);
@ -624,6 +699,8 @@ private:
std::unordered_map<u32, ProgramInfo> program_info_map;
std::string current_gamecard_path;
std::unordered_map<u32, ProductInfo> product_info_map;
u32 priority = -1; ///< For SetPriority and GetPriority service functions
Core::System& system;

View File

@ -282,7 +282,7 @@ ResultStatus AppLoader_THREEDSX::Load(std::shared_ptr<Kernel::Process>& process)
// On real HW this is done with FS:Reg, but we can be lazy
auto fs_user =
Core::System::GetInstance().ServiceManager().GetService<Service::FS::FS_USER>("fs:USER");
fs_user->Register(process->GetObjectId(), process->codeset->program_id, filepath);
fs_user->RegisterProgramInfo(process->GetObjectId(), process->codeset->program_id, filepath);
process->Run(48, Kernel::DEFAULT_STACK_SIZE);

View File

@ -174,7 +174,16 @@ ResultStatus AppLoader_NCCH::LoadExec(std::shared_ptr<Kernel::Process>& process)
auto fs_user =
Core::System::GetInstance().ServiceManager().GetService<Service::FS::FS_USER>(
"fs:USER");
fs_user->Register(process->process_id, process->codeset->program_id, filepath);
fs_user->RegisterProgramInfo(process->process_id, process->codeset->program_id, filepath);
Service::FS::FS_USER::ProductInfo product_info{};
std::memcpy(product_info.product_code.data(), overlay_ncch->ncch_header.product_code,
product_info.product_code.size());
std::memcpy(&product_info.remaster_version,
overlay_ncch->exheader_header.codeset_info.flags.remaster_version,
sizeof(product_info.remaster_version));
product_info.maker_code = overlay_ncch->ncch_header.maker_code;
fs_user->RegisterProductInfo(process->process_id, product_info);
process->Run(priority, stack_size);
return ResultStatus::Success;

View File

@ -78,12 +78,11 @@ static void WriteUniformFloatReg(ShaderRegs& config, Shader::ShaderSetup& setup,
(float_regs_counter >= 3 && !uniform_setup.IsFloat32())) {
float_regs_counter = 0;
auto& uniform = setup.uniforms.f[uniform_setup.index];
if (uniform_setup.index >= 96) {
if (uniform_setup.index >= setup.uniforms.f.size()) {
LOG_ERROR(HW_GPU, "Invalid {} float uniform index {}", GetShaderSetupTypeName(setup),
(int)uniform_setup.index);
} else {
auto& uniform = setup.uniforms.f[uniform_setup.index];
// NOTE: The destination component order indeed is "backwards"
if (uniform_setup.IsFloat32()) {

View File

@ -78,8 +78,8 @@ RasterizerOpenGL::RasterizerOpenGL(Memory::MemorySystem& memory,
VideoCore::CustomTexManager& custom_tex_manager,
VideoCore::RendererBase& renderer, Driver& driver_)
: VideoCore::RasterizerAccelerated{memory}, driver{driver_},
shader_manager{renderer.GetRenderWindow(), driver, !driver.IsOpenGLES()},
runtime{driver, renderer}, res_cache{memory, custom_tex_manager, runtime, regs, renderer},
shader_manager{renderer.GetRenderWindow(), driver}, runtime{driver, renderer},
res_cache{memory, custom_tex_manager, runtime, regs, renderer},
texture_buffer_size{TextureBufferSize()}, vertex_buffer{driver, GL_ARRAY_BUFFER,
VERTEX_BUFFER_SIZE},
uniform_buffer{driver, GL_UNIFORM_BUFFER, UNIFORM_BUFFER_SIZE},

View File

@ -2,14 +2,10 @@
// Licensed under GPLv2 or any later version
// Refer to the license.txt file included.
#include "common/microprofile.h"
#include "video_core/renderer_opengl/gl_resource_manager.h"
#include "video_core/renderer_opengl/gl_shader_util.h"
#include "video_core/renderer_opengl/gl_state.h"
MICROPROFILE_DEFINE(OpenGL_ResourceCreation, "OpenGL", "Resource Creation", MP_RGB(128, 128, 192));
MICROPROFILE_DEFINE(OpenGL_ResourceDeletion, "OpenGL", "Resource Deletion", MP_RGB(128, 128, 192));
namespace OpenGL {
void OGLRenderbuffer::Create() {
@ -17,7 +13,6 @@ void OGLRenderbuffer::Create() {
return;
}
MICROPROFILE_SCOPE(OpenGL_ResourceCreation);
glGenRenderbuffers(1, &handle);
}
@ -26,7 +21,6 @@ void OGLRenderbuffer::Release() {
return;
}
MICROPROFILE_SCOPE(OpenGL_ResourceDeletion);
glDeleteRenderbuffers(1, &handle);
OpenGLState::GetCurState().ResetRenderbuffer(handle).Apply();
handle = 0;
@ -37,7 +31,6 @@ void OGLTexture::Create() {
return;
}
MICROPROFILE_SCOPE(OpenGL_ResourceCreation);
glGenTextures(1, &handle);
}
@ -46,7 +39,6 @@ void OGLTexture::Release() {
return;
}
MICROPROFILE_SCOPE(OpenGL_ResourceDeletion);
glDeleteTextures(1, &handle);
OpenGLState::GetCurState().ResetTexture(handle).Apply();
handle = 0;
@ -88,7 +80,6 @@ void OGLSampler::Create() {
return;
}
MICROPROFILE_SCOPE(OpenGL_ResourceCreation);
glGenSamplers(1, &handle);
}
@ -97,37 +88,41 @@ void OGLSampler::Release() {
return;
}
MICROPROFILE_SCOPE(OpenGL_ResourceDeletion);
glDeleteSamplers(1, &handle);
OpenGLState::GetCurState().ResetSampler(handle).Apply();
handle = 0;
}
void OGLShader::Create(std::string_view source, GLenum type) {
if (handle != 0)
if (handle != 0) {
return;
if (source.empty())
}
if (source.empty()) {
return;
}
MICROPROFILE_SCOPE(OpenGL_ResourceCreation);
handle = LoadShader(source, type);
}
void OGLShader::Release() {
if (handle == 0)
if (handle == 0) {
return;
}
MICROPROFILE_SCOPE(OpenGL_ResourceDeletion);
glDeleteShader(handle);
handle = 0;
}
void OGLProgram::Create(bool separable_program, std::span<const GLuint> shaders) {
if (handle != 0)
void OGLProgram::Create(std::string_view source, GLenum type) {
if (handle != 0) {
return;
}
if (source.empty()) {
return;
}
MICROPROFILE_SCOPE(OpenGL_ResourceCreation);
handle = LoadProgram(separable_program, shaders);
const std::array sources{GetPreamble().data(), source.data()};
handle = glCreateShaderProgramv(type, 2, sources.data());
}
void OGLProgram::Create(std::string_view vert_shader, std::string_view frag_shader) {
@ -135,88 +130,87 @@ void OGLProgram::Create(std::string_view vert_shader, std::string_view frag_shad
vert.Create(vert_shader, GL_VERTEX_SHADER);
frag.Create(frag_shader, GL_FRAGMENT_SHADER);
MICROPROFILE_SCOPE(OpenGL_ResourceCreation);
const std::array shaders{vert.handle, frag.handle};
Create(false, shaders);
handle = LoadProgram(shaders);
}
void OGLProgram::Release() {
if (handle == 0)
if (handle == 0) {
return;
}
MICROPROFILE_SCOPE(OpenGL_ResourceDeletion);
glDeleteProgram(handle);
OpenGLState::GetCurState().ResetProgram(handle).Apply();
handle = 0;
}
void OGLPipeline::Create() {
if (handle != 0)
if (handle != 0) {
return;
}
MICROPROFILE_SCOPE(OpenGL_ResourceCreation);
glGenProgramPipelines(1, &handle);
}
void OGLPipeline::Release() {
if (handle == 0)
if (handle == 0) {
return;
}
MICROPROFILE_SCOPE(OpenGL_ResourceDeletion);
glDeleteProgramPipelines(1, &handle);
OpenGLState::GetCurState().ResetPipeline(handle).Apply();
handle = 0;
}
void OGLBuffer::Create() {
if (handle != 0)
if (handle != 0) {
return;
}
MICROPROFILE_SCOPE(OpenGL_ResourceCreation);
glGenBuffers(1, &handle);
}
void OGLBuffer::Release() {
if (handle == 0)
if (handle == 0) {
return;
}
MICROPROFILE_SCOPE(OpenGL_ResourceDeletion);
glDeleteBuffers(1, &handle);
OpenGLState::GetCurState().ResetBuffer(handle).Apply();
handle = 0;
}
void OGLVertexArray::Create() {
if (handle != 0)
if (handle != 0) {
return;
}
MICROPROFILE_SCOPE(OpenGL_ResourceCreation);
glGenVertexArrays(1, &handle);
}
void OGLVertexArray::Release() {
if (handle == 0)
if (handle == 0) {
return;
}
MICROPROFILE_SCOPE(OpenGL_ResourceDeletion);
glDeleteVertexArrays(1, &handle);
OpenGLState::GetCurState().ResetVertexArray(handle).Apply();
handle = 0;
}
void OGLFramebuffer::Create() {
if (handle != 0)
if (handle != 0) {
return;
}
MICROPROFILE_SCOPE(OpenGL_ResourceCreation);
glGenFramebuffers(1, &handle);
}
void OGLFramebuffer::Release() {
if (handle == 0)
if (handle == 0) {
return;
}
MICROPROFILE_SCOPE(OpenGL_ResourceDeletion);
glDeleteFramebuffers(1, &handle);
OpenGLState::GetCurState().ResetFramebuffer(handle).Apply();
handle = 0;

View File

@ -130,7 +130,7 @@ public:
}
/// Creates a new program from given shader objects
void Create(bool separable_program, std::span<const GLuint> shaders);
void Create(std::string_view source, GLenum type);
/// Creates a new program from given shader soruce code
void Create(std::string_view vert_shader, std::string_view frag_shader);

View File

@ -103,10 +103,8 @@ bool ShaderDiskCacheRaw::Save(FileUtil::IOFile& file) const {
return true;
}
ShaderDiskCache::ShaderDiskCache(bool separable)
: separable{separable}, transferable_file(AppendTransferableFile()),
// seperable shaders use the virtual precompile file, that already has a header.
precompiled_file(AppendPrecompiledFile(!separable)) {}
ShaderDiskCache::ShaderDiskCache()
: transferable_file(AppendTransferableFile()), precompiled_file(AppendPrecompiledFile()) {}
std::optional<std::vector<ShaderDiskCacheRaw>> ShaderDiskCache::LoadTransferable() {
const bool has_title_id = GetProgramID() != 0;
@ -177,7 +175,7 @@ std::optional<std::vector<ShaderDiskCacheRaw>> ShaderDiskCache::LoadTransferable
}
std::pair<std::unordered_map<u64, ShaderDiskCacheDecompiled>, ShaderDumpsMap>
ShaderDiskCache::LoadPrecompiled(bool compressed) {
ShaderDiskCache::LoadPrecompiled() {
if (!IsUsable())
return {};
@ -187,7 +185,7 @@ ShaderDiskCache::LoadPrecompiled(bool compressed) {
return {};
}
const auto result = LoadPrecompiledFile(precompiled_file, compressed);
const auto result = LoadPrecompiledFile(precompiled_file);
if (!result) {
LOG_INFO(Render_OpenGL,
"Failed to load precompiled cache for game with title id={} - removing",
@ -199,22 +197,16 @@ ShaderDiskCache::LoadPrecompiled(bool compressed) {
}
std::optional<std::pair<std::unordered_map<u64, ShaderDiskCacheDecompiled>, ShaderDumpsMap>>
ShaderDiskCache::LoadPrecompiledFile(FileUtil::IOFile& file, bool compressed) {
ShaderDiskCache::LoadPrecompiledFile(FileUtil::IOFile& file) {
// Read compressed file from disk and decompress to virtual precompiled cache file
std::vector<u8> precompiled_file(file.GetSize());
file.ReadBytes(precompiled_file.data(), precompiled_file.size());
if (compressed) {
const std::vector<u8> decompressed =
Common::Compression::DecompressDataZSTD(precompiled_file);
if (decompressed.empty()) {
LOG_ERROR(Render_OpenGL, "Could not decompress precompiled shader cache.");
return std::nullopt;
}
SaveArrayToPrecompiled(decompressed.data(), decompressed.size());
} else {
SaveArrayToPrecompiled(precompiled_file.data(), precompiled_file.size());
const std::vector<u8> decompressed = Common::Compression::DecompressDataZSTD(precompiled_file);
if (decompressed.empty()) {
LOG_ERROR(Render_OpenGL, "Could not decompress precompiled shader cache.");
return std::nullopt;
}
SaveArrayToPrecompiled(decompressed.data(), decompressed.size());
decompressed_precompiled_cache_offset = 0;
ShaderCacheVersionHash file_hash{};
@ -353,7 +345,7 @@ void ShaderDiskCache::InvalidatePrecompiled() {
if (!FileUtil::Delete(GetPrecompiledPath())) {
LOG_ERROR(Render_OpenGL, "Failed to invalidate precompiled file={}", GetPrecompiledPath());
}
precompiled_file = AppendPrecompiledFile(!separable);
precompiled_file = AppendPrecompiledFile();
}
void ShaderDiskCache::SaveRaw(const ShaderDiskCacheRaw& entry) {
@ -471,12 +463,11 @@ FileUtil::IOFile ShaderDiskCache::AppendTransferableFile() {
return file;
}
FileUtil::IOFile ShaderDiskCache::AppendPrecompiledFile(bool write_header) {
FileUtil::IOFile ShaderDiskCache::AppendPrecompiledFile() {
if (!EnsureDirectories())
return {};
const auto precompiled_path{GetPrecompiledPath()};
const bool existed = FileUtil::Exists(precompiled_path);
FileUtil::IOFile file(precompiled_path, "ab+");
if (!file.IsOpen()) {
@ -484,15 +475,6 @@ FileUtil::IOFile ShaderDiskCache::AppendPrecompiledFile(bool write_header) {
return {};
}
// If the file didn't exist, write its version
if (write_header && (!existed || file.GetSize() == 0)) {
const auto hash{GetShaderCacheVersionHash()};
if (file.WriteArray(hash.data(), hash.size()) != hash.size()) {
LOG_ERROR(Render_OpenGL, "Failed to write precompiled cache version in path={}",
precompiled_path);
return {};
}
}
return file;
}
@ -516,7 +498,7 @@ void ShaderDiskCache::SaveVirtualPrecompiledFile() {
if (!FileUtil::Delete(GetPrecompiledPath())) {
LOG_ERROR(Render_OpenGL, "Failed to invalidate precompiled file={}", GetPrecompiledPath());
}
precompiled_file = AppendPrecompiledFile(!separable);
precompiled_file = AppendPrecompiledFile();
if (precompiled_file.WriteBytes(compressed.data(), compressed.size()) != compressed.size()) {
LOG_ERROR(Render_OpenGL, "Failed to write precompiled cache version in path={}",
@ -558,10 +540,7 @@ std::string ShaderDiskCache::GetPrecompiledDir() const {
}
std::string ShaderDiskCache::GetPrecompiledShaderDir() const {
if (separable) {
return GetPrecompiledDir() + DIR_SEP "separable";
}
return GetPrecompiledDir() + DIR_SEP "conventional";
return GetPrecompiledDir() + DIR_SEP "separable";
}
std::string ShaderDiskCache::GetBaseDir() const {

View File

@ -16,11 +16,8 @@
#include <glad/glad.h>
#include "common/assert.h"
#include "common/common_types.h"
#include "common/file_util.h"
#include "video_core/regs.h"
#include "video_core/shader/generator/glsl_shader_gen.h"
#include "video_core/shader/generator/shader_gen.h"
namespace Core {
class System;
@ -90,14 +87,14 @@ struct ShaderDiskCacheDump {
class ShaderDiskCache {
public:
explicit ShaderDiskCache(bool separable);
explicit ShaderDiskCache();
~ShaderDiskCache() = default;
/// Loads transferable cache. If file has a old version or on failure, it deletes the file.
std::optional<std::vector<ShaderDiskCacheRaw>> LoadTransferable();
/// Loads current game's precompiled cache. Invalidates on failure.
std::pair<ShaderDecompiledMap, ShaderDumpsMap> LoadPrecompiled(bool compressed);
std::pair<ShaderDecompiledMap, ShaderDumpsMap> LoadPrecompiled();
/// Removes the transferable (and precompiled) cache file.
void InvalidateAll();
@ -123,7 +120,7 @@ public:
private:
/// Loads the transferable cache. Returns empty on failure.
std::optional<std::pair<ShaderDecompiledMap, ShaderDumpsMap>> LoadPrecompiledFile(
FileUtil::IOFile& file, bool compressed);
FileUtil::IOFile& file);
/// Loads a decompiled cache entry from m_precompiled_cache_virtual_file. Returns empty on
/// failure.
@ -143,7 +140,7 @@ private:
FileUtil::IOFile AppendTransferableFile();
/// Opens current game's precompiled file and write it's header if it doesn't exist
FileUtil::IOFile AppendPrecompiledFile(bool write_header);
FileUtil::IOFile AppendPrecompiledFile();
/// Save precompiled header to precompiled_cache_in_memory
void SavePrecompiledHeaderToVirtualPrecompiledCache();
@ -219,8 +216,6 @@ private:
// The cache has been loaded at boot
bool tried_to_load{};
bool separable{};
u64 program_id{};
std::string title_id;

View File

@ -14,7 +14,7 @@
#include "video_core/renderer_opengl/gl_shader_disk_cache.h"
#include "video_core/renderer_opengl/gl_shader_manager.h"
#include "video_core/renderer_opengl/gl_state.h"
#include "video_core/shader/generator/shader_uniforms.h"
#include "video_core/shader/generator/glsl_shader_gen.h"
#include "video_core/video_core.h"
using namespace Pica::Shader::Generator;
@ -35,19 +35,16 @@ static u64 GetUniqueIdentifier(const Pica::Regs& regs, const ProgramCode& code)
}
static OGLProgram GeneratePrecompiledProgram(const ShaderDiskCacheDump& dump,
const std::set<GLenum>& supported_formats,
bool separable) {
const std::set<GLenum>& supported_formats) {
if (supported_formats.find(dump.binary_format) == supported_formats.end()) {
LOG_INFO(Render_OpenGL, "Precompiled cache entry with unsupported format - removing");
return {};
}
auto shader = OGLProgram();
OGLProgram shader{};
shader.handle = glCreateProgram();
if (separable) {
glProgramParameteri(shader.handle, GL_PROGRAM_SEPARABLE, GL_TRUE);
}
glProgramParameteri(shader.handle, GL_PROGRAM_SEPARABLE, GL_TRUE);
glProgramBinary(shader.handle, dump.binary_format, dump.binary.data(),
static_cast<GLsizei>(dump.binary.size()));
@ -90,91 +87,42 @@ static std::tuple<PicaVSConfig, Pica::Shader::ShaderSetup> BuildVSConfigFromRaw(
setup};
}
/**
* An object representing a shader program staging. It can be either a shader object or a program
* object, depending on whether separable program is used.
*/
class OGLShaderStage {
public:
explicit OGLShaderStage(bool separable) {
if (separable) {
shader_or_program = OGLProgram();
} else {
shader_or_program = OGLShader();
}
}
void Create(const char* source, GLenum type) {
if (shader_or_program.index() == 0) {
std::get<OGLShader>(shader_or_program).Create(source, type);
} else {
OGLShader shader;
shader.Create(source, type);
OGLProgram& program = std::get<OGLProgram>(shader_or_program);
program.Create(true, std::array{shader.handle});
}
}
GLuint GetHandle() const {
if (shader_or_program.index() == 0) {
return std::get<OGLShader>(shader_or_program).handle;
} else {
return std::get<OGLProgram>(shader_or_program).handle;
}
}
void Inject(OGLProgram&& program) {
shader_or_program = std::move(program);
}
private:
std::variant<OGLShader, OGLProgram> shader_or_program;
};
class TrivialVertexShader {
public:
explicit TrivialVertexShader(const Driver& driver, bool separable) : program(separable) {
const auto code =
GLSL::GenerateTrivialVertexShader(driver.HasClipCullDistance(), separable);
program.Create(code.c_str(), GL_VERTEX_SHADER);
explicit TrivialVertexShader(const Driver& driver) {
const auto code = GLSL::GenerateTrivialVertexShader(driver.HasClipCullDistance(), true);
program.Create(code, GL_VERTEX_SHADER);
}
GLuint Get() const {
return program.GetHandle();
return program.handle;
}
private:
OGLShaderStage program;
OGLProgram program;
};
template <typename KeyConfigType, std::string (*CodeGenerator)(const KeyConfigType&, bool),
GLenum ShaderType>
template <typename KeyConfigType, auto CodeGenerator, GLenum ShaderType>
class ShaderCache {
public:
explicit ShaderCache(bool separable) : separable(separable) {}
explicit ShaderCache() = default;
std::tuple<GLuint, std::optional<std::string>> Get(const KeyConfigType& config) {
auto [iter, new_shader] = shaders.emplace(config, OGLShaderStage{separable});
OGLShaderStage& cached_shader = iter->second;
auto [iter, new_shader] = shaders.try_emplace(config);
OGLProgram& cached_shader = iter->second;
std::optional<std::string> result{};
if (new_shader) {
result = CodeGenerator(config, separable);
cached_shader.Create(result->c_str(), ShaderType);
result = CodeGenerator(config, true);
cached_shader.Create(result.value(), ShaderType);
}
return {cached_shader.GetHandle(), std::move(result)};
return {cached_shader.handle, std::move(result)};
}
void Inject(const KeyConfigType& key, OGLProgram&& program) {
OGLShaderStage stage{separable};
stage.Inject(std::move(program));
shaders.emplace(key, std::move(stage));
}
void Inject(const KeyConfigType& key, OGLShaderStage&& stage) {
void Inject(const KeyConfigType& key, OGLProgram&& stage) {
shaders.emplace(key, std::move(stage));
}
private:
bool separable;
std::unordered_map<KeyConfigType, OGLShaderStage> shaders;
std::unordered_map<KeyConfigType, OGLProgram> shaders;
};
// This is a cache designed for shaders translated from PICA shaders. The first cache matches the
@ -182,59 +130,48 @@ private:
// GLSL code. The configuration is like this because there might be leftover code in the PICA shader
// program buffer from the previous shader, which is hashed into the config, resulting several
// different config values from the same shader program.
template <typename KeyConfigType,
std::string (*CodeGenerator)(const Pica::Shader::ShaderSetup&, const KeyConfigType&,
bool),
GLenum ShaderType>
template <typename KeyConfigType, auto CodeGenerator, GLenum ShaderType>
class ShaderDoubleCache {
public:
explicit ShaderDoubleCache(bool separable) : separable(separable) {}
explicit ShaderDoubleCache() = default;
std::tuple<GLuint, std::optional<std::string>> Get(const KeyConfigType& key,
const Pica::Shader::ShaderSetup& setup) {
std::optional<std::string> result{};
auto map_it = shader_map.find(key);
if (map_it == shader_map.end()) {
auto program = CodeGenerator(setup, key, separable);
const auto program = CodeGenerator(setup, key, separable);
if (program.empty()) {
shader_map[key] = nullptr;
return {0, std::nullopt};
}
auto [iter, new_shader] = shader_cache.emplace(program, OGLShaderStage{separable});
OGLShaderStage& cached_shader = iter->second;
const auto [iter, new_shader] = shader_cache.try_emplace(program);
OGLProgram& cached_shader = iter->second;
if (new_shader) {
result = program;
cached_shader.Create(program.c_str(), ShaderType);
cached_shader.Create(program, ShaderType);
}
shader_map[key] = &cached_shader;
return {cached_shader.GetHandle(), std::move(result)};
return {cached_shader.handle, std::move(result)};
}
if (map_it->second == nullptr) {
return {0, std::nullopt};
}
return {map_it->second->GetHandle(), std::nullopt};
return {map_it->second->handle, std::nullopt};
}
void Inject(const KeyConfigType& key, std::string decomp, OGLProgram&& program) {
OGLShaderStage stage{separable};
stage.Inject(std::move(program));
const auto iter = shader_cache.emplace(std::move(decomp), std::move(stage)).first;
OGLShaderStage& cached_shader = iter->second;
shader_map.insert_or_assign(key, &cached_shader);
}
void Inject(const KeyConfigType& key, std::string decomp, OGLShaderStage&& stage) {
const auto iter = shader_cache.emplace(std::move(decomp), std::move(stage)).first;
OGLShaderStage& cached_shader = iter->second;
shader_map.insert_or_assign(key, &cached_shader);
const auto iter = shader_cache.emplace(std::move(decomp), std::move(program)).first;
shader_map.insert_or_assign(key, &iter->second);
}
private:
bool separable;
std::unordered_map<KeyConfigType, OGLShaderStage*> shader_map;
std::unordered_map<std::string, OGLShaderStage> shader_cache;
std::unordered_map<KeyConfigType, OGLProgram*> shader_map;
std::unordered_map<std::string, OGLProgram> shader_cache;
};
using ProgrammableVertexShaders =
@ -248,12 +185,8 @@ using FragmentShaders =
class ShaderProgramManager::Impl {
public:
explicit Impl(const Driver& driver, bool separable)
: separable(separable), programmable_vertex_shaders(separable),
trivial_vertex_shader(driver, separable), fixed_geometry_shaders(separable),
fragment_shaders(separable), disk_cache(separable) {
if (separable)
pipeline.Create();
explicit Impl(const Driver& driver) : trivial_vertex_shader(driver) {
pipeline.Create();
}
struct ShaderTuple {
@ -282,8 +215,6 @@ public:
static_assert(offsetof(ShaderTuple, fs_hash) == sizeof(std::size_t) * 2,
"ShaderTuple layout changed!");
bool separable;
ShaderTuple current;
ProgrammableVertexShaders programmable_vertex_shaders;
@ -297,11 +228,10 @@ public:
ShaderDiskCache disk_cache;
};
ShaderProgramManager::ShaderProgramManager(Frontend::EmuWindow& emu_window_, const Driver& driver_,
bool separable)
ShaderProgramManager::ShaderProgramManager(Frontend::EmuWindow& emu_window_, const Driver& driver_)
: emu_window{emu_window_}, driver{driver_},
strict_context_required{emu_window.StrictContextRequired()}, impl{std::make_unique<Impl>(
driver_, separable)} {}
driver_)} {}
ShaderProgramManager::~ShaderProgramManager() = default;
@ -363,30 +293,18 @@ void ShaderProgramManager::UseFragmentShader(const Pica::Regs& regs, bool use_no
}
void ShaderProgramManager::ApplyTo(OpenGLState& state) {
if (impl->separable) {
if (driver.HasBug(DriverBug::ShaderStageChangeFreeze)) {
glUseProgramStages(
impl->pipeline.handle,
GL_VERTEX_SHADER_BIT | GL_GEOMETRY_SHADER_BIT | GL_FRAGMENT_SHADER_BIT, 0);
}
glUseProgramStages(impl->pipeline.handle, GL_VERTEX_SHADER_BIT, impl->current.vs);
glUseProgramStages(impl->pipeline.handle, GL_GEOMETRY_SHADER_BIT, impl->current.gs);
glUseProgramStages(impl->pipeline.handle, GL_FRAGMENT_SHADER_BIT, impl->current.fs);
state.draw.shader_program = 0;
state.draw.program_pipeline = impl->pipeline.handle;
} else {
const u64 unique_identifier = impl->current.GetConfigHash();
OGLProgram& cached_program = impl->program_cache[unique_identifier];
if (cached_program.handle == 0) {
cached_program.Create(false,
std::array{impl->current.vs, impl->current.gs, impl->current.fs});
auto& disk_cache = impl->disk_cache;
disk_cache.SaveDumpToFile(unique_identifier, cached_program.handle,
VideoCore::g_hw_shader_accurate_mul);
}
state.draw.shader_program = cached_program.handle;
if (driver.HasBug(DriverBug::ShaderStageChangeFreeze)) {
glUseProgramStages(impl->pipeline.handle,
GL_VERTEX_SHADER_BIT | GL_GEOMETRY_SHADER_BIT | GL_FRAGMENT_SHADER_BIT,
0);
}
glUseProgramStages(impl->pipeline.handle, GL_VERTEX_SHADER_BIT, impl->current.vs);
glUseProgramStages(impl->pipeline.handle, GL_GEOMETRY_SHADER_BIT, impl->current.gs);
glUseProgramStages(impl->pipeline.handle, GL_FRAGMENT_SHADER_BIT, impl->current.fs);
state.draw.shader_program = 0;
state.draw.program_pipeline = impl->pipeline.handle;
}
void ShaderProgramManager::LoadDiskCache(const std::atomic_bool& stop_loading,
@ -400,7 +318,7 @@ void ShaderProgramManager::LoadDiskCache(const std::atomic_bool& stop_loading,
// Load uncompressed precompiled file for non-separable shaders.
// Precompiled file for separable shaders is compressed.
auto [decompiled, dumps] = disk_cache.LoadPrecompiled(impl->separable);
auto [decompiled, dumps] = disk_cache.LoadPrecompiled();
if (stop_loading) {
return;
@ -418,117 +336,81 @@ void ShaderProgramManager::LoadDiskCache(const std::atomic_bool& stop_loading,
callback(VideoCore::LoadCallbackStage::Decompile, 0, raws.size());
}
std::vector<std::size_t> load_raws_index;
// Loads both decompiled and precompiled shaders from the cache. If either one is missing for
const auto LoadPrecompiledShader = [&](std::size_t begin, std::size_t end,
std::span<const ShaderDiskCacheRaw> raw_cache,
const ShaderDecompiledMap& decompiled_map,
const ShaderDumpsMap& dump_map) {
for (std::size_t i = begin; i < end; ++i) {
if (stop_loading || compilation_failed) {
return;
}
const auto& raw{raw_cache[i]};
const u64 unique_identifier{raw.GetUniqueIdentifier()};
const u64 calculated_hash =
GetUniqueIdentifier(raw.GetRawShaderConfig(), raw.GetProgramCode());
if (unique_identifier != calculated_hash) {
LOG_ERROR(Render_OpenGL,
"Invalid hash in entry={:016x} (obtained hash={:016x}) - removing "
"shader cache",
raw.GetUniqueIdentifier(), calculated_hash);
disk_cache.InvalidateAll();
return;
}
const auto dump{dump_map.find(unique_identifier)};
const auto decomp{decompiled_map.find(unique_identifier)};
OGLProgram shader;
if (dump != dump_map.end() && decomp != decompiled_map.end()) {
// Only load the vertex shader if its sanitize_mul setting matches
if (raw.GetProgramType() == ProgramType::VS &&
decomp->second.sanitize_mul != VideoCore::g_hw_shader_accurate_mul) {
continue;
}
// If the shader is dumped, attempt to load it
shader =
GeneratePrecompiledProgram(dump->second, supported_formats, impl->separable);
if (shader.handle == 0) {
// If any shader failed, stop trying to compile, delete the cache, and start
// loading from raws
compilation_failed = true;
const auto LoadPrecompiledShader =
[&](std::size_t begin, std::size_t end, std::span<const ShaderDiskCacheRaw> raw_cache,
const ShaderDecompiledMap& decompiled_map, const ShaderDumpsMap& dump_map) {
for (std::size_t i = begin; i < end; ++i) {
if (stop_loading || compilation_failed) {
return;
}
// we have both the binary shader and the decompiled, so inject it into the
// cache
if (raw.GetProgramType() == ProgramType::VS) {
auto [conf, setup] = BuildVSConfigFromRaw(raw, driver);
std::scoped_lock lock(mutex);
impl->programmable_vertex_shaders.Inject(conf, decomp->second.code,
std::move(shader));
} else if (raw.GetProgramType() == ProgramType::FS) {
PicaFSConfig conf(raw.GetRawShaderConfig(), false, driver.IsOpenGLES(), false,
driver.HasBlendMinMaxFactor());
std::scoped_lock lock(mutex);
impl->fragment_shaders.Inject(conf, std::move(shader));
const auto& raw{raw_cache[i]};
const u64 unique_identifier{raw.GetUniqueIdentifier()};
const u64 calculated_hash =
GetUniqueIdentifier(raw.GetRawShaderConfig(), raw.GetProgramCode());
if (unique_identifier != calculated_hash) {
LOG_ERROR(Render_OpenGL,
"Invalid hash in entry={:016x} (obtained hash={:016x}) - removing "
"shader cache",
raw.GetUniqueIdentifier(), calculated_hash);
disk_cache.InvalidateAll();
return;
}
const auto dump{dump_map.find(unique_identifier)};
const auto decomp{decompiled_map.find(unique_identifier)};
OGLProgram shader;
if (dump != dump_map.end() && decomp != decompiled_map.end()) {
// Only load the vertex shader if its sanitize_mul setting matches
if (raw.GetProgramType() == ProgramType::VS &&
decomp->second.sanitize_mul != VideoCore::g_hw_shader_accurate_mul) {
continue;
}
// If the shader is dumped, attempt to load it
shader = GeneratePrecompiledProgram(dump->second, supported_formats);
if (shader.handle == 0) {
// If any shader failed, stop trying to compile, delete the cache, and start
// loading from raws.
compilation_failed = true;
return;
}
// We have both the binary shader and the decompiled, so inject it into the
// cache.
if (raw.GetProgramType() == ProgramType::VS) {
auto [conf, setup] = BuildVSConfigFromRaw(raw, driver);
std::scoped_lock lock(mutex);
impl->programmable_vertex_shaders.Inject(conf, decomp->second.code,
std::move(shader));
} else if (raw.GetProgramType() == ProgramType::FS) {
PicaFSConfig conf(raw.GetRawShaderConfig(), false, driver.IsOpenGLES(),
false, driver.HasBlendMinMaxFactor());
std::scoped_lock lock(mutex);
impl->fragment_shaders.Inject(conf, std::move(shader));
} else {
// Unsupported shader type got stored somehow so nuke the cache
LOG_CRITICAL(Frontend, "failed to load raw ProgramType {}",
raw.GetProgramType());
compilation_failed = true;
return;
}
} else {
// Unsupported shader type got stored somehow so nuke the cache
LOG_CRITICAL(Frontend, "failed to load raw ProgramType {}",
raw.GetProgramType());
compilation_failed = true;
return;
// Since precompiled didn't have the dump, we'll load them in the next phase
std::scoped_lock lock(mutex);
load_raws_index.push_back(i);
}
if (callback) {
callback(VideoCore::LoadCallbackStage::Decompile, i, raw_cache.size());
}
} else {
// Since precompiled didn't have the dump, we'll load them in the next phase
std::scoped_lock lock(mutex);
load_raws_index.push_back(i);
}
if (callback) {
callback(VideoCore::LoadCallbackStage::Decompile, i, raw_cache.size());
}
}
};
};
const auto LoadPrecompiledProgram = [&](const ShaderDecompiledMap& decompiled_map,
const ShaderDumpsMap& dump_map) {
std::size_t i{0};
for (const auto& dump : dump_map) {
if (stop_loading) {
break;
}
const u64 unique_identifier{dump.first};
const auto decomp{decompiled_map.find(unique_identifier)};
// Only load the program if its sanitize_mul setting matches
if (decomp->second.sanitize_mul != VideoCore::g_hw_shader_accurate_mul) {
continue;
}
// If the shader program is dumped, attempt to load it
OGLProgram shader =
GeneratePrecompiledProgram(dump.second, supported_formats, impl->separable);
if (shader.handle != 0) {
impl->program_cache.emplace(unique_identifier, std::move(shader));
} else {
LOG_ERROR(Frontend, "Failed to link Precompiled program!");
compilation_failed = true;
break;
}
if (callback) {
callback(VideoCore::LoadCallbackStage::Decompile, ++i, dump_map.size());
}
}
};
if (impl->separable) {
LoadPrecompiledShader(0, raws.size(), raws, decompiled, dumps);
} else {
LoadPrecompiledProgram(decompiled, dumps);
}
LoadPrecompiledShader(0, raws.size(), raws, decompiled, dumps);
bool load_all_raws = false;
if (compilation_failed) {
@ -539,11 +421,6 @@ void ShaderProgramManager::LoadDiskCache(const std::atomic_bool& stop_loading,
precompiled_cache_altered = true;
load_all_raws = true;
}
// TODO(SachinV): Skip loading raws until we implement a proper way to link non-seperable
// shaders.
if (!impl->separable) {
return;
}
const std::size_t load_raws_size = load_all_raws ? raws.size() : load_raws_index.size();
@ -570,25 +447,25 @@ void ShaderProgramManager::LoadDiskCache(const std::atomic_bool& stop_loading,
GLuint handle{0};
std::string code;
// Otherwise decompile and build the shader at boot and save the result to the
// precompiled file
// precompiled file.
if (raw.GetProgramType() == ProgramType::VS) {
auto [conf, setup] = BuildVSConfigFromRaw(raw, driver);
code = GLSL::GenerateVertexShader(setup, conf, impl->separable);
OGLShaderStage stage{impl->separable};
stage.Create(code.c_str(), GL_VERTEX_SHADER);
handle = stage.GetHandle();
const auto [conf, setup] = BuildVSConfigFromRaw(raw, driver);
code = GLSL::GenerateVertexShader(setup, conf, true);
OGLProgram program{};
program.Create(code, GL_VERTEX_SHADER);
handle = program.handle;
sanitize_mul = conf.state.sanitize_mul;
std::scoped_lock lock(mutex);
impl->programmable_vertex_shaders.Inject(conf, code, std::move(stage));
impl->programmable_vertex_shaders.Inject(conf, code, std::move(program));
} else if (raw.GetProgramType() == ProgramType::FS) {
PicaFSConfig conf(raw.GetRawShaderConfig(), false, driver.IsOpenGLES(), false,
driver.HasBlendMinMaxFactor());
code = GLSL::GenerateFragmentShader(conf, impl->separable);
OGLShaderStage stage{impl->separable};
stage.Create(code.c_str(), GL_FRAGMENT_SHADER);
handle = stage.GetHandle();
code = GLSL::GenerateFragmentShader(conf, true);
OGLProgram program{};
program.Create(code, GL_FRAGMENT_SHADER);
handle = program.handle;
std::scoped_lock lock(mutex);
impl->fragment_shaders.Inject(conf, std::move(stage));
impl->fragment_shaders.Inject(conf, std::move(program));
} else {
// Unsupported shader type got stored somehow so nuke the cache
LOG_ERROR(Frontend, "failed to load raw ProgramType {}", raw.GetProgramType());

View File

@ -33,22 +33,29 @@ enum UniformBindings {
/// A class that manage different shader stages and configures them with given config data.
class ShaderProgramManager {
public:
ShaderProgramManager(Frontend::EmuWindow& emu_window, const Driver& driver, bool separable);
explicit ShaderProgramManager(Frontend::EmuWindow& emu_window, const Driver& driver);
~ShaderProgramManager();
/// Loads the pipeline cache stored to disk.
void LoadDiskCache(const std::atomic_bool& stop_loading,
const VideoCore::DiskResourceLoadCallback& callback);
/// Binds a PICA decompiled vertex shader.
bool UseProgrammableVertexShader(const Pica::Regs& config, Pica::Shader::ShaderSetup& setup);
/// Binds a passthrough vertex shader.
void UseTrivialVertexShader();
/// Binds a passthrough geometry shader.
void UseFixedGeometryShader(const Pica::Regs& regs);
/// Binds no geometry shader.
void UseTrivialGeometryShader();
/// Binds a fragment shader generated from PICA state.
void UseFragmentShader(const Pica::Regs& config, bool use_normal);
/// Binds current shader state to provided OpenGLState.
void ApplyTo(OpenGLState& state);
private:

View File

@ -13,10 +13,9 @@
namespace OpenGL {
GLuint LoadShader(std::string_view source, GLenum type) {
std::string preamble;
std::string_view GetPreamble() {
if (GLES) {
preamble = R"(#version 320 es
return R"(#version 320 es
#if defined(GL_ANDROID_extension_pack_es31a)
#extension GL_ANDROID_extension_pack_es31a : enable
@ -26,9 +25,12 @@ GLuint LoadShader(std::string_view source, GLenum type) {
#extension GL_EXT_clip_cull_distance : enable
#endif // defined(GL_EXT_clip_cull_distance)
)";
} else {
preamble = "#version 430 core\n";
}
return "#version 430 core\n";
}
GLuint LoadShader(std::string_view source, GLenum type) {
const auto preamble = GetPreamble();
std::string_view debug_type;
switch (type) {
@ -72,11 +74,9 @@ GLuint LoadShader(std::string_view source, GLenum type) {
return shader_id;
}
GLuint LoadProgram(bool separable_program, std::span<const GLuint> shaders) {
// Link the program
GLuint LoadProgram(std::span<const GLuint> shaders) {
LOG_DEBUG(Render_OpenGL, "Linking program...");
GLuint program_id = glCreateProgram();
const GLuint program_id = glCreateProgram();
for (GLuint shader : shaders) {
if (shader != 0) {
@ -84,10 +84,6 @@ GLuint LoadProgram(bool separable_program, std::span<const GLuint> shaders) {
}
}
if (separable_program) {
glProgramParameteri(program_id, GL_PROGRAM_SEPARABLE, GL_TRUE);
}
glProgramParameteri(program_id, GL_PROGRAM_BINARY_RETRIEVABLE_HINT, GL_TRUE);
glLinkProgram(program_id);

View File

@ -9,6 +9,11 @@
namespace OpenGL {
/**
* Utility function to retrieve the preamble to compile an OpenGL GLSL shader
*/
std::string_view GetPreamble();
/**
* Utility function to create and compile an OpenGL GLSL shader
* @param source String of the GLSL shader program
@ -22,6 +27,6 @@ GLuint LoadShader(std::string_view source, GLenum type);
* @param shaders ID of shaders to attach to the program
* @returns Handle of the newly created OpenGL program object
*/
GLuint LoadProgram(bool separable_program, std::span<const GLuint> shaders);
GLuint LoadProgram(std::span<const GLuint> shaders);
} // namespace OpenGL

View File

@ -59,6 +59,7 @@ OpenGLState::OpenGLState() {
texture_buffer_lut_lf.texture_buffer = 0;
texture_buffer_lut_rg.texture_buffer = 0;
texture_buffer_lut_rgba.texture_buffer = 0;
color_buffer.texture_2d = 0;
image_shadow_buffer = 0;
image_shadow_texture_px = 0;

View File

@ -1562,7 +1562,7 @@ DebugScope::DebugScope(TextureRuntime& runtime, Common::Vec4f color, std::string
if (!has_debug_tool) {
return;
}
scheduler.Record([color, label](vk::CommandBuffer cmdbuf) {
scheduler.Record([color, label = std::string(label)](vk::CommandBuffer cmdbuf) {
const vk::DebugUtilsLabelEXT debug_label = {
.pLabelName = label.data(),
.color = std::array{color[0], color[1], color[2], color[3]},