octo
/
citra
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

core/file_sys: LayeredFS implementation 

This implementation is different from Luma3DS's which directly hooks the SDK functions. Instead, we read the RomFS's metadata and figure out the directory and file structure. Then, relocations (i.e. replacements/deletions/patches) are applied. Afterwards, we rebuild the metadata, and assign 'fake' data offsets to the files. When we want to read file data from this rebuilt RomFS, we use binary search to find the last data offset smaller or equal to the given offset and read from that file  (either from the original RomFS, or from replacement files, or from buffered data with patches applied) and any later files when length is not enough.

The code that rebuilds the metadata is pretty complex and uses quite a few variables to keep track of necessary information like metadata offsets. According to my tests, it is able to build RomFS-es identical to the original (but without trailing garbage data) when no relocations are applied.
This commit is contained in:
zhupengfei 2020-02-07 12:54:07 +08:00
parent 83e0cc45f4
commit 890405bb7c
No known key found for this signature in database
GPG Key ID: DD129E108BD09378
3 changed files with 670 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/core/CMakeLists.txt
View File

@ -72,6 +72,8 @@ add_library(core STATIC
file_sys/delay_generator.h file_sys/delay_generator.h
file_sys/ivfc_archive.cpp file_sys/ivfc_archive.cpp
file_sys/ivfc_archive.h file_sys/ivfc_archive.h
file_sys/layered_fs.cpp
file_sys/layered_fs.h
file_sys/ncch_container.cpp file_sys/ncch_container.cpp
file_sys/ncch_container.h file_sys/ncch_container.h
file_sys/patch.cpp file_sys/patch.cpp

551
src/core/file_sys/layered_fs.cpp Normal file
View File

@ -0,0 +1,551 @@
// Copyright 2020 Citra Emulator Project
// Licensed under GPLv2 or any later version
// Refer to the license.txt file included.
#include <algorithm>
#include <cstring>
#include "common/alignment.h"
#include "common/assert.h"
#include "common/common_paths.h"
#include "common/file_util.h"
#include "common/string_util.h"
#include "common/swap.h"
#include "core/file_sys/layered_fs.h"
#include "core/file_sys/patch.h"
namespace FileSys {
struct FileRelocationInfo {
int type; // 0 - none, 1 - replaced / created, 2 - patched, 3 - removed
u64 original_offset; // Type 0. Offset is absolute
FileUtil::IOFile replace_file; // Type 1
std::vector<u8> patched_file; // Type 2
u64 size; // Relocated file size
};
struct LayeredFS::File {
std::string name;
std::string path;
FileRelocationInfo relocation{};
Directory* parent;
};
struct DirectoryMetadata {
u32_le parent_directory_offset;
u32_le next_sibling_offset;
u32_le first_child_directory_offset;
u32_le first_file_offset;
u32_le hash_bucket_next;
u32_le name_length;
// Followed by a name of name length (aligned up to 4)
};
static_assert(sizeof(DirectoryMetadata) == 0x18, "Size of DirectoryMetadata is not correct");
struct FileMetadata {
u32_le parent_directory_offset;
u32_le next_sibling_offset;
u64_le file_data_offset;
u64_le file_data_length;
u32_le hash_bucket_next;
u32_le name_length;
// Followed by a name of name length (aligned up to 4)
};
static_assert(sizeof(FileMetadata) == 0x20, "Size of FileMetadata is not correct");
LayeredFS::LayeredFS(std::shared_ptr<RomFSReader> romfs_, std::string patch_path_,
std::string patch_ext_path_)
: romfs(std::move(romfs_)), patch_path(std::move(patch_path_)),
patch_ext_path(std::move(patch_ext_path_)) {
romfs->ReadFile(0, sizeof(header), reinterpret_cast<u8*>(&header));
ASSERT_MSG(header.header_length == sizeof(header), "Header size is incorrect");
// TODO: is root always the first directory in table?
root.parent = &root;
LoadDirectory(root, 0);
LoadRelocations();
LoadExtRelocations();
RebuildMetadata();
}
LayeredFS::~LayeredFS() = default;
void LayeredFS::LoadDirectory(Directory& current, u32 offset) {
DirectoryMetadata metadata;
romfs->ReadFile(header.directory_metadata_table.offset + offset, sizeof(metadata),
reinterpret_cast<u8*>(&metadata));
current.name = ReadName(header.directory_metadata_table.offset + offset + sizeof(metadata),
metadata.name_length);
current.path = current.parent->path + current.name + DIR_SEP;
directory_path_map.emplace(current.path, &current);
if (metadata.first_file_offset != 0xFFFFFFFF) {
LoadFile(current, metadata.first_file_offset);
}
if (metadata.first_child_directory_offset != 0xFFFFFFFF) {
auto child = std::make_unique<Directory>();
auto& directory = *child;
directory.parent = &current;
current.directories.emplace_back(std::move(child));
LoadDirectory(directory, metadata.first_child_directory_offset);
}
if (metadata.next_sibling_offset != 0xFFFFFFFF) {
auto sibling = std::make_unique<Directory>();
auto& directory = *sibling;
directory.parent = current.parent;
current.parent->directories.emplace_back(std::move(sibling));
LoadDirectory(directory, metadata.next_sibling_offset);
}
}
void LayeredFS::LoadFile(Directory& parent, u32 offset) {
FileMetadata metadata;
romfs->ReadFile(header.file_metadata_table.offset + offset, sizeof(metadata),
reinterpret_cast<u8*>(&metadata));
auto file = std::make_unique<File>();
file->name = ReadName(header.file_metadata_table.offset + offset + sizeof(metadata),
metadata.name_length);
file->path = parent.path + file->name;
file->relocation.original_offset = header.file_data_offset + metadata.file_data_offset;
file->relocation.size = metadata.file_data_length;
file->parent = &parent;
file_path_map.emplace(file->path, file.get());
parent.files.emplace_back(std::move(file));
if (metadata.next_sibling_offset != 0xFFFFFFFF) {
LoadFile(parent, metadata.next_sibling_offset);
}
}
std::string LayeredFS::ReadName(u32 offset, u32 name_length) {
std::vector<u16_le> buffer(name_length / sizeof(u16_le));
romfs->ReadFile(offset, name_length, reinterpret_cast<u8*>(buffer.data()));
std::u16string name(buffer.size(), 0);
std::transform(buffer.begin(), buffer.end(), name.begin(), [](u16_le character) {
return static_cast<char16_t>(static_cast<u16>(character));
});
return Common::UTF16ToUTF8(name);
}
void LayeredFS::LoadRelocations() {
if (!FileUtil::Exists(patch_path)) {
return;
}
const FileUtil::DirectoryEntryCallable callback = [this,
&callback](u64* /*num_entries_out*/,
const std::string& directory,
const std::string& virtual_name) {
auto* parent = directory_path_map.at(directory.substr(patch_path.size() - 1));
if (FileUtil::IsDirectory(directory + virtual_name + DIR_SEP)) {
const auto path = (directory + virtual_name + DIR_SEP).substr(patch_path.size() - 1);
if (!directory_path_map.count(path)) { // Add this directory
auto directory = std::make_unique<Directory>();
directory->name = virtual_name;
directory->path = path;
directory->parent = parent;
directory_path_map.emplace(path, directory.get());
parent->directories.emplace_back(std::move(directory));
LOG_INFO(Service_FS, "LayeredFS created directory {}", path);
}
return FileUtil::ForeachDirectoryEntry(nullptr, directory + virtual_name + DIR_SEP,
callback);
}
const auto path = (directory + virtual_name).substr(patch_path.size() - 1);
if (!file_path_map.count(path)) { // Newly created file
auto file = std::make_unique<File>();
file->name = virtual_name;
file->path = path;
file->parent = parent;
file_path_map.emplace(path, file.get());
parent->files.emplace_back(std::move(file));
LOG_INFO(Service_FS, "LayeredFS created file {}", path);
}
auto* file = file_path_map.at(path);
file->relocation.replace_file = FileUtil::IOFile(directory + virtual_name, "rb");
if (file->relocation.replace_file) {
file->relocation.type = 1;
file->relocation.size = file->relocation.replace_file.GetSize();
LOG_INFO(Service_FS, "LayeredFS replacement file in use for {}", path);
} else {
LOG_ERROR(Service_FS, "Could not open replacement file for {}", path);
}
return true;
};
FileUtil::ForeachDirectoryEntry(nullptr, patch_path, callback);
}
void LayeredFS::LoadExtRelocations() {
if (!FileUtil::Exists(patch_ext_path)) {
return;
}
if (patch_ext_path.back() == '/' || patch_ext_path.back() == '\\') {
// ScanDirectoryTree expects a path without trailing '/'
patch_ext_path.erase(patch_ext_path.size() - 1, 1);
}
FileUtil::FSTEntry result;
FileUtil::ScanDirectoryTree(patch_ext_path, result, 256);
for (const auto& entry : result.children) {
if (FileUtil::IsDirectory(entry.physicalName)) {
continue;
}
const auto path = entry.physicalName.substr(patch_ext_path.size());
if (path.size() >= 5 && path.substr(path.size() - 5) == ".stub") {
// Remove the corresponding file if exists
const auto file_path = path.substr(0, path.size() - 5);
if (file_path_map.count(file_path)) {
auto& file = *file_path_map[file_path];
file.relocation.type = 3;
file.relocation.size = 0;
file_path_map.erase(file_path);
LOG_INFO(Service_FS, "LayeredFS removed file {}", file_path);
} else {
LOG_WARNING(Service_FS, "LayeredFS file for stub {} not found", path);
}
} else if (path.size() >= 4) {
const auto extension = path.substr(path.size() - 4);
if (extension != ".ips" && extension != ".bps") {
LOG_WARNING(Service_FS, "LayeredFS unknown ext file {}", path);
}
const auto file_path = path.substr(0, path.size() - 4);
if (!file_path_map.count(file_path)) {
LOG_WARNING(Service_FS, "LayeredFS original file for patch {} not found", path);
continue;
}
FileUtil::IOFile patch_file(entry.physicalName, "rb");
if (!patch_file) {
LOG_ERROR(Service_FS, "LayeredFS Could not open file {}", entry.physicalName);
continue;
}
const auto size = patch_file.GetSize();
std::vector<u8> patch(size);
if (patch_file.ReadBytes(patch.data(), size) != size) {
LOG_ERROR(Service_FS, "LayeredFS Could not read file {}", entry.physicalName);
continue;
}
auto& file = *file_path_map[file_path];
std::vector<u8> buffer(file.relocation.size); // Original size
romfs->ReadFile(file.relocation.original_offset, buffer.size(), buffer.data());
bool ret = false;
if (extension == ".ips") {
ret = Patch::ApplyIpsPatch(patch, buffer);
} else {
ret = Patch::ApplyBpsPatch(patch, buffer);
}
if (ret) {
LOG_INFO(Service_FS, "LayeredFS patched file {}", file_path);
file.relocation.type = 2;
file.relocation.size = buffer.size();
file.relocation.patched_file = std::move(buffer);
} else {
LOG_ERROR(Service_FS, "LayeredFS failed to patch file {}", file_path);
}
} else {
LOG_WARNING(Service_FS, "LayeredFS unknown ext file {}", path);
}
}
}
std::size_t GetNameSize(const std::string& name) {
std::u16string u16name = Common::UTF8ToUTF16(name);
return Common::AlignUp(u16name.size() * 2, 4);
}
void LayeredFS::PrepareBuildDirectory(Directory& current) {
directory_metadata_offset_map.emplace(&current, current_directory_offset);
directory_list.emplace_back(&current);
current_directory_offset += sizeof(DirectoryMetadata) + GetNameSize(current.name);
}
void LayeredFS::PrepareBuildFile(File& current) {
if (current.relocation.type == 3) { // Deleted files are not counted
return;
}
file_metadata_offset_map.emplace(&current, current_file_offset);
file_list.emplace_back(&current);
current_file_offset += sizeof(FileMetadata) + GetNameSize(current.name);
}
void LayeredFS::PrepareBuild(Directory& current) {
for (const auto& child : current.files) {
PrepareBuildFile(*child);
}
for (const auto& child : current.directories) {
PrepareBuildDirectory(*child);
}
for (const auto& child : current.directories) {
PrepareBuild(*child);
}
}
// Implementation from 3dbrew
u32 CalcHash(const std::string& name, u32 parent_offset) {
u32 hash = parent_offset ^ 123456789;
std::u16string u16name = Common::UTF8ToUTF16(name);
std::vector<u16_le> tmp_buffer(u16name.size());
std::transform(u16name.begin(), u16name.end(), tmp_buffer.begin(), [](char16_t character) {
return static_cast<u16_le>(static_cast<u16>(character));
});
std::vector<u8> buffer(tmp_buffer.size() * 2);
std::memcpy(buffer.data(), tmp_buffer.data(), buffer.size());
for (std::size_t i = 0; i < buffer.size(); i += 2) {
hash = (hash >> 5) | (hash << 27);
hash ^= static_cast<u16>((buffer[i]) | (buffer[i + 1] << 8));
}
return hash;
}
std::size_t WriteName(u8* dest, std::u16string name) {
const auto buffer_size = Common::AlignUp(name.size() * 2, 4);
std::vector<u16_le> buffer(buffer_size / 2);
std::transform(name.begin(), name.end(), buffer.begin(), [](char16_t character) {
return static_cast<u16_le>(static_cast<u16>(character));
});
std::memcpy(dest, buffer.data(), buffer_size);
return buffer_size;
}
void LayeredFS::BuildDirectories() {
directory_metadata_table.resize(current_directory_offset, 0xFF);
std::size_t written = 0;
for (const auto& directory : directory_list) {
DirectoryMetadata metadata;
std::memset(&metadata, 0xFF, sizeof(metadata));
metadata.parent_directory_offset = directory_metadata_offset_map.at(directory->parent);
if (directory->parent != directory) {
bool flag = false;
for (const auto& sibling : directory->parent->directories) {
if (flag) {
metadata.next_sibling_offset = directory_metadata_offset_map.at(sibling.get());
break;
} else if (sibling.get() == directory) {
flag = true;
}
}
}
if (!directory->directories.empty()) {
metadata.first_child_directory_offset =
directory_metadata_offset_map.at(directory->directories.front().get());
}
if (!directory->files.empty()) {
metadata.first_file_offset =
file_metadata_offset_map.at(directory->files.front().get());
}
const auto bucket = CalcHash(directory->name, metadata.parent_directory_offset) %
directory_hash_table.size();
metadata.hash_bucket_next = directory_hash_table[bucket];
directory_hash_table[bucket] = directory_metadata_offset_map.at(directory);
// Write metadata and name
std::u16string u16name = Common::UTF8ToUTF16(directory->name);
metadata.name_length = u16name.size() * 2;
std::memcpy(directory_metadata_table.data() + written, &metadata, sizeof(metadata));
written += sizeof(metadata);
written += WriteName(directory_metadata_table.data() + written, u16name);
}
ASSERT_MSG(written == directory_metadata_table.size(),
"Calculated size for directory metadata table is wrong");
}
void LayeredFS::BuildFiles() {
file_metadata_table.resize(current_file_offset, 0xFF);
std::size_t written = 0;
for (const auto& file : file_list) {
FileMetadata metadata;
std::memset(&metadata, 0xFF, sizeof(metadata));
metadata.parent_directory_offset = directory_metadata_offset_map.at(file->parent);
bool flag = false;
for (const auto& sibling : file->parent->files) {
if (sibling->relocation.type == 3) { // removed file
continue;
}
if (flag) {
metadata.next_sibling_offset = file_metadata_offset_map.at(sibling.get());
break;
} else if (sibling.get() == file) {
flag = true;
}
}
metadata.file_data_offset = current_data_offset;
metadata.file_data_length = file->relocation.size;
current_data_offset += Common::AlignUp(metadata.file_data_length, 16);
if (metadata.file_data_length != 0) {
data_offset_map.emplace(metadata.file_data_offset, file);
}
const auto bucket =
CalcHash(file->name, metadata.parent_directory_offset) % file_hash_table.size();
metadata.hash_bucket_next = file_hash_table[bucket];
file_hash_table[bucket] = file_metadata_offset_map.at(file);
// Write metadata and name
std::u16string u16name = Common::UTF8ToUTF16(file->name);
metadata.name_length = u16name.size() * 2;
std::memcpy(file_metadata_table.data() + written, &metadata, sizeof(metadata));
written += sizeof(metadata);
written += WriteName(file_metadata_table.data() + written, u16name);
}
ASSERT_MSG(written == file_metadata_table.size(),
"Calculated size for file metadata table is wrong");
}
// Implementation from 3dbrew
std::size_t GetHashTableSize(std::size_t entry_count) {
if (entry_count < 3) {
return 3;
} else if (entry_count < 19) {
return entry_count | 1;
} else {
std::size_t count = entry_count;
while (count % 2 == 0 || count % 3 == 0 || count % 5 == 0 || count % 7 == 0 ||
count % 11 == 0 || count % 13 == 0 || count % 17 == 0) {
count++;
}
return count;
}
}
void LayeredFS::RebuildMetadata() {
PrepareBuildDirectory(root);
PrepareBuild(root);
directory_hash_table.resize(GetHashTableSize(directory_list.size()), 0xFFFFFFFF);
file_hash_table.resize(GetHashTableSize(file_list.size()), 0xFFFFFFFF);
BuildDirectories();
BuildFiles();
// Create header
RomFSHeader header;
header.header_length = sizeof(header);
header.directory_hash_table = {
/*offset*/ sizeof(header),
/*length*/ static_cast<u32_le>(directory_hash_table.size() * sizeof(u32_le))};
header.directory_metadata_table = {
/*offset*/
header.directory_hash_table.offset + header.directory_hash_table.length,
/*length*/ static_cast<u32_le>(directory_metadata_table.size())};
header.file_hash_table = {
/*offset*/
header.directory_metadata_table.offset + header.directory_metadata_table.length,
/*length*/ static_cast<u32_le>(file_hash_table.size() * sizeof(u32_le))};
header.file_metadata_table = {/*offset*/ header.file_hash_table.offset +
header.file_hash_table.length,
/*length*/ static_cast<u32_le>(file_metadata_table.size())};
header.file_data_offset =
Common::AlignUp(header.file_metadata_table.offset + header.file_metadata_table.length, 16);
// Write hash table and metadata table
metadata.resize(header.file_data_offset);
std::memcpy(metadata.data(), &header, header.header_length);
std::memcpy(metadata.data() + header.directory_hash_table.offset, directory_hash_table.data(),
header.directory_hash_table.length);
std::memcpy(metadata.data() + header.directory_metadata_table.offset,
directory_metadata_table.data(), header.directory_metadata_table.length);
std::memcpy(metadata.data() + header.file_hash_table.offset, file_hash_table.data(),
header.file_hash_table.length);
std::memcpy(metadata.data() + header.file_metadata_table.offset, file_metadata_table.data(),
header.file_metadata_table.length);
}
std::size_t LayeredFS::GetSize() const {
return metadata.size() + current_data_offset;
}
std::size_t LayeredFS::ReadFile(std::size_t offset, std::size_t length, u8* buffer) {
ASSERT_MSG(offset + length <= GetSize(), "Out of bound");
std::size_t read_size = 0;
if (offset < metadata.size()) {
// First read the metadata
const auto to_read = std::min(metadata.size() - offset, length);
std::memcpy(buffer, metadata.data() + offset, to_read);
read_size += to_read;
offset = 0;
} else {
offset -= metadata.size();
}
// Read files
auto current = (--data_offset_map.upper_bound(offset));
while (read_size < length) {
const auto relative_offset = offset - current->first;
std::size_t to_read{};
if (current->second->relocation.size > relative_offset) {
to_read =
std::min(current->second->relocation.size - relative_offset, length - read_size);
}
const auto alignment =
std::min(Common::AlignUp(current->second->relocation.size, 16) - relative_offset,
length - read_size) -
to_read;
// Read the file in different ways depending on relocation type
auto& relocation = current->second->relocation;
if (relocation.type == 0) { // none
romfs->ReadFile(relocation.original_offset + relative_offset, to_read,
buffer + read_size);
} else if (relocation.type == 1) { // replace
relocation.replace_file.Seek(relative_offset, SEEK_SET);
relocation.replace_file.ReadBytes(buffer + read_size, to_read);
} else if (relocation.type == 2) { // patch
std::memcpy(buffer + read_size, relocation.patched_file.data() + relative_offset,
to_read);
} else {
UNREACHABLE();
}
std::memset(buffer + read_size + to_read, 0, alignment);
read_size += to_read + alignment;
offset += to_read + alignment;
current++;
}
return read_size;
}
} // namespace FileSys

117
src/core/file_sys/layered_fs.h Normal file
View File

@ -0,0 +1,117 @@
// Copyright 2020 Citra Emulator Project
// Licensed under GPLv2 or any later version
// Refer to the license.txt file included.
#pragma once
#include <map>
#include <memory>
#include <string>
#include <unordered_map>
#include <vector>
#include "common/common_types.h"
#include "common/swap.h"
#include "core/file_sys/romfs_reader.h"
namespace FileSys {
struct RomFSHeader {
struct Descriptor {
u32_le offset;
u32_le length;
};
u32_le header_length;
Descriptor directory_hash_table;
Descriptor directory_metadata_table;
Descriptor file_hash_table;
Descriptor file_metadata_table;
u32_le file_data_offset;
};
static_assert(sizeof(RomFSHeader) == 0x28, "Size of RomFSHeader is not correct");
/**
* LayeredFS implementation. This basically adds a layer to another RomFSReader.
*
* patch_path: Path for RomFS replacements. Files present in this path replace or create
* corresponding files in RomFS.
* patch_ext_path: Path for RomFS extensions. Files present in this path:
* - When with an extension of ".stub", remove the corresponding file in the RomFS.
* - When with an extension of ".ips" or ".bps", patch the file in the RomFS.
*/
class LayeredFS : public RomFSReader {
public:
explicit LayeredFS(std::shared_ptr<RomFSReader> romfs, std::string patch_path,
std::string patch_ext_path);
~LayeredFS();
std::size_t GetSize() const override;
std::size_t ReadFile(std::size_t offset, std::size_t length, u8* buffer) override;
private:
struct File;
struct Directory {
std::string name;
std::string path; // with trailing '/'
std::vector<std::unique_ptr<File>> files;
std::vector<std::unique_ptr<Directory>> directories;
Directory* parent;
};
std::string ReadName(u32 offset, u32 name_length);
// Loads the current directory, then its siblings, and then its children.
void LoadDirectory(Directory& current, u32 offset);
// Load the file at offset, and then its siblings.
void LoadFile(Directory& parent, u32 offset);
// Load replace/create relocations
void LoadRelocations();
// Load patch/remove relocations
void LoadExtRelocations();
// Calculate the offset of a single directory add it to the map and list of directories
void PrepareBuildDirectory(Directory& current);
// Calculate the offset of a single file add it to the map and list of files
void PrepareBuildFile(File& current);
// Recursively generate a sequence of files and directories and their offsets for all
// children of current. (The current directory itself is not handled.)
void PrepareBuild(Directory& current);
void BuildDirectories();
void BuildFiles();
void RebuildMetadata();
std::shared_ptr<RomFSReader> romfs;
std::string patch_path;
std::string patch_ext_path;
RomFSHeader header;
Directory root;
std::unordered_map<std::string, File*> file_path_map;
std::unordered_map<std::string, Directory*> directory_path_map;
std::map<u64, File*> data_offset_map; // assigned data offset -> file
std::vector<u8> metadata; // Includes header, hash table and metadata
// Used for rebuilding header
std::vector<u32_le> directory_hash_table;
std::vector<u32_le> file_hash_table;
std::unordered_map<Directory*, u32>
directory_metadata_offset_map; // directory -> metadata offset
std::vector<Directory*> directory_list; // sequence of directories to be written to metadata
u64 current_directory_offset{}; // current directory metadata offset
std::vector<u8> directory_metadata_table; // rebuilt directory metadata table
std::unordered_map<File*, u32> file_metadata_offset_map; // file -> metadata offset
std::vector<File*> file_list; // sequence of files to be written to metadata
u64 current_file_offset{}; // current file metadata offset
std::vector<u8> file_metadata_table; // rebuilt file metadata table
u64 current_data_offset{}; // current assigned data offset
};
} // namespace FileSys