From f62227aa95f96901b75670d3914971593f65f119 Mon Sep 17 00:00:00 2001 From: Zach Hilman Date: Sun, 30 Sep 2018 18:13:02 -0400 Subject: [PATCH 1/6] hex_util: Add HexVectorToString and HexStringToVector Converts between bytes and strings when the size is not known at compile time. --- src/common/hex_util.cpp | 19 +++++++++++++++++++ src/common/hex_util.h | 5 +++++ 2 files changed, 24 insertions(+) diff --git a/src/common/hex_util.cpp b/src/common/hex_util.cpp index 589ae5cbf..e516e5bbd 100644 --- a/src/common/hex_util.cpp +++ b/src/common/hex_util.cpp @@ -18,6 +18,25 @@ u8 ToHexNibble(char c1) { return 0; } +std::vector HexStringToVector(std::string_view str, bool little_endian) { + std::vector out(str.size() / 2); + if (little_endian) { + for (std::size_t i = str.size() - 2; i <= str.size(); i -= 2) + out[i / 2] = (ToHexNibble(str[i]) << 4) | ToHexNibble(str[i + 1]); + } else { + for (std::size_t i = 0; i < str.size(); i += 2) + out[i / 2] = (ToHexNibble(str[i]) << 4) | ToHexNibble(str[i + 1]); + } + return out; +} + +std::string HexVectorToString(std::vector vector, bool upper) { + std::string out; + for (u8 c : vector) + out += fmt::format(upper ? "{:02X}" : "{:02x}", c); + return out; +} + std::array operator""_array16(const char* str, std::size_t len) { if (len != 32) { LOG_ERROR(Common, diff --git a/src/common/hex_util.h b/src/common/hex_util.h index 863a5ccd9..0b7d3592a 100644 --- a/src/common/hex_util.h +++ b/src/common/hex_util.h @@ -7,6 +7,7 @@ #include #include #include +#include #include #include "common/common_types.h" @@ -14,6 +15,8 @@ namespace Common { u8 ToHexNibble(char c1); +std::vector HexStringToVector(std::string_view str, bool little_endian); + template std::array HexStringToArray(std::string_view str) { std::array out{}; @@ -27,6 +30,8 @@ std::array HexStringToArray(std::string_view str) { return out; } +std::string HexVectorToString(std::vector vector, bool upper = true); + template std::string HexArrayToString(std::array array, bool upper = true) { std::string out; From 306739c2c479b646270f7cd8000bb11483613d50 Mon Sep 17 00:00:00 2001 From: Zach Hilman Date: Sun, 30 Sep 2018 18:13:22 -0400 Subject: [PATCH 2/6] ips_layer: Add IPSwitchCompiler to process IPSwitch format --- src/core/file_sys/ips_layer.cpp | 142 ++++++++++++++++++++++++++++++++ src/core/file_sys/ips_layer.h | 26 ++++++ 2 files changed, 168 insertions(+) diff --git a/src/core/file_sys/ips_layer.cpp b/src/core/file_sys/ips_layer.cpp index df933ee36..6bce138a8 100644 --- a/src/core/file_sys/ips_layer.cpp +++ b/src/core/file_sys/ips_layer.cpp @@ -2,7 +2,9 @@ // Licensed under GPLv2 or any later version // Refer to the license.txt file included. +#include #include "common/assert.h" +#include "common/hex_util.h" #include "common/swap.h" #include "core/file_sys/ips_layer.h" #include "core/file_sys/vfs_vector.h" @@ -85,4 +87,144 @@ VirtualFile PatchIPS(const VirtualFile& in, const VirtualFile& ips) { return std::make_shared(in_data, in->GetName(), in->GetContainingDirectory()); } +IPSwitchCompiler::IPSwitchCompiler(VirtualFile patch_text_) + : valid(false), patch_text(std::move(patch_text_)), nso_build_id{}, is_little_endian(false), + offset_shift(0), print_values(false) { + Parse(); +} + +std::array IPSwitchCompiler::GetBuildID() const { + return nso_build_id; +} + +bool IPSwitchCompiler::IsValid() const { + return valid; +} + +static bool StartsWith(const std::string& base, const std::string& check) { + return base.size() >= check.size() && base.substr(0, check.size()) == check; +} + +void IPSwitchCompiler::Parse() { + const auto bytes = patch_text->ReadAllBytes(); + std::stringstream s; + s.write(reinterpret_cast(bytes.data()), bytes.size()); + + std::vector lines; + std::string line; + while (std::getline(s, line)) { + // Remove a trailing \r + if (!line.empty() && line[line.size() - 1] == '\r') + line = line.substr(0, line.size() - 1); + lines.push_back(line); + } + + for (std::size_t i = 0; i < lines.size(); ++i) { + auto line = lines[i]; + if (StartsWith(line, "@stop")) { + // Force stop + break; + } else if (StartsWith(line, "@nsobid-")) { + // NSO Build ID Specifier + auto raw_build_id = line.substr(8); + if (raw_build_id.size() != 0x40) + raw_build_id.resize(0x40, '0'); + nso_build_id = Common::HexStringToArray<0x20>(raw_build_id); + } else if (StartsWith(line, "@flag offset_shift ")) { + // Offset Shift Flag + offset_shift = std::stoull(line.substr(19), nullptr, 0); + } else if (StartsWith(line, "#")) { + // Mandatory Comment + LOG_INFO(Loader, "[IPSwitchCompiler ('{}')] Forced output comment: {}", + patch_text->GetName(), line.substr(1)); + } else if (StartsWith(line, "@little-endian")) { + // Set values to read as little endian + is_little_endian = true; + } else if (StartsWith(line, "@big-endian")) { + // Set values to read as big endian + is_little_endian = false; + } else if (StartsWith(line, "@flag print_values")) { + // Force printing of applied values + print_values = true; + } else if (StartsWith(line, "@enabled") || StartsWith(line, "@disabled")) { + // Start of patch + const auto enabled = StartsWith(line, "@enabled"); + if (i == 0) + return; + const auto name = lines[i - 1].substr(3); + LOG_INFO(Loader, "[IPSwitchCompiler ('{}')] Parsing patch '{}' ({})", + patch_text->GetName(), name, line.substr(1)); + + IPSwitchPatch patch{name, enabled, {}}; + + // Read rest of patch + while (true) { + if (i + 1 >= lines.size()) + break; + line = lines[++i]; + + // 11 - 8 hex digit offset + space + minimum two digit overwrite val + if (line.length() < 11) + break; + auto offset = std::stoul(line.substr(0, 8), nullptr, 16); + offset += offset_shift; + + std::vector replace; + // 9 - first char of replacement val + if (line[9] == '\"') { + // string replacement + const auto end_index = line.find_last_of('\"'); + if (end_index == std::string::npos || end_index < 10) + return; + const auto value = line.substr(10, end_index - 10); + replace.reserve(value.size()); + std::copy(value.begin(), value.end(), std::back_inserter(replace)); + } else { + // hex replacement + const auto value = line.substr(9); + replace.reserve(value.size() / 2); + replace = Common::HexStringToVector(value, is_little_endian); + } + + if (print_values) { + LOG_INFO(Loader, + "[IPSwitchCompiler ('{}')] - Patching value at offset 0x{:08X} " + "with byte string '{}'", + patch_text->GetName(), offset, Common::HexVectorToString(replace)); + } + + patch.records.emplace(offset, replace); + } + + patches.push_back(std::move(patch)); + } + } + + valid = true; +} + +VirtualFile IPSwitchCompiler::Apply(const VirtualFile& in) const { + if (in == nullptr || !valid) + return nullptr; + + auto in_data = in->ReadAllBytes(); + + for (const auto& patch : patches) { + if (!patch.enabled) + continue; + + for (const auto& record : patch.records) { + if (record.first >= in_data.size()) + continue; + auto replace_size = record.second.size(); + if (record.first + replace_size > in_data.size()) + replace_size = in_data.size() - record.first; + for (std::size_t i = 0; i < replace_size; ++i) + in_data[i + record.first] = record.second[i]; + } + } + + return std::make_shared(in_data, in->GetName(), in->GetContainingDirectory()); +} + } // namespace FileSys diff --git a/src/core/file_sys/ips_layer.h b/src/core/file_sys/ips_layer.h index 81c163494..bb35542c8 100644 --- a/src/core/file_sys/ips_layer.h +++ b/src/core/file_sys/ips_layer.h @@ -12,4 +12,30 @@ namespace FileSys { VirtualFile PatchIPS(const VirtualFile& in, const VirtualFile& ips); +class IPSwitchCompiler { +public: + explicit IPSwitchCompiler(VirtualFile patch_text); + std::array GetBuildID() const; + bool IsValid() const; + VirtualFile Apply(const VirtualFile& in) const; + +private: + void Parse(); + + bool valid; + + struct IPSwitchPatch { + std::string name; + bool enabled; + std::map> records; + }; + + VirtualFile patch_text; + std::vector patches; + std::array nso_build_id; + bool is_little_endian; + u64 offset_shift; + bool print_values; +}; + } // namespace FileSys From 8886f2e55ef7a1acc5dab613c42cce8022fae996 Mon Sep 17 00:00:00 2001 From: Zach Hilman Date: Sun, 30 Sep 2018 18:13:38 -0400 Subject: [PATCH 3/6] patch_manager: Add support for IPSwitch format patches --- src/core/file_sys/patch_manager.cpp | 76 +++++++++++++++++++++-------- 1 file changed, 55 insertions(+), 21 deletions(-) diff --git a/src/core/file_sys/patch_manager.cpp b/src/core/file_sys/patch_manager.cpp index 539698f6e..f148e7719 100644 --- a/src/core/file_sys/patch_manager.cpp +++ b/src/core/file_sys/patch_manager.cpp @@ -73,27 +73,38 @@ VirtualDir PatchManager::PatchExeFS(VirtualDir exefs) const { return exefs; } -static std::vector CollectIPSPatches(const std::vector& patch_dirs, - const std::string& build_id) { - std::vector ips; +static std::vector CollectPatches(const std::vector& patch_dirs, + const std::string& build_id) { + std::vector out; ips.reserve(patch_dirs.size()); for (const auto& subdir : patch_dirs) { auto exefs_dir = subdir->GetSubdirectory("exefs"); if (exefs_dir != nullptr) { for (const auto& file : exefs_dir->GetFiles()) { - if (file->GetExtension() != "ips") - continue; - auto name = file->GetName(); - const auto p1 = name.substr(0, name.find('.')); - const auto this_build_id = p1.substr(0, p1.find_last_not_of('0') + 1); + if (file->GetExtension() == "ips") { + auto name = file->GetName(); + const auto p1 = name.substr(0, name.find('.')); + const auto this_build_id = p1.substr(0, p1.find_last_not_of('0') + 1); - if (build_id == this_build_id) - ips.push_back(file); + if (build_id == this_build_id) + out.push_back(file); + } else if (file->GetExtension() == "pchtxt") { + IPSwitchCompiler compiler{file}; + if (!compiler.IsValid()) + continue; + + auto this_build_id = Common::HexArrayToString(compiler.GetBuildID()); + this_build_id = + this_build_id.substr(0, this_build_id.find_last_not_of('0') + 1); + + if (build_id == this_build_id) + out.push_back(file); + } } } } - return ips; + return out; } std::vector PatchManager::PatchNSO(const std::vector& nso) const { @@ -115,15 +126,24 @@ std::vector PatchManager::PatchNSO(const std::vector& nso) const { auto patch_dirs = load_dir->GetSubdirectories(); std::sort(patch_dirs.begin(), patch_dirs.end(), [](const VirtualDir& l, const VirtualDir& r) { return l->GetName() < r->GetName(); }); - const auto ips = CollectIPSPatches(patch_dirs, build_id); + const auto patches = CollectPatches(patch_dirs, build_id); auto out = nso; - for (const auto& ips_file : ips) { - LOG_INFO(Loader, " - Appling IPS patch from mod \"{}\"", - ips_file->GetContainingDirectory()->GetParentDirectory()->GetName()); - const auto patched = PatchIPS(std::make_shared(out), ips_file); - if (patched != nullptr) - out = patched->ReadAllBytes(); + for (const auto& patch_file : patches) { + if (patch_file->GetExtension() == "ips") { + LOG_INFO(Loader, " - Applying IPS patch from mod \"{}\"", + patch_file->GetContainingDirectory()->GetParentDirectory()->GetName()); + const auto patched = PatchIPS(std::make_shared(out), patch_file); + if (patched != nullptr) + out = patched->ReadAllBytes(); + } else if (patch_file->GetExtension() == "pchtxt") { + LOG_INFO(Loader, " - Applying IPSwitch patch from mod \"{}\"", + patch_file->GetContainingDirectory()->GetParentDirectory()->GetName()); + const IPSwitchCompiler compiler{patch_file}; + const auto patched = compiler.Apply(std::make_shared(out)); + if (patched != nullptr) + out = patched->ReadAllBytes(); + } } if (out.size() < 0x100) @@ -143,7 +163,7 @@ bool PatchManager::HasNSOPatch(const std::array& build_id_) const { std::sort(patch_dirs.begin(), patch_dirs.end(), [](const VirtualDir& l, const VirtualDir& r) { return l->GetName() < r->GetName(); }); - return !CollectIPSPatches(patch_dirs, build_id).empty(); + return !CollectPatches(patch_dirs, build_id).empty(); } static void ApplyLayeredFS(VirtualFile& romfs, u64 title_id, ContentRecordType type) { @@ -253,8 +273,22 @@ std::map> PatchManager::GetPatchVersionNam if (mod_dir != nullptr && mod_dir->GetSize() > 0) { for (const auto& mod : mod_dir->GetSubdirectories()) { std::string types; - if (IsDirValidAndNonEmpty(mod->GetSubdirectory("exefs"))) - AppendCommaIfNotEmpty(types, "IPS"); + if (IsDirValidAndNonEmpty(mod->GetSubdirectory("exefs"))) { + bool ips = false; + bool ipswitch = false; + + for (const auto& file : mod->GetSubdirectory("exefs")->GetFiles()) { + if (file->GetExtension() == "ips") + ips = true; + else if (file->GetExtension() == "pchtxt") + ipswitch = true; + } + + if (ips) + AppendCommaIfNotEmpty(types, "IPS"); + if (ipswitch) + AppendCommaIfNotEmpty(types, "IPSwitch"); + } if (IsDirValidAndNonEmpty(mod->GetSubdirectory("romfs"))) AppendCommaIfNotEmpty(types, "LayeredFS"); From 9669cdb710e3242b7c0b705ea613587d36f79e00 Mon Sep 17 00:00:00 2001 From: Zach Hilman Date: Mon, 1 Oct 2018 08:31:34 -0400 Subject: [PATCH 4/6] ips_layer: Add support for escape sequences and midline comments More accurately follows IPSwitch specification. --- src/core/file_sys/ips_layer.cpp | 45 ++++++++++++++++++++++++++----- src/core/file_sys/ips_layer.h | 3 ++- src/core/file_sys/patch_manager.h | 1 + 3 files changed, 41 insertions(+), 8 deletions(-) diff --git a/src/core/file_sys/ips_layer.cpp b/src/core/file_sys/ips_layer.cpp index 6bce138a8..6c5535f83 100644 --- a/src/core/file_sys/ips_layer.cpp +++ b/src/core/file_sys/ips_layer.cpp @@ -17,6 +17,11 @@ enum class IPSFileType { Error, }; +const std::map ESCAPE_CHARACTER_MAP{ + {"\\a", "\a"}, {"\\b", "\b"}, {"\\f", "\f"}, {"\\n", "\n"}, {"\\r", "\r"}, {"\\t", "\t"}, + {"\\v", "\v"}, {"\\\\", "\\"}, {"\\\'", "\'"}, {"\\\"", "\""}, {"\\\?", "\?"}, +}; + static IPSFileType IdentifyMagic(const std::vector& magic) { if (magic.size() != 5) return IPSFileType::Error; @@ -89,7 +94,7 @@ VirtualFile PatchIPS(const VirtualFile& in, const VirtualFile& ips) { IPSwitchCompiler::IPSwitchCompiler(VirtualFile patch_text_) : valid(false), patch_text(std::move(patch_text_)), nso_build_id{}, is_little_endian(false), - offset_shift(0), print_values(false) { + offset_shift(0), print_values(false), last_comment("") { Parse(); } @@ -105,6 +110,18 @@ static bool StartsWith(const std::string& base, const std::string& check) { return base.size() >= check.size() && base.substr(0, check.size()) == check; } +static std::string EscapeStringSequences(std::string in) { + for (const auto& seq : ESCAPE_CHARACTER_MAP) { + for (auto index = in.find(seq.first); index != std::string::npos; + index = in.find(seq.first, index)) { + in.replace(index, std::strlen(seq.first), seq.second); + index += std::strlen(seq.second); + } + } + + return in; +} + void IPSwitchCompiler::Parse() { const auto bytes = patch_text->ReadAllBytes(); std::stringstream s; @@ -121,6 +138,13 @@ void IPSwitchCompiler::Parse() { for (std::size_t i = 0; i < lines.size(); ++i) { auto line = lines[i]; + + // Remove midline comments + if (!StartsWith(line, "//") && line.find("//") != std::string::npos) { + last_comment = line.substr(line.find("//") + 2); + line = line.substr(0, line.find("//")); + } + if (StartsWith(line, "@stop")) { // Force stop break; @@ -132,11 +156,18 @@ void IPSwitchCompiler::Parse() { nso_build_id = Common::HexStringToArray<0x20>(raw_build_id); } else if (StartsWith(line, "@flag offset_shift ")) { // Offset Shift Flag - offset_shift = std::stoull(line.substr(19), nullptr, 0); + offset_shift = std::stoll(line.substr(19), nullptr, 0); } else if (StartsWith(line, "#")) { // Mandatory Comment LOG_INFO(Loader, "[IPSwitchCompiler ('{}')] Forced output comment: {}", patch_text->GetName(), line.substr(1)); + } else if (StartsWith(line, "//")) { + // Normal Comment + last_comment = line.substr(2); + if (last_comment.find_first_not_of(' ') == std::string::npos) + continue; + if (last_comment.find_first_not_of(' ') != 0) + last_comment = last_comment.substr(last_comment.find_first_not_of(' ')); } else if (StartsWith(line, "@little-endian")) { // Set values to read as little endian is_little_endian = true; @@ -151,11 +182,10 @@ void IPSwitchCompiler::Parse() { const auto enabled = StartsWith(line, "@enabled"); if (i == 0) return; - const auto name = lines[i - 1].substr(3); LOG_INFO(Loader, "[IPSwitchCompiler ('{}')] Parsing patch '{}' ({})", - patch_text->GetName(), name, line.substr(1)); + patch_text->GetName(), last_comment, line.substr(1)); - IPSwitchPatch patch{name, enabled, {}}; + IPSwitchPatch patch{last_comment, enabled, {}}; // Read rest of patch while (true) { @@ -173,10 +203,11 @@ void IPSwitchCompiler::Parse() { // 9 - first char of replacement val if (line[9] == '\"') { // string replacement - const auto end_index = line.find_last_of('\"'); + const auto end_index = line.find('\"', 10); if (end_index == std::string::npos || end_index < 10) return; - const auto value = line.substr(10, end_index - 10); + auto value = line.substr(10, end_index - 10); + value = EscapeStringSequences(value); replace.reserve(value.size()); std::copy(value.begin(), value.end(), std::back_inserter(replace)); } else { diff --git a/src/core/file_sys/ips_layer.h b/src/core/file_sys/ips_layer.h index bb35542c8..847e9bf3c 100644 --- a/src/core/file_sys/ips_layer.h +++ b/src/core/file_sys/ips_layer.h @@ -34,8 +34,9 @@ private: std::vector patches; std::array nso_build_id; bool is_little_endian; - u64 offset_shift; + s64 offset_shift; bool print_values; + std::string last_comment; }; } // namespace FileSys diff --git a/src/core/file_sys/patch_manager.h b/src/core/file_sys/patch_manager.h index 6a864ec43..66fdba148 100644 --- a/src/core/file_sys/patch_manager.h +++ b/src/core/file_sys/patch_manager.h @@ -36,6 +36,7 @@ public: // Currently tracked NSO patches: // - IPS + // - IPSwitch std::vector PatchNSO(const std::vector& nso) const; // Checks to see if PatchNSO() will have any effect given the NSO's build ID. From 70bd2bb1d3aec23fc052c9612f9e530c1a2de72d Mon Sep 17 00:00:00 2001 From: Zach Hilman Date: Mon, 1 Oct 2018 17:17:08 -0400 Subject: [PATCH 5/6] ips_layer: Deduplicate resource usage --- src/common/hex_util.cpp | 2 +- src/common/hex_util.h | 2 +- src/core/file_sys/ips_layer.cpp | 48 +++++++++++++++-------------- src/core/file_sys/ips_layer.h | 14 +++++---- src/core/file_sys/patch_manager.cpp | 6 ++-- 5 files changed, 39 insertions(+), 33 deletions(-) diff --git a/src/common/hex_util.cpp b/src/common/hex_util.cpp index e516e5bbd..5b63f9e81 100644 --- a/src/common/hex_util.cpp +++ b/src/common/hex_util.cpp @@ -30,7 +30,7 @@ std::vector HexStringToVector(std::string_view str, bool little_endian) { return out; } -std::string HexVectorToString(std::vector vector, bool upper) { +std::string HexVectorToString(const std::vector& vector, bool upper) { std::string out; for (u8 c : vector) out += fmt::format(upper ? "{:02X}" : "{:02x}", c); diff --git a/src/common/hex_util.h b/src/common/hex_util.h index 0b7d3592a..68f003cb6 100644 --- a/src/common/hex_util.h +++ b/src/common/hex_util.h @@ -30,7 +30,7 @@ std::array HexStringToArray(std::string_view str) { return out; } -std::string HexVectorToString(std::vector vector, bool upper = true); +std::string HexVectorToString(const std::vector& vector, bool upper = true); template std::string HexArrayToString(std::array array, bool upper = true) { diff --git a/src/core/file_sys/ips_layer.cpp b/src/core/file_sys/ips_layer.cpp index 6c5535f83..abe3dcf62 100644 --- a/src/core/file_sys/ips_layer.cpp +++ b/src/core/file_sys/ips_layer.cpp @@ -17,9 +17,10 @@ enum class IPSFileType { Error, }; -const std::map ESCAPE_CHARACTER_MAP{ - {"\\a", "\a"}, {"\\b", "\b"}, {"\\f", "\f"}, {"\\n", "\n"}, {"\\r", "\r"}, {"\\t", "\t"}, - {"\\v", "\v"}, {"\\\\", "\\"}, {"\\\'", "\'"}, {"\\\"", "\""}, {"\\\?", "\?"}, +constexpr std::array, 11> ESCAPE_CHARACTER_MAP{ + std::pair{"\\a", "\a"}, {"\\b", "\b"}, {"\\f", "\f"}, {"\\n", "\n"}, + {"\\r", "\r"}, {"\\t", "\t"}, {"\\v", "\v"}, {"\\\\", "\\"}, + {"\\\'", "\'"}, {"\\\"", "\""}, {"\\\?", "\?"}, }; static IPSFileType IdentifyMagic(const std::vector& magic) { @@ -92,12 +93,12 @@ VirtualFile PatchIPS(const VirtualFile& in, const VirtualFile& ips) { return std::make_shared(in_data, in->GetName(), in->GetContainingDirectory()); } -IPSwitchCompiler::IPSwitchCompiler(VirtualFile patch_text_) - : valid(false), patch_text(std::move(patch_text_)), nso_build_id{}, is_little_endian(false), - offset_shift(0), print_values(false), last_comment("") { +IPSwitchCompiler::IPSwitchCompiler(VirtualFile patch_text_) : patch_text(std::move(patch_text_)) { Parse(); } +IPSwitchCompiler::~IPSwitchCompiler() = default; + std::array IPSwitchCompiler::GetBuildID() const { return nso_build_id; } @@ -106,7 +107,7 @@ bool IPSwitchCompiler::IsValid() const { return valid; } -static bool StartsWith(const std::string& base, const std::string& check) { +static bool StartsWith(std::string_view base, std::string_view check) { return base.size() >= check.size() && base.substr(0, check.size()) == check; } @@ -128,21 +129,22 @@ void IPSwitchCompiler::Parse() { s.write(reinterpret_cast(bytes.data()), bytes.size()); std::vector lines; - std::string line; - while (std::getline(s, line)) { + std::string stream_line; + while (std::getline(s, stream_line)) { // Remove a trailing \r - if (!line.empty() && line[line.size() - 1] == '\r') - line = line.substr(0, line.size() - 1); - lines.push_back(line); + if (!stream_line.empty() && stream_line.back() == '\r') + stream_line.pop_back(); + lines.push_back(std::move(stream_line)); } for (std::size_t i = 0; i < lines.size(); ++i) { auto line = lines[i]; // Remove midline comments - if (!StartsWith(line, "//") && line.find("//") != std::string::npos) { - last_comment = line.substr(line.find("//") + 2); - line = line.substr(0, line.find("//")); + const auto comment_index = line.find("//"); + if (!StartsWith(line, "//") && comment_index != std::string::npos) { + last_comment = line.substr(comment_index + 2); + line = line.substr(0, comment_index); } if (StartsWith(line, "@stop")) { @@ -191,28 +193,28 @@ void IPSwitchCompiler::Parse() { while (true) { if (i + 1 >= lines.size()) break; - line = lines[++i]; + const auto patch_line = lines[++i]; // 11 - 8 hex digit offset + space + minimum two digit overwrite val - if (line.length() < 11) + if (patch_line.length() < 11) break; - auto offset = std::stoul(line.substr(0, 8), nullptr, 16); + auto offset = std::stoul(patch_line.substr(0, 8), nullptr, 16); offset += offset_shift; std::vector replace; // 9 - first char of replacement val - if (line[9] == '\"') { + if (patch_line[9] == '\"') { // string replacement - const auto end_index = line.find('\"', 10); + const auto end_index = patch_line.find('\"', 10); if (end_index == std::string::npos || end_index < 10) return; - auto value = line.substr(10, end_index - 10); + auto value = patch_line.substr(10, end_index - 10); value = EscapeStringSequences(value); replace.reserve(value.size()); std::copy(value.begin(), value.end(), std::back_inserter(replace)); } else { // hex replacement - const auto value = line.substr(9); + const auto value = patch_line.substr(9); replace.reserve(value.size() / 2); replace = Common::HexStringToVector(value, is_little_endian); } @@ -224,7 +226,7 @@ void IPSwitchCompiler::Parse() { patch_text->GetName(), offset, Common::HexVectorToString(replace)); } - patch.records.emplace(offset, replace); + patch.records.emplace(offset, std::move(replace)); } patches.push_back(std::move(patch)); diff --git a/src/core/file_sys/ips_layer.h b/src/core/file_sys/ips_layer.h index 847e9bf3c..b07cc5673 100644 --- a/src/core/file_sys/ips_layer.h +++ b/src/core/file_sys/ips_layer.h @@ -15,6 +15,8 @@ VirtualFile PatchIPS(const VirtualFile& in, const VirtualFile& ips); class IPSwitchCompiler { public: explicit IPSwitchCompiler(VirtualFile patch_text); + ~IPSwitchCompiler(); + std::array GetBuildID() const; bool IsValid() const; VirtualFile Apply(const VirtualFile& in) const; @@ -22,7 +24,7 @@ public: private: void Parse(); - bool valid; + bool valid = false; struct IPSwitchPatch { std::string name; @@ -32,11 +34,11 @@ private: VirtualFile patch_text; std::vector patches; - std::array nso_build_id; - bool is_little_endian; - s64 offset_shift; - bool print_values; - std::string last_comment; + std::array nso_build_id{}; + bool is_little_endian = false; + s64 offset_shift = 0; + bool print_values = false; + std::string last_comment = ""; }; } // namespace FileSys diff --git a/src/core/file_sys/patch_manager.cpp b/src/core/file_sys/patch_manager.cpp index f148e7719..03df24906 100644 --- a/src/core/file_sys/patch_manager.cpp +++ b/src/core/file_sys/patch_manager.cpp @@ -273,11 +273,13 @@ std::map> PatchManager::GetPatchVersionNam if (mod_dir != nullptr && mod_dir->GetSize() > 0) { for (const auto& mod : mod_dir->GetSubdirectories()) { std::string types; - if (IsDirValidAndNonEmpty(mod->GetSubdirectory("exefs"))) { + + const auto exefs_dir = mod->GetSubdirectory("exefs"); + if (IsDirValidAndNonEmpty(exefs_dir)) { bool ips = false; bool ipswitch = false; - for (const auto& file : mod->GetSubdirectory("exefs")->GetFiles()) { + for (const auto& file : exefs_dir->GetFiles()) { if (file->GetExtension() == "ips") ips = true; else if (file->GetExtension() == "pchtxt") From 110d5784702282c594fe57f84f8d6bda21a82d50 Mon Sep 17 00:00:00 2001 From: Zach Hilman Date: Thu, 4 Oct 2018 12:23:18 -0400 Subject: [PATCH 6/6] ips_layer: Fix inaccuracies with comments and flags Specifically bugs/crashes that arise when putting them in positions that are legal but not typical, such as midline, between patch data, or between patch records. --- src/core/file_sys/ips_layer.cpp | 64 ++++++++++++++++++++++------- src/core/file_sys/ips_layer.h | 1 + src/core/file_sys/patch_manager.cpp | 2 +- 3 files changed, 51 insertions(+), 16 deletions(-) diff --git a/src/core/file_sys/ips_layer.cpp b/src/core/file_sys/ips_layer.cpp index abe3dcf62..0cadbc375 100644 --- a/src/core/file_sys/ips_layer.cpp +++ b/src/core/file_sys/ips_layer.cpp @@ -123,6 +123,22 @@ static std::string EscapeStringSequences(std::string in) { return in; } +void IPSwitchCompiler::ParseFlag(const std::string& line) { + if (StartsWith(line, "@flag offset_shift ")) { + // Offset Shift Flag + offset_shift = std::stoll(line.substr(19), nullptr, 0); + } else if (StartsWith(line, "@little-endian")) { + // Set values to read as little endian + is_little_endian = true; + } else if (StartsWith(line, "@big-endian")) { + // Set values to read as big endian + is_little_endian = false; + } else if (StartsWith(line, "@flag print_values")) { + // Force printing of applied values + print_values = true; + } +} + void IPSwitchCompiler::Parse() { const auto bytes = patch_text->ReadAllBytes(); std::stringstream s; @@ -141,7 +157,17 @@ void IPSwitchCompiler::Parse() { auto line = lines[i]; // Remove midline comments - const auto comment_index = line.find("//"); + std::size_t comment_index = std::string::npos; + bool within_string = false; + for (std::size_t k = 0; k < line.size(); ++k) { + if (line[k] == '\"' && (k > 0 && line[k - 1] != '\\')) { + within_string = !within_string; + } else if (line[k] == '\\' && (k < line.size() - 1 && line[k + 1] == '\\')) { + comment_index = k; + break; + } + } + if (!StartsWith(line, "//") && comment_index != std::string::npos) { last_comment = line.substr(comment_index + 2); line = line.substr(0, comment_index); @@ -156,9 +182,6 @@ void IPSwitchCompiler::Parse() { if (raw_build_id.size() != 0x40) raw_build_id.resize(0x40, '0'); nso_build_id = Common::HexStringToArray<0x20>(raw_build_id); - } else if (StartsWith(line, "@flag offset_shift ")) { - // Offset Shift Flag - offset_shift = std::stoll(line.substr(19), nullptr, 0); } else if (StartsWith(line, "#")) { // Mandatory Comment LOG_INFO(Loader, "[IPSwitchCompiler ('{}')] Forced output comment: {}", @@ -170,15 +193,6 @@ void IPSwitchCompiler::Parse() { continue; if (last_comment.find_first_not_of(' ') != 0) last_comment = last_comment.substr(last_comment.find_first_not_of(' ')); - } else if (StartsWith(line, "@little-endian")) { - // Set values to read as little endian - is_little_endian = true; - } else if (StartsWith(line, "@big-endian")) { - // Set values to read as big endian - is_little_endian = false; - } else if (StartsWith(line, "@flag print_values")) { - // Force printing of applied values - print_values = true; } else if (StartsWith(line, "@enabled") || StartsWith(line, "@disabled")) { // Start of patch const auto enabled = StartsWith(line, "@enabled"); @@ -195,6 +209,18 @@ void IPSwitchCompiler::Parse() { break; const auto patch_line = lines[++i]; + // Start of new patch + if (StartsWith(patch_line, "@enabled") || StartsWith(patch_line, "@disabled")) { + --i; + break; + } + + // Check for a flag + if (StartsWith(patch_line, "@")) { + ParseFlag(patch_line); + continue; + } + // 11 - 8 hex digit offset + space + minimum two digit overwrite val if (patch_line.length() < 11) break; @@ -205,9 +231,15 @@ void IPSwitchCompiler::Parse() { // 9 - first char of replacement val if (patch_line[9] == '\"') { // string replacement - const auto end_index = patch_line.find('\"', 10); + auto end_index = patch_line.find('\"', 10); if (end_index == std::string::npos || end_index < 10) return; + while (patch_line[end_index - 1] == '\\') { + end_index = patch_line.find('\"', end_index + 1); + if (end_index == std::string::npos || end_index < 10) + return; + } + auto value = patch_line.substr(10, end_index - 10); value = EscapeStringSequences(value); replace.reserve(value.size()); @@ -226,10 +258,12 @@ void IPSwitchCompiler::Parse() { patch_text->GetName(), offset, Common::HexVectorToString(replace)); } - patch.records.emplace(offset, std::move(replace)); + patch.records.insert_or_assign(offset, std::move(replace)); } patches.push_back(std::move(patch)); + } else if (StartsWith(line, "@")) { + ParseFlag(line); } } diff --git a/src/core/file_sys/ips_layer.h b/src/core/file_sys/ips_layer.h index b07cc5673..57da00da8 100644 --- a/src/core/file_sys/ips_layer.h +++ b/src/core/file_sys/ips_layer.h @@ -22,6 +22,7 @@ public: VirtualFile Apply(const VirtualFile& in) const; private: + void ParseFlag(const std::string& flag); void Parse(); bool valid = false; diff --git a/src/core/file_sys/patch_manager.cpp b/src/core/file_sys/patch_manager.cpp index 03df24906..ab2e5e43d 100644 --- a/src/core/file_sys/patch_manager.cpp +++ b/src/core/file_sys/patch_manager.cpp @@ -76,7 +76,7 @@ VirtualDir PatchManager::PatchExeFS(VirtualDir exefs) const { static std::vector CollectPatches(const std::vector& patch_dirs, const std::string& build_id) { std::vector out; - ips.reserve(patch_dirs.size()); + out.reserve(patch_dirs.size()); for (const auto& subdir : patch_dirs) { auto exefs_dir = subdir->GetSubdirectory("exefs"); if (exefs_dir != nullptr) {