From 51f70b6f4b054fbe19a1e813e0cf036d9093a032 Mon Sep 17 00:00:00 2001 From: water111 <48171810+water111@users.noreply.github.com> Date: Wed, 27 Jan 2021 20:46:58 -0500 Subject: [PATCH] [Tools] Add DGO packer and unpacker (#219) * add dgo tools * make codacy happy --- CMakeLists.txt | 3 ++ common/CMakeLists.txt | 2 + common/util/BinaryReader.h | 22 +++------ common/util/DgoReader.cpp | 54 ++++++++++++++++++++ common/util/DgoReader.h | 22 +++++++++ common/util/dgo_util.cpp | 53 ++++++++++++++++++++ common/util/dgo_util.h | 7 +++ decompiler/ObjectFile/ObjectFileDB.cpp | 68 +++----------------------- tools/CMakeLists.txt | 7 +++ tools/dgo_packer.cpp | 50 +++++++++++++++++++ tools/dgo_tools.md | 11 +++++ tools/dgo_unpacker.cpp | 37 ++++++++++++++ 12 files changed, 261 insertions(+), 75 deletions(-) create mode 100644 common/util/DgoReader.cpp create mode 100644 common/util/DgoReader.h create mode 100644 common/util/dgo_util.cpp create mode 100644 common/util/dgo_util.h create mode 100644 tools/CMakeLists.txt create mode 100644 tools/dgo_packer.cpp create mode 100644 tools/dgo_tools.md create mode 100644 tools/dgo_unpacker.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 7fa5af6ec..4f499d03c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -76,6 +76,9 @@ add_subdirectory(game) # build the compiler add_subdirectory(goalc) +# build standalone tools +add_subdirectory(tools) + # build the gtest libraries add_subdirectory(third-party/googletest) diff --git a/common/CMakeLists.txt b/common/CMakeLists.txt index 02a6f48f6..70421fb5b 100644 --- a/common/CMakeLists.txt +++ b/common/CMakeLists.txt @@ -14,6 +14,8 @@ add_library(common type_system/TypeFieldLookup.cpp type_system/TypeSpec.cpp type_system/TypeSystem.cpp + util/dgo_util.cpp + util/DgoReader.cpp util/DgoWriter.cpp util/FileUtil.cpp util/Timer.cpp diff --git a/common/util/BinaryReader.h b/common/util/BinaryReader.h index 7a678380a..5b123d9a4 100644 --- a/common/util/BinaryReader.h +++ b/common/util/BinaryReader.h @@ -11,32 +11,26 @@ class BinaryReader { public: - BinaryReader(uint8_t* _buffer, uint32_t _size) : buffer(_buffer), size(_size) {} - - explicit BinaryReader(std::vector& _buffer) - : buffer((uint8_t*)_buffer.data()), size(_buffer.size()) {} + explicit BinaryReader(const std::vector& _buffer) : buffer(_buffer) {} template T read() { - assert(seek + sizeof(T) <= size); - T& obj = *(T*)(buffer + seek); + assert(seek + sizeof(T) <= buffer.size()); + T& obj = *(T*)(buffer.data() + seek); seek += sizeof(T); return obj; } void ffwd(int amount) { seek += amount; - assert(seek <= size); + assert(seek <= buffer.size()); } - uint32_t bytes_left() const { return size - seek; } - - uint8_t* here() { return buffer + seek; } - - uint32_t get_seek() { return seek; } + uint32_t bytes_left() const { return buffer.size() - seek; } + uint8_t* here() { return buffer.data() + seek; } + uint32_t get_seek() const { return seek; } private: - uint8_t* buffer; - uint32_t size; + std::vector buffer; uint32_t seek = 0; }; diff --git a/common/util/DgoReader.cpp b/common/util/DgoReader.cpp new file mode 100644 index 000000000..fb25f10ab --- /dev/null +++ b/common/util/DgoReader.cpp @@ -0,0 +1,54 @@ +#include +#include +#include +#include "DgoReader.h" +#include "BinaryReader.h" +#include "common/link_types.h" +#include "third-party/json.hpp" +#include "dgo_util.h" + +DgoReader::DgoReader(std::string file_name, const std::vector& data) + : m_file_name(std::move(file_name)) { + BinaryReader reader(data); + auto header = reader.read(); + m_internal_name = header.name; + std::unordered_set all_unique_names; + + // get all obj files... + for (uint32_t i = 0; i < header.object_count; i++) { + auto obj_header = reader.read(); + assert(reader.bytes_left() >= obj_header.size); + assert_string_empty_after(obj_header.name, 60); + + DgoDataEntry entry; + entry.internal_name = obj_header.name; + entry.unique_name = get_object_file_name(entry.internal_name, reader.here(), obj_header.size); + all_unique_names.insert(entry.unique_name); + entry.data.resize(obj_header.size); + + assert((reader.get_seek() % 16) == 0); + memcpy(entry.data.data(), reader.here(), obj_header.size); + m_entries.push_back(entry); + + reader.ffwd(obj_header.size); + } + + // check we're at the end + assert(0 == reader.bytes_left()); + assert(all_unique_names.size() == m_entries.size()); +} + +std::string DgoReader::description_as_json() const { + using namespace nlohmann; + json j; + j["file_name"] = m_file_name; + j["internal_name"] = m_internal_name; + for (auto& entry : m_entries) { + json entry_desc; + entry_desc["unique_name"] = entry.unique_name; + entry_desc["internal_name"] = entry.internal_name; + j["objects"].push_back(entry_desc); + } + + return j.dump(4); +} \ No newline at end of file diff --git a/common/util/DgoReader.h b/common/util/DgoReader.h new file mode 100644 index 000000000..ff7693fdb --- /dev/null +++ b/common/util/DgoReader.h @@ -0,0 +1,22 @@ +#pragma once + +#include +#include +#include "common/common_types.h" + +struct DgoDataEntry { + std::vector data; + std::string internal_name; + std::string unique_name; +}; + +class DgoReader { + public: + DgoReader(std::string file_name, const std::vector& data); + const std::vector entries() const { return m_entries; } + std::string description_as_json() const; + + private: + std::vector m_entries; + std::string m_internal_name, m_file_name; +}; \ No newline at end of file diff --git a/common/util/dgo_util.cpp b/common/util/dgo_util.cpp new file mode 100644 index 000000000..6cda9dce9 --- /dev/null +++ b/common/util/dgo_util.cpp @@ -0,0 +1,53 @@ +#include +#include +#include "dgo_util.h" +#include "common/versions.h" +#include "third-party/fmt/core.h" + +/*! + * Assert false if the char[] has non-null data after the null terminated string. + * Used to sanity check the sizes of strings in DGO/object file headers. + */ +void assert_string_empty_after(const char* str, int size) { + auto ptr = str; + while (*ptr) + ptr++; + while (ptr - str < size) { + assert(!*ptr); + ptr++; + } +} + +std::string get_object_file_name(const std::string& original_name, u8* data, int size) { + const std::string art_group_text = + fmt::format("/src/next/data/art-group{}/", + versions::ART_FILE_VERSION); // todo, this may change in other games + const std::string suffix = "-ag.go"; + + int len = int(art_group_text.length()); + for (int start = 0; start < size; start++) { + bool failed = false; + for (int i = 0; i < len; i++) { + if (start + i >= size || data[start + i] != art_group_text[i]) { + failed = true; + break; + } + } + + if (!failed) { + for (int i = 0; i < int(original_name.length()); i++) { + if (start + len + i >= size || data[start + len + i] != original_name[i]) { + assert(false); + } + } + + assert(int(suffix.length()) + start + len + int(original_name.length()) < size); + assert( + !memcmp(data + start + len + original_name.length(), suffix.data(), suffix.length() + 1)); + + return original_name + "-ag"; + } + } + + return original_name; +} \ No newline at end of file diff --git a/common/util/dgo_util.h b/common/util/dgo_util.h new file mode 100644 index 000000000..10baaef46 --- /dev/null +++ b/common/util/dgo_util.h @@ -0,0 +1,7 @@ +#pragma once + +#include +#include "common/common_types.h" + +void assert_string_empty_after(const char* str, int size); +std::string get_object_file_name(const std::string& original_name, u8* data, int size); \ No newline at end of file diff --git a/decompiler/ObjectFile/ObjectFileDB.cpp b/decompiler/ObjectFile/ObjectFileDB.cpp index 63155d1d4..9862fa24c 100644 --- a/decompiler/ObjectFile/ObjectFileDB.cpp +++ b/decompiler/ObjectFile/ObjectFileDB.cpp @@ -10,6 +10,8 @@ #include #include #include +#include "common/link_types.h" +#include "common/util/dgo_util.h" #include "decompiler/data/tpage.h" #include "decompiler/data/game_text.h" #include "decompiler/data/StrFileReader.h" @@ -180,62 +182,6 @@ void ObjectFileDB::load_map_file(const std::string& map_data) { } } -// Header for a DGO file -struct DgoHeader { - uint32_t size; - char name[60]; -}; - -namespace { -/*! - * Assert false if the char[] has non-null data after the null terminated string. - * Used to sanity check the sizes of strings in DGO/object file headers. - */ -void assert_string_empty_after(const char* str, int size) { - auto ptr = str; - while (*ptr) - ptr++; - while (ptr - str < size) { - assert(!*ptr); - ptr++; - } -} -} // namespace - -namespace { -std::string get_object_file_name(const std::string& original_name, uint8_t* data, int size) { - const char art_group_text[] = - "/src/next/data/art-group6/"; // todo, this may change in other games - const char suffix[] = "-ag.go"; - - int len = int(strlen(art_group_text)); - for (int start = 0; start < size; start++) { - bool failed = false; - for (int i = 0; i < len; i++) { - if (start + i >= size || data[start + i] != art_group_text[i]) { - failed = true; - break; - } - } - - if (!failed) { - for (int i = 0; i < int(original_name.length()); i++) { - if (start + len + i >= size || data[start + len + i] != original_name[i]) { - assert(false); - } - } - - assert(int(strlen(suffix)) + start + len + int(original_name.length()) < size); - assert(!memcmp(data + start + len + original_name.length(), suffix, strlen(suffix) + 1)); - - return original_name + "-ag"; - } - } - - return original_name; -} -} // namespace - constexpr int MAX_CHUNK_SIZE = 0x8000; /*! * Load the objects stored in the given DGO into the ObjectFileDB @@ -303,9 +249,9 @@ void ObjectFileDB::get_objs_from_dgo(const std::string& filename) { assert_string_empty_after(header.name, 60); // get all obj files... - for (uint32_t i = 0; i < header.size; i++) { + for (uint32_t i = 0; i < header.object_count; i++) { auto obj_header = reader.read(); - assert(reader.bytes_left() >= obj_header.size); + assert(reader.bytes_left() >= obj_header.object_count); assert_string_empty_after(obj_header.name, 60); if (std::string(obj_header.name).find("-ag") != std::string::npos) { @@ -316,10 +262,10 @@ void ObjectFileDB::get_objs_from_dgo(const std::string& filename) { assert(false); } - auto name = get_object_file_name(obj_header.name, reader.here(), obj_header.size); + auto name = get_object_file_name(obj_header.name, reader.here(), obj_header.object_count); - add_obj_from_dgo(name, obj_header.name, reader.here(), obj_header.size, dgo_base_name); - reader.ffwd(obj_header.size); + add_obj_from_dgo(name, obj_header.name, reader.here(), obj_header.object_count, dgo_base_name); + reader.ffwd(obj_header.object_count); } // check we're at the end diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt new file mode 100644 index 000000000..aca1428d2 --- /dev/null +++ b/tools/CMakeLists.txt @@ -0,0 +1,7 @@ +add_executable(dgo_unpacker + dgo_unpacker.cpp) +target_link_libraries(dgo_unpacker common) + +add_executable(dgo_packer + dgo_packer.cpp) +target_link_libraries(dgo_packer common) \ No newline at end of file diff --git a/tools/dgo_packer.cpp b/tools/dgo_packer.cpp new file mode 100644 index 000000000..a33f9e46f --- /dev/null +++ b/tools/dgo_packer.cpp @@ -0,0 +1,50 @@ +#include +#include "common/versions.h" +#include "common/util/FileUtil.h" +#include "common/util/BinaryWriter.h" +#include "third-party/json.hpp" + +int main(int argc, char** argv) { + printf("OpenGOAL version %d.%d\n", versions::GOAL_VERSION_MAJOR, versions::GOAL_VERSION_MINOR); + printf("DGO Packing Tool\n"); + + if (argc < 3) { + printf("usage: dgo_packer \n"); + return 1; + } + + std::string out_path = argv[1]; + + for (int i = 2; i < argc; i++) { + std::string file_name = argv[i]; + std::string file_text = file_util::read_text_file(file_name); + + auto x = nlohmann::json::parse(file_text); + std::string out_file_name = x["file_name"]; + std::string internal_name = x["internal_name"]; + printf("Packing %s\n", internal_name.c_str()); + + BinaryWriter writer; + writer.add(x["objects"].size()); + writer.add_cstr_len(x["internal_name"].get().c_str(), 60); + + for (auto& entry : x["objects"]) { + auto obj_data = + file_util::read_binary_file(file_util::combine_path(out_path, entry["unique_name"])); + // size + writer.add(obj_data.size()); + // name + writer.add_str_len(entry["internal_name"].get().c_str(), 60); + // data + writer.add_data(obj_data.data(), obj_data.size()); + // pad + while (writer.get_size() & 0xf) { + writer.add(0); + } + } + writer.write_to_file(file_util::combine_path(out_path, "mod_" + out_file_name)); + } + + printf("Done\n"); + return 0; +} diff --git a/tools/dgo_tools.md b/tools/dgo_tools.md new file mode 100644 index 000000000..4241120dc --- /dev/null +++ b/tools/dgo_tools.md @@ -0,0 +1,11 @@ +## DGO Tools +The DGO packer and unpacker can be used to extract and repack files in DGOs. + +### Unpacking +Create a folder for the output, then run: +```tools/dgo_unpacker ``` +It will then place the files in the output folder. You may specify multiple DGO files, but this is not recommended because sometimes different DGO files have object files with the same name but different data. It will also create a DGO description file with the same name as the DGO file but with a `.txt` extension. + +### Repacking +```tools/dgo_packer ``` +It will repack the DGO. The name will be the same as the original DGO, but with `mod_` in the front. \ No newline at end of file diff --git a/tools/dgo_unpacker.cpp b/tools/dgo_unpacker.cpp new file mode 100644 index 000000000..cd22445d6 --- /dev/null +++ b/tools/dgo_unpacker.cpp @@ -0,0 +1,37 @@ +#include +#include "common/versions.h" +#include "common/util/FileUtil.h" +#include "common/util/DgoReader.h" + +int main(int argc, char** argv) { + printf("OpenGOAL version %d.%d\n", versions::GOAL_VERSION_MAJOR, versions::GOAL_VERSION_MINOR); + printf("DGO Unpacking Tool\n"); + + if (argc < 3) { + printf("usage: dgo_unpacker \n"); + return 1; + } + + std::string out_path = argv[1]; + + for (int i = 2; i < argc; i++) { + std::string file_name = argv[i]; + std::string base = file_util::base_name(file_name); + printf("Unpacking %s\n", base.c_str()); + // read the file + auto data = file_util::read_binary_file(file_name); + // read as a DGO + auto dgo = DgoReader(base, data); + // write dgo description + file_util::write_text_file(file_util::combine_path(out_path, base + ".txt"), + dgo.description_as_json()); + // write files: + for (auto& entry : dgo.entries()) { + file_util::write_binary_file(file_util::combine_path(out_path, entry.unique_name), + (void*)entry.data.data(), entry.data.size()); + } + } + + printf("Done\n"); + return 0; +}