From 332f0b2f2b41a6ed7c3ec616dd108d4cf274f41a Mon Sep 17 00:00:00 2001 From: Tyler Wilding Date: Sat, 24 Sep 2022 16:04:52 -0400 Subject: [PATCH] tools: add a tool to search for types based on size / type chain / fields (#1906) Just a small simple tool that can search through `all-types` for a type based on a bunch of criteria. For example: ![image](https://user-images.githubusercontent.com/13153231/192043561-181e5c5d-d5b1-41a9-8891-5cc3ed1a0efa.png) The results are printed to stdout, as well as output to a json file so they can be consumed by another tool (in my plans, the VSCode extension) --- .github/workflows/windows-build-clang.yaml | 2 +- .github/workflows/windows-build-msvc.yaml | 2 +- .vs/launch.vs.json | 9 +- Taskfile.yml | 4 + common/type_system/TypeSystem.cpp | 120 +++++++++++++++++++++ common/type_system/TypeSystem.h | 17 +++ scripts/tasks/Taskfile_darwin.yml | 1 + scripts/tasks/Taskfile_linux.yml | 1 + scripts/tasks/Taskfile_windows.yml | 1 + test/CMakeLists.txt | 2 +- tools/CMakeLists.txt | 3 + tools/memory_dump_tool/main.cpp | 4 +- tools/type_searcher/main.cpp | 100 +++++++++++++++++ 13 files changed, 260 insertions(+), 6 deletions(-) create mode 100644 tools/type_searcher/main.cpp diff --git a/.github/workflows/windows-build-clang.yaml b/.github/workflows/windows-build-clang.yaml index 7891494a2..c1c2bec10 100644 --- a/.github/workflows/windows-build-clang.yaml +++ b/.github/workflows/windows-build-clang.yaml @@ -50,7 +50,7 @@ jobs: timeout-minutes: 10 env: GTEST_OUTPUT: "xml:opengoal-test-report.xml" - run: ./build/bin/goalc-test.exe --gtest_color=yes --gtest_brief=1 --gtest_filter="-*MANUAL_TEST*" + run: ./build/bin/goalc-test.exe --gtest_color=yes --gtest_brief=0 --gtest_filter="-*MANUAL_TEST*" - name: Upload artifact uses: actions/upload-artifact@v3 diff --git a/.github/workflows/windows-build-msvc.yaml b/.github/workflows/windows-build-msvc.yaml index b5101fe1e..90b7ac37c 100644 --- a/.github/workflows/windows-build-msvc.yaml +++ b/.github/workflows/windows-build-msvc.yaml @@ -52,5 +52,5 @@ jobs: env: GTEST_OUTPUT: "xml:opengoal-test-report.xml" run: | - ./build/bin/goalc-test.exe --gtest_color=yes --gtest_brief=1 --gtest_filter="-*MANUAL_TEST*" + ./build/bin/goalc-test.exe --gtest_color=yes --gtest_brief=0 --gtest_filter="-*MANUAL_TEST*" diff --git a/.vs/launch.vs.json b/.vs/launch.vs.json index afe5cd74a..d44c9fd7e 100644 --- a/.vs/launch.vs.json +++ b/.vs/launch.vs.json @@ -173,8 +173,15 @@ "type" : "default", "project" : "CMakeLists.txt", "projectTarget" : "lsp.exe (bin\\lsp.exe)", - "name" : "Run - LSP", + "name" : "Tools - LSP", "args" : [] + }, + { + "type" : "default", + "project" : "CMakeLists.txt", + "projectTarget" : "type_searcher.exe (bin\\type_searcher.exe)", + "name" : "Tools - Type Searcher", + "args" : ["--game", "jak2", "--output-path", "./search-results.json", "--fields", "[{\\\"type\\\":\\\"int16\\\",\\\"offset\\\":2},{\\\"type\\\":\\\"int16\\\",\\\"offset\\\":4}]"] } ] } diff --git a/Taskfile.yml b/Taskfile.yml index cb2af92df..4beddd2ea 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -106,6 +106,10 @@ tasks: - watchmedo shell-command --drop --patterns="*.p2s" --recursive --command='task analyze-ee-memory FILE="${watch_src_path}"' "{{.SAVESTATE_DIR}}" vars: SAVESTATE_DIR: '{{default "." .SAVESTATE_DIR}}' + type-search: + desc: Just an example to show it running + cmds: + - "{{.TYPESEARCH_BIN_RELEASE_DIR}}/type_searcher --output-path ./search-results.json --game {{.GAME}} --fields '[{\"type\":\"int16\",\"offset\":2},{\"type\":\"int16\",\"offset\":4}]'" # TESTS offline-tests: cmds: diff --git a/common/type_system/TypeSystem.cpp b/common/type_system/TypeSystem.cpp index 41d4b6710..4490649ff 100644 --- a/common/type_system/TypeSystem.cpp +++ b/common/type_system/TypeSystem.cpp @@ -1304,6 +1304,126 @@ int TypeSystem::get_size_in_type(const Field& field) const { } } +std::vector TypeSystem::search_types_by_parent_type( + const std::string& parent_type, + const std::vector& existing_matches) { + std::vector results = {}; + // If we've been given a list of already matched types, narrow it down from there, otherwise + // iterate through the entire map + if (!existing_matches.empty()) { + for (const auto& type_name : existing_matches) { + if (typecheck_base_types(type_name, parent_type, false)) { + results.push_back(type_name); + } + } + } else { + for (const auto& [type_name, type_info] : m_types) { + // Only NullType's have no parent + if (!type_info->has_parent()) { + continue; + } + if (typecheck_base_types(type_name, parent_type, false)) { + results.push_back(type_name); + } + } + } + + return results; +} + +std::vector TypeSystem::search_types_by_size( + const int search_size, + const std::vector& existing_matches) { + std::vector results = {}; + // If we've been given a list of already matched types, narrow it down from there, otherwise + // iterate through the entire map + if (!existing_matches.empty()) { + for (const auto& type_name : existing_matches) { + if (m_types[type_name]->get_size_in_memory() == search_size) { + results.push_back(type_name); + } + } + } else { + for (const auto& [type_name, type_info] : m_types) { + // Only NullType's have no parent + if (!type_info->has_parent()) { + continue; + } + if (type_info->get_size_in_memory() == search_size) { + results.push_back(type_name); + } + } + } + + return results; +} + +std::vector TypeSystem::search_types_by_fields( + const std::vector& search_fields, + const std::vector& existing_matches) { + // TODO - maybe support partial matches eventually + std::vector results = {}; + if (!existing_matches.empty()) { + for (const auto& type_name : existing_matches) { + // For each type, look at it's fields + if (dynamic_cast(m_types[type_name].get()) != nullptr) { + bool type_valid = true; + auto struct_type = dynamic_cast(m_types[type_name].get()); + for (const auto& req_field : search_fields) { + bool field_valid = false; + // iterate through the type's fields until one is found with the right offset + // once found, check the underlying type name, if it doesn't match it's invalid + // if we don't find one with that offset, it's also invalid + for (const auto& type_field : struct_type->fields()) { + if (type_field.offset() == req_field.field_offset && + type_field.type().base_type() == req_field.field_type_name) { + field_valid = true; + break; + } + } + if (!field_valid) { + type_valid = false; + break; + } + } + if (type_valid) { + results.push_back(type_name); + } + } + } + } else { + for (const auto& [type_name, type_info] : m_types) { + // For each type, look at it's fields + if (dynamic_cast(type_info.get()) != nullptr) { + bool type_valid = true; + auto struct_type = dynamic_cast(type_info.get()); + for (const auto& req_field : search_fields) { + bool field_valid = false; + // iterate through the type's fields until one is found with the right offset + // once found, check the underlying type name, if it doesn't match it's invalid + // if we don't find one with that offset, it's also invalid + for (const auto& type_field : struct_type->fields()) { + if (type_field.offset() == req_field.field_offset && + type_field.type().base_type() == req_field.field_type_name) { + field_valid = true; + break; + } + } + if (!field_valid) { + type_valid = false; + break; + } + } + if (type_valid) { + results.push_back(type_name); + } + } + } + } + + return results; +} + /*! * Add a simple structure type - don't use this outside of add_builtin_types as it forces you to do * things in the wrong order. diff --git a/common/type_system/TypeSystem.h b/common/type_system/TypeSystem.h index 5029a32d3..69212a167 100644 --- a/common/type_system/TypeSystem.h +++ b/common/type_system/TypeSystem.h @@ -263,6 +263,23 @@ class TypeSystem { m_types_allowed_to_be_redefined.push_back(type_name); } + std::vector search_types_by_parent_type( + const std::string& parent_type, + const std::vector& existing_matches = {}); + + std::vector search_types_by_size( + const int search_size, + const std::vector& existing_matches = {}); + + struct TypeSearchFieldInput { + std::string field_type_name; + int field_offset; + }; + + std::vector search_types_by_fields( + const std::vector& search_fields, + const std::vector& existing_matches = {}); + private: std::string lca_base(const std::string& a, const std::string& b) const; bool typecheck_base_types(const std::string& expected, diff --git a/scripts/tasks/Taskfile_darwin.yml b/scripts/tasks/Taskfile_darwin.yml index cd2ffd928..41e66c06e 100644 --- a/scripts/tasks/Taskfile_darwin.yml +++ b/scripts/tasks/Taskfile_darwin.yml @@ -5,6 +5,7 @@ vars: GK_BIN_RELEASE_DIR: './build/game' DECOMP_BIN_RELEASE_DIR: './build/decompiler' MEMDUMP_BIN_RELEASE_DIR: './build/tools' + TYPESEARCH_BIN_RELEASE_DIR: './build/tools' OFFLINETEST_BIN_RELEASE_DIR: './build' GOALCTEST_BIN_RELEASE_DIR: './build' EXE_FILE_EXTENSION: '' diff --git a/scripts/tasks/Taskfile_linux.yml b/scripts/tasks/Taskfile_linux.yml index cd2ffd928..41e66c06e 100644 --- a/scripts/tasks/Taskfile_linux.yml +++ b/scripts/tasks/Taskfile_linux.yml @@ -5,6 +5,7 @@ vars: GK_BIN_RELEASE_DIR: './build/game' DECOMP_BIN_RELEASE_DIR: './build/decompiler' MEMDUMP_BIN_RELEASE_DIR: './build/tools' + TYPESEARCH_BIN_RELEASE_DIR: './build/tools' OFFLINETEST_BIN_RELEASE_DIR: './build' GOALCTEST_BIN_RELEASE_DIR: './build' EXE_FILE_EXTENSION: '' diff --git a/scripts/tasks/Taskfile_windows.yml b/scripts/tasks/Taskfile_windows.yml index 1ec940b20..a0a989f65 100644 --- a/scripts/tasks/Taskfile_windows.yml +++ b/scripts/tasks/Taskfile_windows.yml @@ -5,6 +5,7 @@ vars: GK_BIN_RELEASE_DIR: './out/build/Release/bin' DECOMP_BIN_RELEASE_DIR: './out/build/Release/bin' MEMDUMP_BIN_RELEASE_DIR: './out/build/Release/bin' + TYPESEARCH_BIN_RELEASE_DIR: './out/build/Release/bin' OFFLINETEST_BIN_RELEASE_DIR: './out/build/Release/bin' GOALCTEST_BIN_RELEASE_DIR: './out/build/Release/bin' EXE_FILE_EXTENSION: '.exe' diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index a8700f504..2537c9edb 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -51,7 +51,7 @@ if(UNIX AND CMAKE_COMPILER_IS_GNUCXX AND CODE_COVERAGE) include(CodeCoverage) append_coverage_compiler_flags() setup_target_for_coverage_lcov(NAME goalc-test_coverage - EXECUTABLE goalc-test --gtest_color=yes --gtest_brief=1 --gtest_filter="-*MANUAL_TEST*" + EXECUTABLE goalc-test --gtest_color=yes --gtest_brief=0 --gtest_filter="-*MANUAL_TEST*" DEPENDENCIES goalc-test EXCLUDE "third-party/*" "/usr/include/*") endif() diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 1dc90bd8b..9342494f0 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -12,3 +12,6 @@ add_executable(memory_dump_tool memory_dump_tool/main.cpp) target_link_libraries(memory_dump_tool common decomp) +add_executable(type_searcher + type_searcher/main.cpp) +target_link_libraries(type_searcher common decomp) diff --git a/tools/memory_dump_tool/main.cpp b/tools/memory_dump_tool/main.cpp index ad1ef6f9f..ecb381e64 100644 --- a/tools/memory_dump_tool/main.cpp +++ b/tools/memory_dump_tool/main.cpp @@ -9,7 +9,7 @@ #include "common/type_system/TypeSystem.h" #include "common/util/Assert.h" #include "common/util/FileUtil.h" -#include +#include "common/util/unicode_util.h" #include "decompiler/util/DecompilerTypeSystem.h" @@ -634,7 +634,7 @@ int main(int argc, char** argv) { decompiler::DecompilerTypeSystem dts(game_version); - // TODO - this could be better + // TODO - this could be better (have a `jak1` folder) if (game_version == GameVersion::Jak1) { dts.parse_type_defs({"decompiler", "config", "all-types.gc"}); } else if (game_version == GameVersion::Jak2) { diff --git a/tools/type_searcher/main.cpp b/tools/type_searcher/main.cpp new file mode 100644 index 000000000..b03678114 --- /dev/null +++ b/tools/type_searcher/main.cpp @@ -0,0 +1,100 @@ +// Iterates through the `all-types` DTS to find types that meet a variety of criteria, such as: +// - type size +// - field types at given offsets +// - parent-types +// - ... + +#include "common/log/log.h" +#include "common/util/FileUtil.h" +#include "common/util/json_util.h" +#include "common/util/unicode_util.h" + +#include "decompiler/util/DecompilerTypeSystem.h" + +#include "third-party/CLI11.hpp" +#include "third-party/fmt/core.h" +#include "third-party/json.hpp" + +int main(int argc, char** argv) { + ArgumentGuard u8_guard(argc, argv); + + fs::path output_path; + std::string game_name = "jak1"; + std::string parent_type = ""; + int type_size = -1; + std::string field_json = ""; + + lg::initialize(); + + CLI::App app{"OpenGOAL Type Searcher"}; + app.add_option("--output-path", output_path, "Where to output the search results file"); + app.add_option("-g,--game", game_name, "Specify the game name, defaults to 'jak1'"); + app.add_option("-s,--size", type_size, "The size of the type we are searching for"); + app.add_option("-p,--parent", parent_type, "The type of which it is an descendent of"); + app.add_option("-f,--fields", field_json, + "JSON encoded string specifying which field types and their offsets are required " + "- [{offset,type}]"); + app.validate_positionals(); + CLI11_PARSE(app, argc, argv); + + auto ok = file_util::setup_project_path({}); + if (!ok) { + lg::error("couldn't setup project path, exiting"); + return 1; + } + lg::info("Loading type definitions from all-types.gc..."); + + auto game_version = game_name_to_version(game_name); + + decompiler::DecompilerTypeSystem dts(game_version); + + // TODO - this could be better (have a jak1 folder) + if (game_version == GameVersion::Jak1) { + dts.parse_type_defs({"decompiler", "config", "all-types.gc"}); + } else if (game_version == GameVersion::Jak2) { + dts.parse_type_defs({"decompiler", "config", "jak2", "all-types.gc"}); + } else { + lg::error("unsupported game version"); + return 1; + } + + std::vector potential_types = {}; + + // First filter by parent type is available + if (!parent_type.empty()) { + potential_types = dts.ts.search_types_by_parent_type(parent_type); + } + + // Filter out types by size next + if (type_size != -1) { + potential_types = dts.ts.search_types_by_size(type_size, potential_types); + } + + // Filter out by fields + if (!field_json.empty()) { + std::vector search_fields = {}; + if (!field_json.empty()) { + auto data = parse_commented_json(field_json, "--fields arg"); + for (auto& item : data) { + TypeSystem::TypeSearchFieldInput new_field; + try { + new_field.field_offset = item.at("offset").get(); + new_field.field_type_name = item.at("type").get(); + search_fields.push_back(new_field); + } catch (std::exception& ex) { + fmt::print("Bad field search entry - {}", ex.what()); + } + } + } + potential_types = dts.ts.search_types_by_fields(search_fields, potential_types); + } + + auto results = nlohmann::json::array({}); + for (const auto& val : potential_types) { + fmt::print("{}\n", val); + results.push_back(val); + } + + // Output the results as a json list + file_util::write_text_file(output_path.string(), results.dump()); +}