mirror of
https://github.com/open-goal/jak-project.git
synced 2024-10-20 00:57:44 -04:00
tools: add a tool to search for types based on size / type chain / fields (#1906)
Just a small simple tool that can search through `all-types` for a type based on a bunch of criteria. For example: ![image](https://user-images.githubusercontent.com/13153231/192043561-181e5c5d-d5b1-41a9-8891-5cc3ed1a0efa.png) The results are printed to stdout, as well as output to a json file so they can be consumed by another tool (in my plans, the VSCode extension)
This commit is contained in:
parent
7dd8053697
commit
332f0b2f2b
2
.github/workflows/windows-build-clang.yaml
vendored
2
.github/workflows/windows-build-clang.yaml
vendored
|
@ -50,7 +50,7 @@ jobs:
|
|||
timeout-minutes: 10
|
||||
env:
|
||||
GTEST_OUTPUT: "xml:opengoal-test-report.xml"
|
||||
run: ./build/bin/goalc-test.exe --gtest_color=yes --gtest_brief=1 --gtest_filter="-*MANUAL_TEST*"
|
||||
run: ./build/bin/goalc-test.exe --gtest_color=yes --gtest_brief=0 --gtest_filter="-*MANUAL_TEST*"
|
||||
|
||||
- name: Upload artifact
|
||||
uses: actions/upload-artifact@v3
|
||||
|
|
2
.github/workflows/windows-build-msvc.yaml
vendored
2
.github/workflows/windows-build-msvc.yaml
vendored
|
@ -52,5 +52,5 @@ jobs:
|
|||
env:
|
||||
GTEST_OUTPUT: "xml:opengoal-test-report.xml"
|
||||
run: |
|
||||
./build/bin/goalc-test.exe --gtest_color=yes --gtest_brief=1 --gtest_filter="-*MANUAL_TEST*"
|
||||
./build/bin/goalc-test.exe --gtest_color=yes --gtest_brief=0 --gtest_filter="-*MANUAL_TEST*"
|
||||
|
||||
|
|
|
@ -173,8 +173,15 @@
|
|||
"type" : "default",
|
||||
"project" : "CMakeLists.txt",
|
||||
"projectTarget" : "lsp.exe (bin\\lsp.exe)",
|
||||
"name" : "Run - LSP",
|
||||
"name" : "Tools - LSP",
|
||||
"args" : []
|
||||
},
|
||||
{
|
||||
"type" : "default",
|
||||
"project" : "CMakeLists.txt",
|
||||
"projectTarget" : "type_searcher.exe (bin\\type_searcher.exe)",
|
||||
"name" : "Tools - Type Searcher",
|
||||
"args" : ["--game", "jak2", "--output-path", "./search-results.json", "--fields", "[{\\\"type\\\":\\\"int16\\\",\\\"offset\\\":2},{\\\"type\\\":\\\"int16\\\",\\\"offset\\\":4}]"]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
|
@ -106,6 +106,10 @@ tasks:
|
|||
- watchmedo shell-command --drop --patterns="*.p2s" --recursive --command='task analyze-ee-memory FILE="${watch_src_path}"' "{{.SAVESTATE_DIR}}"
|
||||
vars:
|
||||
SAVESTATE_DIR: '{{default "." .SAVESTATE_DIR}}'
|
||||
type-search:
|
||||
desc: Just an example to show it running
|
||||
cmds:
|
||||
- "{{.TYPESEARCH_BIN_RELEASE_DIR}}/type_searcher --output-path ./search-results.json --game {{.GAME}} --fields '[{\"type\":\"int16\",\"offset\":2},{\"type\":\"int16\",\"offset\":4}]'"
|
||||
# TESTS
|
||||
offline-tests:
|
||||
cmds:
|
||||
|
|
|
@ -1304,6 +1304,126 @@ int TypeSystem::get_size_in_type(const Field& field) const {
|
|||
}
|
||||
}
|
||||
|
||||
std::vector<std::string> TypeSystem::search_types_by_parent_type(
|
||||
const std::string& parent_type,
|
||||
const std::vector<std::string>& existing_matches) {
|
||||
std::vector<std::string> results = {};
|
||||
// If we've been given a list of already matched types, narrow it down from there, otherwise
|
||||
// iterate through the entire map
|
||||
if (!existing_matches.empty()) {
|
||||
for (const auto& type_name : existing_matches) {
|
||||
if (typecheck_base_types(type_name, parent_type, false)) {
|
||||
results.push_back(type_name);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
for (const auto& [type_name, type_info] : m_types) {
|
||||
// Only NullType's have no parent
|
||||
if (!type_info->has_parent()) {
|
||||
continue;
|
||||
}
|
||||
if (typecheck_base_types(type_name, parent_type, false)) {
|
||||
results.push_back(type_name);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
std::vector<std::string> TypeSystem::search_types_by_size(
|
||||
const int search_size,
|
||||
const std::vector<std::string>& existing_matches) {
|
||||
std::vector<std::string> results = {};
|
||||
// If we've been given a list of already matched types, narrow it down from there, otherwise
|
||||
// iterate through the entire map
|
||||
if (!existing_matches.empty()) {
|
||||
for (const auto& type_name : existing_matches) {
|
||||
if (m_types[type_name]->get_size_in_memory() == search_size) {
|
||||
results.push_back(type_name);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
for (const auto& [type_name, type_info] : m_types) {
|
||||
// Only NullType's have no parent
|
||||
if (!type_info->has_parent()) {
|
||||
continue;
|
||||
}
|
||||
if (type_info->get_size_in_memory() == search_size) {
|
||||
results.push_back(type_name);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
std::vector<std::string> TypeSystem::search_types_by_fields(
|
||||
const std::vector<TypeSearchFieldInput>& search_fields,
|
||||
const std::vector<std::string>& existing_matches) {
|
||||
// TODO - maybe support partial matches eventually
|
||||
std::vector<std::string> results = {};
|
||||
if (!existing_matches.empty()) {
|
||||
for (const auto& type_name : existing_matches) {
|
||||
// For each type, look at it's fields
|
||||
if (dynamic_cast<StructureType*>(m_types[type_name].get()) != nullptr) {
|
||||
bool type_valid = true;
|
||||
auto struct_type = dynamic_cast<StructureType*>(m_types[type_name].get());
|
||||
for (const auto& req_field : search_fields) {
|
||||
bool field_valid = false;
|
||||
// iterate through the type's fields until one is found with the right offset
|
||||
// once found, check the underlying type name, if it doesn't match it's invalid
|
||||
// if we don't find one with that offset, it's also invalid
|
||||
for (const auto& type_field : struct_type->fields()) {
|
||||
if (type_field.offset() == req_field.field_offset &&
|
||||
type_field.type().base_type() == req_field.field_type_name) {
|
||||
field_valid = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!field_valid) {
|
||||
type_valid = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (type_valid) {
|
||||
results.push_back(type_name);
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
for (const auto& [type_name, type_info] : m_types) {
|
||||
// For each type, look at it's fields
|
||||
if (dynamic_cast<StructureType*>(type_info.get()) != nullptr) {
|
||||
bool type_valid = true;
|
||||
auto struct_type = dynamic_cast<StructureType*>(type_info.get());
|
||||
for (const auto& req_field : search_fields) {
|
||||
bool field_valid = false;
|
||||
// iterate through the type's fields until one is found with the right offset
|
||||
// once found, check the underlying type name, if it doesn't match it's invalid
|
||||
// if we don't find one with that offset, it's also invalid
|
||||
for (const auto& type_field : struct_type->fields()) {
|
||||
if (type_field.offset() == req_field.field_offset &&
|
||||
type_field.type().base_type() == req_field.field_type_name) {
|
||||
field_valid = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!field_valid) {
|
||||
type_valid = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (type_valid) {
|
||||
results.push_back(type_name);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
/*!
|
||||
* Add a simple structure type - don't use this outside of add_builtin_types as it forces you to do
|
||||
* things in the wrong order.
|
||||
|
|
|
@ -263,6 +263,23 @@ class TypeSystem {
|
|||
m_types_allowed_to_be_redefined.push_back(type_name);
|
||||
}
|
||||
|
||||
std::vector<std::string> search_types_by_parent_type(
|
||||
const std::string& parent_type,
|
||||
const std::vector<std::string>& existing_matches = {});
|
||||
|
||||
std::vector<std::string> search_types_by_size(
|
||||
const int search_size,
|
||||
const std::vector<std::string>& existing_matches = {});
|
||||
|
||||
struct TypeSearchFieldInput {
|
||||
std::string field_type_name;
|
||||
int field_offset;
|
||||
};
|
||||
|
||||
std::vector<std::string> search_types_by_fields(
|
||||
const std::vector<TypeSearchFieldInput>& search_fields,
|
||||
const std::vector<std::string>& existing_matches = {});
|
||||
|
||||
private:
|
||||
std::string lca_base(const std::string& a, const std::string& b) const;
|
||||
bool typecheck_base_types(const std::string& expected,
|
||||
|
|
|
@ -5,6 +5,7 @@ vars:
|
|||
GK_BIN_RELEASE_DIR: './build/game'
|
||||
DECOMP_BIN_RELEASE_DIR: './build/decompiler'
|
||||
MEMDUMP_BIN_RELEASE_DIR: './build/tools'
|
||||
TYPESEARCH_BIN_RELEASE_DIR: './build/tools'
|
||||
OFFLINETEST_BIN_RELEASE_DIR: './build'
|
||||
GOALCTEST_BIN_RELEASE_DIR: './build'
|
||||
EXE_FILE_EXTENSION: ''
|
||||
|
|
|
@ -5,6 +5,7 @@ vars:
|
|||
GK_BIN_RELEASE_DIR: './build/game'
|
||||
DECOMP_BIN_RELEASE_DIR: './build/decompiler'
|
||||
MEMDUMP_BIN_RELEASE_DIR: './build/tools'
|
||||
TYPESEARCH_BIN_RELEASE_DIR: './build/tools'
|
||||
OFFLINETEST_BIN_RELEASE_DIR: './build'
|
||||
GOALCTEST_BIN_RELEASE_DIR: './build'
|
||||
EXE_FILE_EXTENSION: ''
|
||||
|
|
|
@ -5,6 +5,7 @@ vars:
|
|||
GK_BIN_RELEASE_DIR: './out/build/Release/bin'
|
||||
DECOMP_BIN_RELEASE_DIR: './out/build/Release/bin'
|
||||
MEMDUMP_BIN_RELEASE_DIR: './out/build/Release/bin'
|
||||
TYPESEARCH_BIN_RELEASE_DIR: './out/build/Release/bin'
|
||||
OFFLINETEST_BIN_RELEASE_DIR: './out/build/Release/bin'
|
||||
GOALCTEST_BIN_RELEASE_DIR: './out/build/Release/bin'
|
||||
EXE_FILE_EXTENSION: '.exe'
|
||||
|
|
|
@ -51,7 +51,7 @@ if(UNIX AND CMAKE_COMPILER_IS_GNUCXX AND CODE_COVERAGE)
|
|||
include(CodeCoverage)
|
||||
append_coverage_compiler_flags()
|
||||
setup_target_for_coverage_lcov(NAME goalc-test_coverage
|
||||
EXECUTABLE goalc-test --gtest_color=yes --gtest_brief=1 --gtest_filter="-*MANUAL_TEST*"
|
||||
EXECUTABLE goalc-test --gtest_color=yes --gtest_brief=0 --gtest_filter="-*MANUAL_TEST*"
|
||||
DEPENDENCIES goalc-test
|
||||
EXCLUDE "third-party/*" "/usr/include/*")
|
||||
endif()
|
||||
|
|
|
@ -12,3 +12,6 @@ add_executable(memory_dump_tool
|
|||
memory_dump_tool/main.cpp)
|
||||
target_link_libraries(memory_dump_tool common decomp)
|
||||
|
||||
add_executable(type_searcher
|
||||
type_searcher/main.cpp)
|
||||
target_link_libraries(type_searcher common decomp)
|
||||
|
|
|
@ -9,7 +9,7 @@
|
|||
#include "common/type_system/TypeSystem.h"
|
||||
#include "common/util/Assert.h"
|
||||
#include "common/util/FileUtil.h"
|
||||
#include <common/util/unicode_util.h>
|
||||
#include "common/util/unicode_util.h"
|
||||
|
||||
#include "decompiler/util/DecompilerTypeSystem.h"
|
||||
|
||||
|
@ -634,7 +634,7 @@ int main(int argc, char** argv) {
|
|||
|
||||
decompiler::DecompilerTypeSystem dts(game_version);
|
||||
|
||||
// TODO - this could be better
|
||||
// TODO - this could be better (have a `jak1` folder)
|
||||
if (game_version == GameVersion::Jak1) {
|
||||
dts.parse_type_defs({"decompiler", "config", "all-types.gc"});
|
||||
} else if (game_version == GameVersion::Jak2) {
|
||||
|
|
100
tools/type_searcher/main.cpp
Normal file
100
tools/type_searcher/main.cpp
Normal file
|
@ -0,0 +1,100 @@
|
|||
// Iterates through the `all-types` DTS to find types that meet a variety of criteria, such as:
|
||||
// - type size
|
||||
// - field types at given offsets
|
||||
// - parent-types
|
||||
// - ...
|
||||
|
||||
#include "common/log/log.h"
|
||||
#include "common/util/FileUtil.h"
|
||||
#include "common/util/json_util.h"
|
||||
#include "common/util/unicode_util.h"
|
||||
|
||||
#include "decompiler/util/DecompilerTypeSystem.h"
|
||||
|
||||
#include "third-party/CLI11.hpp"
|
||||
#include "third-party/fmt/core.h"
|
||||
#include "third-party/json.hpp"
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
ArgumentGuard u8_guard(argc, argv);
|
||||
|
||||
fs::path output_path;
|
||||
std::string game_name = "jak1";
|
||||
std::string parent_type = "";
|
||||
int type_size = -1;
|
||||
std::string field_json = "";
|
||||
|
||||
lg::initialize();
|
||||
|
||||
CLI::App app{"OpenGOAL Type Searcher"};
|
||||
app.add_option("--output-path", output_path, "Where to output the search results file");
|
||||
app.add_option("-g,--game", game_name, "Specify the game name, defaults to 'jak1'");
|
||||
app.add_option("-s,--size", type_size, "The size of the type we are searching for");
|
||||
app.add_option("-p,--parent", parent_type, "The type of which it is an descendent of");
|
||||
app.add_option("-f,--fields", field_json,
|
||||
"JSON encoded string specifying which field types and their offsets are required "
|
||||
"- [{offset,type}]");
|
||||
app.validate_positionals();
|
||||
CLI11_PARSE(app, argc, argv);
|
||||
|
||||
auto ok = file_util::setup_project_path({});
|
||||
if (!ok) {
|
||||
lg::error("couldn't setup project path, exiting");
|
||||
return 1;
|
||||
}
|
||||
lg::info("Loading type definitions from all-types.gc...");
|
||||
|
||||
auto game_version = game_name_to_version(game_name);
|
||||
|
||||
decompiler::DecompilerTypeSystem dts(game_version);
|
||||
|
||||
// TODO - this could be better (have a jak1 folder)
|
||||
if (game_version == GameVersion::Jak1) {
|
||||
dts.parse_type_defs({"decompiler", "config", "all-types.gc"});
|
||||
} else if (game_version == GameVersion::Jak2) {
|
||||
dts.parse_type_defs({"decompiler", "config", "jak2", "all-types.gc"});
|
||||
} else {
|
||||
lg::error("unsupported game version");
|
||||
return 1;
|
||||
}
|
||||
|
||||
std::vector<std::string> potential_types = {};
|
||||
|
||||
// First filter by parent type is available
|
||||
if (!parent_type.empty()) {
|
||||
potential_types = dts.ts.search_types_by_parent_type(parent_type);
|
||||
}
|
||||
|
||||
// Filter out types by size next
|
||||
if (type_size != -1) {
|
||||
potential_types = dts.ts.search_types_by_size(type_size, potential_types);
|
||||
}
|
||||
|
||||
// Filter out by fields
|
||||
if (!field_json.empty()) {
|
||||
std::vector<TypeSystem::TypeSearchFieldInput> search_fields = {};
|
||||
if (!field_json.empty()) {
|
||||
auto data = parse_commented_json(field_json, "--fields arg");
|
||||
for (auto& item : data) {
|
||||
TypeSystem::TypeSearchFieldInput new_field;
|
||||
try {
|
||||
new_field.field_offset = item.at("offset").get<int>();
|
||||
new_field.field_type_name = item.at("type").get<std::string>();
|
||||
search_fields.push_back(new_field);
|
||||
} catch (std::exception& ex) {
|
||||
fmt::print("Bad field search entry - {}", ex.what());
|
||||
}
|
||||
}
|
||||
}
|
||||
potential_types = dts.ts.search_types_by_fields(search_fields, potential_types);
|
||||
}
|
||||
|
||||
auto results = nlohmann::json::array({});
|
||||
for (const auto& val : potential_types) {
|
||||
fmt::print("{}\n", val);
|
||||
results.push_back(val);
|
||||
}
|
||||
|
||||
// Output the results as a json list
|
||||
file_util::write_text_file(output_path.string(), results.dump());
|
||||
}
|
Loading…
Reference in a new issue