tools: add a tool to search for types based on size / type chain / fields (#1906)

Just a small simple tool that can search through `all-types` for a type
based on a bunch of criteria.

For example:

![image](https://user-images.githubusercontent.com/13153231/192043561-181e5c5d-d5b1-41a9-8891-5cc3ed1a0efa.png)

The results are printed to stdout, as well as output to a json file so
they can be consumed by another tool (in my plans, the VSCode extension)

.github/workflows/windows-build-clang.yaml

@@ -50,7 +50,7 @@ jobs:
         timeout-minutes: 10
         env:
           GTEST_OUTPUT: "xml:opengoal-test-report.xml"
-        run: ./build/bin/goalc-test.exe --gtest_color=yes --gtest_brief=1 --gtest_filter="-*MANUAL_TEST*"
+        run: ./build/bin/goalc-test.exe --gtest_color=yes --gtest_brief=0 --gtest_filter="-*MANUAL_TEST*"
 
       - name: Upload artifact
         uses: actions/upload-artifact@v3

.github/workflows/windows-build-msvc.yaml

@@ -52,5 +52,5 @@ jobs:
         env:
           GTEST_OUTPUT: "xml:opengoal-test-report.xml"
         run: |
-          ./build/bin/goalc-test.exe --gtest_color=yes --gtest_brief=1 --gtest_filter="-*MANUAL_TEST*"
+          ./build/bin/goalc-test.exe --gtest_color=yes --gtest_brief=0 --gtest_filter="-*MANUAL_TEST*"
 

.vs/launch.vs.json

@@ -173,8 +173,15 @@
       "type" : "default",
       "project" : "CMakeLists.txt",
       "projectTarget" : "lsp.exe (bin\\lsp.exe)",
-      "name" : "Run - LSP",
+      "name" : "Tools - LSP",
       "args" : []
+    },
+    {
+      "type" : "default",
+      "project" : "CMakeLists.txt",
+      "projectTarget" : "type_searcher.exe (bin\\type_searcher.exe)",
+      "name" : "Tools - Type Searcher",
+      "args" : ["--game", "jak2", "--output-path", "./search-results.json", "--fields", "[{\\\"type\\\":\\\"int16\\\",\\\"offset\\\":2},{\\\"type\\\":\\\"int16\\\",\\\"offset\\\":4}]"]
     }
   ]
 }

Taskfile.yml

@@ -106,6 +106,10 @@ tasks:
       - watchmedo shell-command --drop --patterns="*.p2s" --recursive --command='task analyze-ee-memory FILE="${watch_src_path}"' "{{.SAVESTATE_DIR}}"
     vars:
       SAVESTATE_DIR: '{{default "." .SAVESTATE_DIR}}'
+  type-search:
+    desc: Just an example to show it running
+    cmds:
+      - "{{.TYPESEARCH_BIN_RELEASE_DIR}}/type_searcher --output-path ./search-results.json --game {{.GAME}} --fields '[{\"type\":\"int16\",\"offset\":2},{\"type\":\"int16\",\"offset\":4}]'"
   # TESTS
   offline-tests:
     cmds:

common/type_system/TypeSystem.cpp

@@ -1304,6 +1304,126 @@ int TypeSystem::get_size_in_type(const Field& field) const {
   }
 }
 
+std::vector<std::string> TypeSystem::search_types_by_parent_type(
+    const std::string& parent_type,
+    const std::vector<std::string>& existing_matches) {
+  std::vector<std::string> results = {};
+  // If we've been given a list of already matched types, narrow it down from there, otherwise
+  // iterate through the entire map
+  if (!existing_matches.empty()) {
+    for (const auto& type_name : existing_matches) {
+      if (typecheck_base_types(type_name, parent_type, false)) {
+        results.push_back(type_name);
+      }
+    }
+  } else {
+    for (const auto& [type_name, type_info] : m_types) {
+      // Only NullType's have no parent
+      if (!type_info->has_parent()) {
+        continue;
+      }
+      if (typecheck_base_types(type_name, parent_type, false)) {
+        results.push_back(type_name);
+      }
+    }
+  }
+
+  return results;
+}
+
+std::vector<std::string> TypeSystem::search_types_by_size(
+    const int search_size,
+    const std::vector<std::string>& existing_matches) {
+  std::vector<std::string> results = {};
+  // If we've been given a list of already matched types, narrow it down from there, otherwise
+  // iterate through the entire map
+  if (!existing_matches.empty()) {
+    for (const auto& type_name : existing_matches) {
+      if (m_types[type_name]->get_size_in_memory() == search_size) {
+        results.push_back(type_name);
+      }
+    }
+  } else {
+    for (const auto& [type_name, type_info] : m_types) {
+      // Only NullType's have no parent
+      if (!type_info->has_parent()) {
+        continue;
+      }
+      if (type_info->get_size_in_memory() == search_size) {
+        results.push_back(type_name);
+      }
+    }
+  }
+
+  return results;
+}
+
+std::vector<std::string> TypeSystem::search_types_by_fields(
+    const std::vector<TypeSearchFieldInput>& search_fields,
+    const std::vector<std::string>& existing_matches) {
+  // TODO - maybe support partial matches eventually
+  std::vector<std::string> results = {};
+  if (!existing_matches.empty()) {
+    for (const auto& type_name : existing_matches) {
+      // For each type, look at it's fields
+      if (dynamic_cast<StructureType*>(m_types[type_name].get()) != nullptr) {
+        bool type_valid = true;
+        auto struct_type = dynamic_cast<StructureType*>(m_types[type_name].get());
+        for (const auto& req_field : search_fields) {
+          bool field_valid = false;
+          // iterate through the type's fields until one is found with the right offset
+          // once found, check the underlying type name, if it doesn't match it's invalid
+          // if we don't find one with that offset, it's also invalid
+          for (const auto& type_field : struct_type->fields()) {
+            if (type_field.offset() == req_field.field_offset &&
+                type_field.type().base_type() == req_field.field_type_name) {
+              field_valid = true;
+              break;
+            }
+          }
+          if (!field_valid) {
+            type_valid = false;
+            break;
+          }
+        }
+        if (type_valid) {
+          results.push_back(type_name);
+        }
+      }
+    }
+  } else {
+    for (const auto& [type_name, type_info] : m_types) {
+      // For each type, look at it's fields
+      if (dynamic_cast<StructureType*>(type_info.get()) != nullptr) {
+        bool type_valid = true;
+        auto struct_type = dynamic_cast<StructureType*>(type_info.get());
+        for (const auto& req_field : search_fields) {
+          bool field_valid = false;
+          // iterate through the type's fields until one is found with the right offset
+          // once found, check the underlying type name, if it doesn't match it's invalid
+          // if we don't find one with that offset, it's also invalid
+          for (const auto& type_field : struct_type->fields()) {
+            if (type_field.offset() == req_field.field_offset &&
+                type_field.type().base_type() == req_field.field_type_name) {
+              field_valid = true;
+              break;
+            }
+          }
+          if (!field_valid) {
+            type_valid = false;
+            break;
+          }
+        }
+        if (type_valid) {
+          results.push_back(type_name);
+        }
+      }
+    }
+  }
+
+  return results;
+}
+
 /*!
  * Add a simple structure type - don't use this outside of add_builtin_types as it forces you to do
  * things in the wrong order.

common/type_system/TypeSystem.h

@@ -263,6 +263,23 @@ class TypeSystem {
     m_types_allowed_to_be_redefined.push_back(type_name);
   }
 
+  std::vector<std::string> search_types_by_parent_type(
+      const std::string& parent_type,
+      const std::vector<std::string>& existing_matches = {});
+
+  std::vector<std::string> search_types_by_size(
+      const int search_size,
+      const std::vector<std::string>& existing_matches = {});
+
+  struct TypeSearchFieldInput {
+    std::string field_type_name;
+    int field_offset;
+  };
+
+  std::vector<std::string> search_types_by_fields(
+      const std::vector<TypeSearchFieldInput>& search_fields,
+      const std::vector<std::string>& existing_matches = {});
+
  private:
   std::string lca_base(const std::string& a, const std::string& b) const;
   bool typecheck_base_types(const std::string& expected,

scripts/tasks/Taskfile_darwin.yml

@@ -5,6 +5,7 @@ vars:
   GK_BIN_RELEASE_DIR: './build/game'
   DECOMP_BIN_RELEASE_DIR: './build/decompiler'
   MEMDUMP_BIN_RELEASE_DIR: './build/tools'
+  TYPESEARCH_BIN_RELEASE_DIR: './build/tools'
   OFFLINETEST_BIN_RELEASE_DIR: './build'
   GOALCTEST_BIN_RELEASE_DIR: './build'
   EXE_FILE_EXTENSION: ''

scripts/tasks/Taskfile_linux.yml

@@ -5,6 +5,7 @@ vars:
   GK_BIN_RELEASE_DIR: './build/game'
   DECOMP_BIN_RELEASE_DIR: './build/decompiler'
   MEMDUMP_BIN_RELEASE_DIR: './build/tools'
+  TYPESEARCH_BIN_RELEASE_DIR: './build/tools'
   OFFLINETEST_BIN_RELEASE_DIR: './build'
   GOALCTEST_BIN_RELEASE_DIR: './build'
   EXE_FILE_EXTENSION: ''

scripts/tasks/Taskfile_windows.yml

@@ -5,6 +5,7 @@ vars:
   GK_BIN_RELEASE_DIR: './out/build/Release/bin'
   DECOMP_BIN_RELEASE_DIR: './out/build/Release/bin'
   MEMDUMP_BIN_RELEASE_DIR: './out/build/Release/bin'
+  TYPESEARCH_BIN_RELEASE_DIR: './out/build/Release/bin'
   OFFLINETEST_BIN_RELEASE_DIR: './out/build/Release/bin'
   GOALCTEST_BIN_RELEASE_DIR: './out/build/Release/bin'
   EXE_FILE_EXTENSION: '.exe'

test/CMakeLists.txt

@@ -51,7 +51,7 @@ if(UNIX AND CMAKE_COMPILER_IS_GNUCXX AND CODE_COVERAGE)
   include(CodeCoverage)
   append_coverage_compiler_flags()
   setup_target_for_coverage_lcov(NAME goalc-test_coverage
-                                 EXECUTABLE goalc-test --gtest_color=yes --gtest_brief=1 --gtest_filter="-*MANUAL_TEST*"
+                                 EXECUTABLE goalc-test --gtest_color=yes --gtest_brief=0 --gtest_filter="-*MANUAL_TEST*"
                                  DEPENDENCIES goalc-test
                                  EXCLUDE "third-party/*" "/usr/include/*")
 endif()

tools/CMakeLists.txt

@@ -12,3 +12,6 @@ add_executable(memory_dump_tool
         memory_dump_tool/main.cpp)
 target_link_libraries(memory_dump_tool common decomp)
 
+add_executable(type_searcher
+        type_searcher/main.cpp)
+target_link_libraries(type_searcher common decomp)

tools/memory_dump_tool/main.cpp

@@ -9,7 +9,7 @@
 #include "common/type_system/TypeSystem.h"
 #include "common/util/Assert.h"
 #include "common/util/FileUtil.h"
-#include <common/util/unicode_util.h>
+#include "common/util/unicode_util.h"
 
 #include "decompiler/util/DecompilerTypeSystem.h"
 
@@ -634,7 +634,7 @@ int main(int argc, char** argv) {
 
   decompiler::DecompilerTypeSystem dts(game_version);
 
-  // TODO - this could be better
+  // TODO - this could be better (have a `jak1` folder)
   if (game_version == GameVersion::Jak1) {
     dts.parse_type_defs({"decompiler", "config", "all-types.gc"});
   } else if (game_version == GameVersion::Jak2) {

tools/type_searcher/main.cpp

@@ -0,0 +1,100 @@
+// Iterates through the `all-types` DTS to find types that meet a variety of criteria, such as:
+// - type size
+// - field types at given offsets
+// - parent-types
+// - ...
+
+#include "common/log/log.h"
+#include "common/util/FileUtil.h"
+#include "common/util/json_util.h"
+#include "common/util/unicode_util.h"
+
+#include "decompiler/util/DecompilerTypeSystem.h"
+
+#include "third-party/CLI11.hpp"
+#include "third-party/fmt/core.h"
+#include "third-party/json.hpp"
+
+int main(int argc, char** argv) {
+  ArgumentGuard u8_guard(argc, argv);
+
+  fs::path output_path;
+  std::string game_name = "jak1";
+  std::string parent_type = "";
+  int type_size = -1;
+  std::string field_json = "";
+
+  lg::initialize();
+
+  CLI::App app{"OpenGOAL Type Searcher"};
+  app.add_option("--output-path", output_path, "Where to output the search results file");
+  app.add_option("-g,--game", game_name, "Specify the game name, defaults to 'jak1'");
+  app.add_option("-s,--size", type_size, "The size of the type we are searching for");
+  app.add_option("-p,--parent", parent_type, "The type of which it is an descendent of");
+  app.add_option("-f,--fields", field_json,
+                 "JSON encoded string specifying which field types and their offsets are required "
+                 "- [{offset,type}]");
+  app.validate_positionals();
+  CLI11_PARSE(app, argc, argv);
+
+  auto ok = file_util::setup_project_path({});
+  if (!ok) {
+    lg::error("couldn't setup project path, exiting");
+    return 1;
+  }
+  lg::info("Loading type definitions from all-types.gc...");
+
+  auto game_version = game_name_to_version(game_name);
+
+  decompiler::DecompilerTypeSystem dts(game_version);
+
+  // TODO - this could be better (have a jak1 folder)
+  if (game_version == GameVersion::Jak1) {
+    dts.parse_type_defs({"decompiler", "config", "all-types.gc"});
+  } else if (game_version == GameVersion::Jak2) {
+    dts.parse_type_defs({"decompiler", "config", "jak2", "all-types.gc"});
+  } else {
+    lg::error("unsupported game version");
+    return 1;
+  }
+
+  std::vector<std::string> potential_types = {};
+
+  // First filter by parent type is available
+  if (!parent_type.empty()) {
+    potential_types = dts.ts.search_types_by_parent_type(parent_type);
+  }
+
+  // Filter out types by size next
+  if (type_size != -1) {
+    potential_types = dts.ts.search_types_by_size(type_size, potential_types);
+  }
+
+  // Filter out by fields
+  if (!field_json.empty()) {
+    std::vector<TypeSystem::TypeSearchFieldInput> search_fields = {};
+    if (!field_json.empty()) {
+      auto data = parse_commented_json(field_json, "--fields arg");
+      for (auto& item : data) {
+        TypeSystem::TypeSearchFieldInput new_field;
+        try {
+          new_field.field_offset = item.at("offset").get<int>();
+          new_field.field_type_name = item.at("type").get<std::string>();
+          search_fields.push_back(new_field);
+        } catch (std::exception& ex) {
+          fmt::print("Bad field search entry - {}", ex.what());
+        }
+      }
+    }
+    potential_types = dts.ts.search_types_by_fields(search_fields, potential_types);
+  }
+
+  auto results = nlohmann::json::array({});
+  for (const auto& val : potential_types) {
+    fmt::print("{}\n", val);
+    results.push_back(val);
+  }
+
+  // Output the results as a json list
+  file_util::write_text_file(output_path.string(), results.dump());
+}