tools: Fix the memory dump analysis tool (#1690)

* stash

* tools: memory dump working for jak 1 again

* tools: memory dump working for jak 2, kinda hacky?

.vs/launch.vs.json

@@ -151,8 +151,15 @@
       "type": "default",
       "project": "CMakeLists.txt",
       "projectTarget": "memory_dump_tool.exe (bin\\memory_dump_tool.exe)",
-      "name": "Tools - EE Memory Analyze",
+      "name": "Tools - EE Memory Analyze - Jak 1",
-      "args": ["${workspaceRoot}/eeMemory.bin", "${workspaceRoot}"]
+      "args": ["\"${workspaceRoot}/eeMemory.bin\"", "--output-path", "\"${workspaceRoot}\"", "--game", "jak1"]
+    },
+    {
+      "type": "default",
+      "project": "CMakeLists.txt",
+      "projectTarget": "memory_dump_tool.exe (bin\\memory_dump_tool.exe)",
+      "name": "Tools - EE Memory Analyze - Jak 2",
+      "args": ["\"${workspaceRoot}/eeMemory.bin\"", "--output-path", "\"${workspaceRoot}\"", "--game", "jak2"]
     },
     {
       "type": "default",

LICENSE

@@ -1,6 +1,6 @@
 ISC License
 
-Copyright (c) 2020-2021 OpenGOAL Team
+Copyright (c) 2020-2022 OpenGOAL Team
 
 Permission to use, copy, modify, and/or distribute this software for any
 purpose with or without fee is hereby granted, provided that the above

common/goal_constants.h

@@ -18,6 +18,9 @@ constexpr int ARRAY_DATA_OFFSET = 12;  // not including type tag
 // constexpr s32 SYM_INFO_OFFSET = 8167 * 8 - 4;
 
 namespace jak1 {
+constexpr s32 ORIGINAL_MAX_GOAL_SYMBOLS = 8192;
+constexpr s32 ORIGINAL_SYM_TO_STRING_OFFSET = 0xff38;
+
 constexpr s32 GOAL_MAX_SYMBOLS = 16384;
 constexpr s32 SYM_INFO_OFFSET = GOAL_MAX_SYMBOLS * 8 - 4;
 

tools/CMakeLists.txt

@@ -9,6 +9,6 @@ add_executable(dgo_packer
 target_link_libraries(dgo_packer common)
 
 add_executable(memory_dump_tool
-        MemoryDumpTool/main.cpp)
+        memory_dump_tool/main.cpp)
 target_link_libraries(memory_dump_tool common decomp)
 

tools/memory_dump_tool/main.cpp

@@ -4,6 +4,7 @@
 #include <string>
 
 #include "common/goal_constants.h"
+#include "common/log/log.h"
 #include "common/symbols.h"
 #include "common/type_system/TypeSystem.h"
 #include "common/util/Assert.h"
@@ -12,13 +13,10 @@
 
 #include "decompiler/util/DecompilerTypeSystem.h"
 
+#include "third-party/CLI11.hpp"
 #include "third-party/fmt/core.h"
 #include "third-party/json.hpp"
 
-namespace fs = fs;
-
-constexpr GameVersion kGameVersion = GameVersion::Jak1;
-
 struct Ram {
   const u8* data = nullptr;
   u32 size = 0;
@@ -79,22 +77,35 @@ struct Ram {
   bool word_in_memory(u32 addr) const { return in_memory<u32>(addr); }
 };
 
-u32 scan_for_symbol_table(const Ram& ram, u32 start_addr, u32 end_addr) {
+u32 scan_for_symbol_table(const Ram& ram,
+                          const GameVersion& game_version,
+                          u32 start_addr,
+                          u32 end_addr) {
   fmt::print("scanning for symbol table in 0x{:x} - 0x{:x}\n", start_addr, end_addr);
   std::vector<u32> candidates;
 
   // look for the false symbol.
+  if (game_version == GameVersion::Jak1) {
     for (u32 addr = (start_addr & 0xfffffff0); addr < end_addr; addr += 8) {
       if (ram.word(addr + 4) == addr + 4) {
         candidates.push_back(addr);
+        lg::info("candidate 0x{:x}", addr);
+      }
+    }
+  } else {
+    for (u32 addr = (start_addr & 0xfffffff0); addr < end_addr; addr += 4) {
+      if (ram.word(addr) == addr + 1) {
+        candidates.push_back(addr + 1);
+        lg::info("candidate 0x{:x}", addr + 1);
+      }
     }
   }
 
   fmt::print("got {} candidates for #f:\n", candidates.size());
 
+  if (game_version == GameVersion::Jak1) {
     for (auto addr : candidates) {
-    // todo: this is wrong
+      auto str = addr + jak1::ORIGINAL_SYM_TO_STRING_OFFSET;
-    auto str = addr + BASIC_OFFSET + jak1::SYM_INFO_OFFSET;
       fmt::print(" trying 0x{:x}:\n", addr);
       if (ram.word_in_memory(str)) {
         auto mem = ram.word(str + 4);         // offset of str in SymInfo
@@ -108,6 +119,23 @@ u32 scan_for_symbol_table(const Ram& ram, u32 start_addr, u32 end_addr) {
         }
       }
     }
+  } else {
+    for (auto addr : candidates) {
+      auto str = addr + jak2::SYM_TO_STRING_OFFSET;
+      fmt::print(" trying 0x{:x}:\n", addr);
+      if (ram.word_in_memory(str)) {
+        auto mem = ram.word(str);             // offset of str in SymInfo
+        auto name = ram.try_string(mem + 4);  // offset of data in GOAL string
+        if (name) {
+          fmt::print("   name: {}\n", *name);
+        }
+        if (name == "#f") {
+          fmt::print("Got #f = 0x{:x}!\n", addr + 4);
+          return addr;
+        }
+      }
+    }
+  }
 
   return 0;
 }
@@ -118,26 +146,17 @@ struct SymbolMap {
   std::unordered_map<u32, std::string> addr_to_name;
 };
 
-SymbolMap build_symbol_map(const Ram& ram, u32 s7) {
+SymbolMap build_symbol_map(const GameVersion& game_version, const Ram& ram, u32 s7) {
-  // TODO jak 1 specific
+  lg::info("building symbol map...");
-  fmt::print("finding symbols...\n");
   SymbolMap map;
-  /*
-   s7 = symbol_table + (GOAL_MAX_SYMBOLS / 2) * 8 + BASIC_OFFSET;
-  // pointer to the first symbol (SymbolTable2 is the "lower" symbol table)
-  SymbolTable2 = symbol_table + BASIC_OFFSET;
-  // the last symbol we will ever access.
-  LastSymbol = symbol_table + 0xff00;
-   */
 
-  // todo wrong
+  if (game_version == GameVersion::Jak1) {
-  auto symbol_table = s7 - ((jak1::GOAL_MAX_SYMBOLS / 2) * 8 + BASIC_OFFSET);
+    auto addr_start_of_sym_table = s7 - ((jak1::ORIGINAL_MAX_GOAL_SYMBOLS / 2) * 8);
-  auto SymbolTable2 = symbol_table + BASIC_OFFSET;
+    auto addr_last_symbol = addr_start_of_sym_table + 0xff00;
-  auto LastSymbol = symbol_table + 0xff00;
 
-  for (u32 sym = SymbolTable2; sym < LastSymbol; sym += 8) {
+    for (u32 sym = addr_start_of_sym_table; sym < addr_last_symbol; sym += 8) {
-    auto info = sym + jak1::SYM_INFO_OFFSET;  // already has basic offset
+      auto info = sym + jak1::ORIGINAL_SYM_TO_STRING_OFFSET;  // already has basic offset
-    auto str = ram.word(info + 4);
+      auto str = ram.word(info);
       if (str) {
         auto name = ram.string(str + 4);
         if (name != "asize-of-basic-func") {
@@ -148,18 +167,37 @@ SymbolMap build_symbol_map(const Ram& ram, u32 s7) {
         }
       }
     }
+  } else {
+    auto addr_start_of_sym_table = s7 - ((jak2::GOAL_MAX_SYMBOLS / 2) * 4) + BASIC_OFFSET;
+    auto addr_last_symbol = addr_start_of_sym_table + 0xff00;  // ? the same ?
+
+    for (u32 sym = addr_start_of_sym_table; sym < addr_last_symbol; sym += 4) {
+      auto info = sym + jak2::SYM_TO_STRING_OFFSET;
+      auto str = ram.word(info);
+      if (str) {
+        auto name = ram.string(str + 4);
+        if (name != "asize-of-basic-func") {
+          ASSERT(map.name_to_addr.find(name) == map.name_to_addr.end());
+          map.name_to_addr[name] = sym;
+          map.addr_to_name[sym] = name;
+          map.name_to_value[name] = ram.word(sym);
+        }
+      }
+    }
+  }
 
   ASSERT(map.name_to_addr.size() == map.addr_to_name.size());
-  fmt::print("found {} symbols.\n", map.name_to_addr.size());
+  lg::info("found {} symbols", map.name_to_addr.size());
   return map;
 }
 
 std::unordered_map<u32, std::string> build_type_map(const Ram& ram,
                                                     const SymbolMap& symbols,
+                                                    const GameVersion& game_version,
                                                     u32 s7) {
-  // TODO jak 1 specific
   std::unordered_map<u32, std::string> result;
-  fmt::print("finding types...\n");
+  lg::info("finding types...");
+  if (game_version == GameVersion::Jak1) {
     u32 type_of_type = ram.word(s7 + jak1_symbols::FIX_SYM_TYPE_TYPE);
     ASSERT(type_of_type == ram.word(symbols.name_to_addr.at("type")));
 
@@ -171,15 +209,31 @@ std::unordered_map<u32, std::string> build_type_map(const Ram& ram,
         }
       }
     }
+  } else {
+    u32 type_of_type = ram.word(s7 + jak2_symbols::FIX_SYM_TYPE_TYPE - 1);
+    ASSERT(type_of_type == ram.word(symbols.name_to_addr.at("type") - 1));
 
-  fmt::print("found {} types\n", result.size());
+    for (const auto& [name, addr] : symbols.name_to_addr) {
+      u32 value = ram.word(addr - 1);
+      if (ram.word_in_memory(value - 4) && ((value & 0x7) == BASIC_OFFSET)) {
+        if (ram.word(value - 4) == type_of_type) {
+          result[value] = name;
+        }
+      }
+    }
+  }
+
+  lg::info("found {} types", result.size());
   return result;
 }
 
+const std::vector<std::string> ignored_types = {"symbol", "string", "function", "object",
+                                                "integer"};
+
 std::unordered_map<std::string, std::vector<u32>> find_basics(
     const Ram& ram,
     const std::unordered_map<u32, std::string>& type_map) {
-  fmt::print("Scanning memory for objects. This may take a while...\n");
+  lg::info("Scanning memory for objects. This may take a while...");
 
   std::unordered_map<std::string, std::vector<u32>> result;
   int total_objects = 0;
@@ -188,14 +242,14 @@ std::unordered_map<std::string, std::vector<u32>> find_basics(
     u32 tag = ram.word(addr);
     auto iter = type_map.find(tag);
     // ignore the stupid types.
-    if (iter != type_map.end() && iter->second != "symbol" && iter->second != "string" &&
+    if (iter != type_map.end() && std::find(ignored_types.begin(), ignored_types.end(),
-        iter->second != "function" && iter->second != "object" && iter->second != "integer") {
+                                            iter->second) == ignored_types.end()) {
       result[iter->second].push_back(addr);
       total_objects++;
     }
   }
 
-  fmt::print("Got {} objects of {} unique types\n", total_objects, result.size());
+  lg::info("Got {} objects of {} unique types\n", total_objects, result.size());
   return result;
 }
 
@@ -563,66 +617,92 @@ int main(int argc, char** argv) {
     exit(EXIT_FAILURE);
   }
 
-  fmt::print("MemoryDumpTool\n");
+  fs::path dump_path;
+  fs::path output_path;
+  std::string game_name = "jak1";
 
-  if (argc != 2 && argc != 3) {
+  lg::initialize();
-    fmt::print("usage: memory_dump_tool <ee_ram.bin|savestate.p2s> [output folder]\n");
+
+  CLI::App app{"OpenGOAL Memory Dump Analyzer"};
+  app.add_option("dump-path", dump_path, "The path to the dump file to analyze")->required();
+  app.add_option("--output-path", output_path,
+                 "Where the output files should be sent, defaults to current directory otherwise");
+  app.add_option("-g,--game", game_name, "Specify the game name, defaults to 'jak1'");
+  app.validate_positionals();
+  CLI11_PARSE(app, argc, argv);
+
+  auto ok = file_util::setup_project_path({});
+  if (!ok) {
+    lg::error("couldn't setup project path, exiting");
+    return 1;
+  }
+  lg::info("Loading type definitions from all-types.gc...");
+
+  auto game_version = game_name_to_version(game_name);
+
+  decompiler::DecompilerTypeSystem dts(game_version);
+
+  // TODO - this could be better
+  if (game_version == GameVersion::Jak1) {
+    dts.parse_type_defs({"decompiler", "config", "all-types.gc"});
+  } else if (game_version == GameVersion::Jak2) {
+    dts.parse_type_defs({"decompiler", "config", "jak2", "all-types.gc"});
+  } else {
+    lg::error("unsupported game version");
     return 1;
   }
 
-  fmt::print("Loading type definitions from all-types.gc...\n");
-  decompiler::DecompilerTypeSystem dts(kGameVersion);
-  dts.parse_type_defs({"decompiler", "config", "all-types.gc"});
-
-  std::string file_name = argv[1];
   fs::path output_folder;
 
-  if (!fs::exists(output_folder) || argc < 3) {
+  if (output_folder.empty() || !fs::exists(output_folder)) {
-    fmt::print("Output folder not found, defaulting to current directory");
+    lg::warn("Output folder not found or not provided, defaulting to current directory");
-    output_folder = ".";
+    output_folder = "./";
-  } else {
-    output_folder = argv[2];
   }
 
-  if (ends_with(file_name, "p2s")) {
+  if (dump_path.extension() == "p2s") {
-    fmt::print("PS2 savestates are not directly supported. Please extract contents beforehand.\n");
+    lg::error("PCSX2 savestates are not directly supported. Please extract contents beforehand");
     return 1;
   }
 
-  fmt::print("Loading memory from '{}'\n", file_name);
+  lg::info("Loading memory from '{}'", dump_path.string());
-  auto data = file_util::read_binary_file(file_name);
+  auto data = file_util::read_binary_file(dump_path);
 
   u32 one_mb = (1 << 20);
 
   if (data.size() == 32 * one_mb) {
-    fmt::print("Got 32MB file\n");
+    lg::info("Got 32MB file");
   } else if (data.size() == 128 * one_mb) {
-    fmt::print("Got 128MB file\n");
+    lg::info("Got 128MB file");
   } else if (data.size() == 127 * one_mb) {
-    fmt::print("Got a 127MB file. Assuming this is a dump with the first 1 MB missing.\n");
+    lg::warn("Got a 127MB file. Assuming this is a dump with the first 1 MB missing.\n");
     data.insert(data.begin(), one_mb, 0);
-    ASSERT(data.size() == 128 * one_mb);
+    if (data.size() != 128 * one_mb) {
+      lg::error("it was not!");
+      return 1;
+    }
   } else {
-    fmt::print("Invalid size: {} bytes\n", data.size());
+    lg::error("Invalid size: {} bytes", data.size());
+    return 1;
   }
 
   Ram ram(data.data(), data.size());
 
-  u32 s7 = scan_for_symbol_table(ram, one_mb, 2 * one_mb);
+  u32 s7 = scan_for_symbol_table(ram, game_version, one_mb, 2 * one_mb);
   if (!s7) {
-    fmt::print("Failed to find symbol table\n");
+    lg::error("Failed to find symbol table");
     return 1;
   }
 
   nlohmann::json results;
-  if (fs::exists(output_folder / "ee-results.json")) {
+  auto json_path = fmt::format("ee-results-{}.json", game_name);
-    fmt::print("Found existing result file, appending results to it!\n");
+  if (fs::exists(output_folder / json_path)) {
-    std::ifstream i(output_folder / "ee-results.json");
+    lg::info("Found existing result file, appending results to it!");
+    std::ifstream i(output_folder / json_path);
     i >> results;
   }
 
-  auto symbol_map = build_symbol_map(ram, s7);
+  auto symbol_map = build_symbol_map(game_version, ram, s7);
-  auto types = build_type_map(ram, symbol_map, s7);
+  auto types = build_type_map(ram, symbol_map, game_version, s7);
   auto basics = find_basics(ram, types);
 
   follow_references_to_find_pointers(ram, dts.ts, basics, s7 + 0x100);
@@ -631,10 +711,10 @@ int main(int argc, char** argv) {
   inspect_symbols(ram, types, symbol_map);
   inspect_process_self(ram, basics, types, dts.ts);
 
-  if (fs::exists(output_folder / "ee-results.json")) {
+  if (fs::exists(output_folder / json_path)) {
-    fs::remove(output_folder / "ee-results.json");
+    fs::remove(output_folder / json_path);
   }
-  std::ofstream o(output_folder / "ee-results.json");
+  std::ofstream o(output_folder / json_path);
   o << std::setw(2) << results << std::endl;
 
   return 0;