From 6ce1d2a0c7f59580953cbb2d77d39af7ded24cd4 Mon Sep 17 00:00:00 2001 From: Tyler Wilding Date: Fri, 22 Jul 2022 11:55:18 -0400 Subject: [PATCH] tools: Fix the memory dump analysis tool (#1690) * stash * tools: memory dump working for jak 1 again * tools: memory dump working for jak 2, kinda hacky? --- .vs/launch.vs.json | 11 +- LICENSE | 2 +- common/goal_constants.h | 3 + tools/CMakeLists.txt | 2 +- .../main.cpp | 262 ++++++++++++------ 5 files changed, 185 insertions(+), 95 deletions(-) rename tools/{MemoryDumpTool => memory_dump_tool}/main.cpp (70%) diff --git a/.vs/launch.vs.json b/.vs/launch.vs.json index 24b20316c..62799b4db 100644 --- a/.vs/launch.vs.json +++ b/.vs/launch.vs.json @@ -151,8 +151,15 @@ "type": "default", "project": "CMakeLists.txt", "projectTarget": "memory_dump_tool.exe (bin\\memory_dump_tool.exe)", - "name": "Tools - EE Memory Analyze", - "args": ["${workspaceRoot}/eeMemory.bin", "${workspaceRoot}"] + "name": "Tools - EE Memory Analyze - Jak 1", + "args": ["\"${workspaceRoot}/eeMemory.bin\"", "--output-path", "\"${workspaceRoot}\"", "--game", "jak1"] + }, + { + "type": "default", + "project": "CMakeLists.txt", + "projectTarget": "memory_dump_tool.exe (bin\\memory_dump_tool.exe)", + "name": "Tools - EE Memory Analyze - Jak 2", + "args": ["\"${workspaceRoot}/eeMemory.bin\"", "--output-path", "\"${workspaceRoot}\"", "--game", "jak2"] }, { "type": "default", diff --git a/LICENSE b/LICENSE index 04b75384c..ebec2b5ee 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ ISC License -Copyright (c) 2020-2021 OpenGOAL Team +Copyright (c) 2020-2022 OpenGOAL Team Permission to use, copy, modify, and/or distribute this software for any purpose with or without fee is hereby granted, provided that the above diff --git a/common/goal_constants.h b/common/goal_constants.h index 2f7a8ccde..99f9091c8 100644 --- a/common/goal_constants.h +++ b/common/goal_constants.h @@ -18,6 +18,9 @@ constexpr int ARRAY_DATA_OFFSET = 12; // not including type tag // constexpr s32 SYM_INFO_OFFSET = 8167 * 8 - 4; namespace jak1 { +constexpr s32 ORIGINAL_MAX_GOAL_SYMBOLS = 8192; +constexpr s32 ORIGINAL_SYM_TO_STRING_OFFSET = 0xff38; + constexpr s32 GOAL_MAX_SYMBOLS = 16384; constexpr s32 SYM_INFO_OFFSET = GOAL_MAX_SYMBOLS * 8 - 4; diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 900f43bd2..1dc90bd8b 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -9,6 +9,6 @@ add_executable(dgo_packer target_link_libraries(dgo_packer common) add_executable(memory_dump_tool - MemoryDumpTool/main.cpp) + memory_dump_tool/main.cpp) target_link_libraries(memory_dump_tool common decomp) diff --git a/tools/MemoryDumpTool/main.cpp b/tools/memory_dump_tool/main.cpp similarity index 70% rename from tools/MemoryDumpTool/main.cpp rename to tools/memory_dump_tool/main.cpp index 51c433f15..9ca51d9f4 100644 --- a/tools/MemoryDumpTool/main.cpp +++ b/tools/memory_dump_tool/main.cpp @@ -4,6 +4,7 @@ #include #include "common/goal_constants.h" +#include "common/log/log.h" #include "common/symbols.h" #include "common/type_system/TypeSystem.h" #include "common/util/Assert.h" @@ -12,13 +13,10 @@ #include "decompiler/util/DecompilerTypeSystem.h" +#include "third-party/CLI11.hpp" #include "third-party/fmt/core.h" #include "third-party/json.hpp" -namespace fs = fs; - -constexpr GameVersion kGameVersion = GameVersion::Jak1; - struct Ram { const u8* data = nullptr; u32 size = 0; @@ -79,32 +77,62 @@ struct Ram { bool word_in_memory(u32 addr) const { return in_memory(addr); } }; -u32 scan_for_symbol_table(const Ram& ram, u32 start_addr, u32 end_addr) { +u32 scan_for_symbol_table(const Ram& ram, + const GameVersion& game_version, + u32 start_addr, + u32 end_addr) { fmt::print("scanning for symbol table in 0x{:x} - 0x{:x}\n", start_addr, end_addr); std::vector candidates; // look for the false symbol. - for (u32 addr = (start_addr & 0xfffffff0); addr < end_addr; addr += 8) { - if (ram.word(addr + 4) == addr + 4) { - candidates.push_back(addr); + if (game_version == GameVersion::Jak1) { + for (u32 addr = (start_addr & 0xfffffff0); addr < end_addr; addr += 8) { + if (ram.word(addr + 4) == addr + 4) { + candidates.push_back(addr); + lg::info("candidate 0x{:x}", addr); + } + } + } else { + for (u32 addr = (start_addr & 0xfffffff0); addr < end_addr; addr += 4) { + if (ram.word(addr) == addr + 1) { + candidates.push_back(addr + 1); + lg::info("candidate 0x{:x}", addr + 1); + } } } fmt::print("got {} candidates for #f:\n", candidates.size()); - for (auto addr : candidates) { - // todo: this is wrong - auto str = addr + BASIC_OFFSET + jak1::SYM_INFO_OFFSET; - fmt::print(" trying 0x{:x}:\n", addr); - if (ram.word_in_memory(str)) { - auto mem = ram.word(str + 4); // offset of str in SymInfo - auto name = ram.try_string(mem + 4); // offset of data in GOAL string - if (name) { - fmt::print(" name: {}\n", *name); + if (game_version == GameVersion::Jak1) { + for (auto addr : candidates) { + auto str = addr + jak1::ORIGINAL_SYM_TO_STRING_OFFSET; + fmt::print(" trying 0x{:x}:\n", addr); + if (ram.word_in_memory(str)) { + auto mem = ram.word(str + 4); // offset of str in SymInfo + auto name = ram.try_string(mem + 4); // offset of data in GOAL string + if (name) { + fmt::print(" name: {}\n", *name); + } + if (name == "#f") { + fmt::print("Got #f = 0x{:x}!\n", addr + 4); + return addr + 4; + } } - if (name == "#f") { - fmt::print("Got #f = 0x{:x}!\n", addr + 4); - return addr + 4; + } + } else { + for (auto addr : candidates) { + auto str = addr + jak2::SYM_TO_STRING_OFFSET; + fmt::print(" trying 0x{:x}:\n", addr); + if (ram.word_in_memory(str)) { + auto mem = ram.word(str); // offset of str in SymInfo + auto name = ram.try_string(mem + 4); // offset of data in GOAL string + if (name) { + fmt::print(" name: {}\n", *name); + } + if (name == "#f") { + fmt::print("Got #f = 0x{:x}!\n", addr + 4); + return addr; + } } } } @@ -118,68 +146,94 @@ struct SymbolMap { std::unordered_map addr_to_name; }; -SymbolMap build_symbol_map(const Ram& ram, u32 s7) { - // TODO jak 1 specific - fmt::print("finding symbols...\n"); +SymbolMap build_symbol_map(const GameVersion& game_version, const Ram& ram, u32 s7) { + lg::info("building symbol map..."); SymbolMap map; - /* - s7 = symbol_table + (GOAL_MAX_SYMBOLS / 2) * 8 + BASIC_OFFSET; - // pointer to the first symbol (SymbolTable2 is the "lower" symbol table) - SymbolTable2 = symbol_table + BASIC_OFFSET; - // the last symbol we will ever access. - LastSymbol = symbol_table + 0xff00; - */ - // todo wrong - auto symbol_table = s7 - ((jak1::GOAL_MAX_SYMBOLS / 2) * 8 + BASIC_OFFSET); - auto SymbolTable2 = symbol_table + BASIC_OFFSET; - auto LastSymbol = symbol_table + 0xff00; + if (game_version == GameVersion::Jak1) { + auto addr_start_of_sym_table = s7 - ((jak1::ORIGINAL_MAX_GOAL_SYMBOLS / 2) * 8); + auto addr_last_symbol = addr_start_of_sym_table + 0xff00; - for (u32 sym = SymbolTable2; sym < LastSymbol; sym += 8) { - auto info = sym + jak1::SYM_INFO_OFFSET; // already has basic offset - auto str = ram.word(info + 4); - if (str) { - auto name = ram.string(str + 4); - if (name != "asize-of-basic-func") { - ASSERT(map.name_to_addr.find(name) == map.name_to_addr.end()); - map.name_to_addr[name] = sym; - map.addr_to_name[sym] = name; - map.name_to_value[name] = ram.word(sym); + for (u32 sym = addr_start_of_sym_table; sym < addr_last_symbol; sym += 8) { + auto info = sym + jak1::ORIGINAL_SYM_TO_STRING_OFFSET; // already has basic offset + auto str = ram.word(info); + if (str) { + auto name = ram.string(str + 4); + if (name != "asize-of-basic-func") { + ASSERT(map.name_to_addr.find(name) == map.name_to_addr.end()); + map.name_to_addr[name] = sym; + map.addr_to_name[sym] = name; + map.name_to_value[name] = ram.word(sym); + } + } + } + } else { + auto addr_start_of_sym_table = s7 - ((jak2::GOAL_MAX_SYMBOLS / 2) * 4) + BASIC_OFFSET; + auto addr_last_symbol = addr_start_of_sym_table + 0xff00; // ? the same ? + + for (u32 sym = addr_start_of_sym_table; sym < addr_last_symbol; sym += 4) { + auto info = sym + jak2::SYM_TO_STRING_OFFSET; + auto str = ram.word(info); + if (str) { + auto name = ram.string(str + 4); + if (name != "asize-of-basic-func") { + ASSERT(map.name_to_addr.find(name) == map.name_to_addr.end()); + map.name_to_addr[name] = sym; + map.addr_to_name[sym] = name; + map.name_to_value[name] = ram.word(sym); + } } } } ASSERT(map.name_to_addr.size() == map.addr_to_name.size()); - fmt::print("found {} symbols.\n", map.name_to_addr.size()); + lg::info("found {} symbols", map.name_to_addr.size()); return map; } std::unordered_map build_type_map(const Ram& ram, const SymbolMap& symbols, + const GameVersion& game_version, u32 s7) { - // TODO jak 1 specific std::unordered_map result; - fmt::print("finding types...\n"); - u32 type_of_type = ram.word(s7 + jak1_symbols::FIX_SYM_TYPE_TYPE); - ASSERT(type_of_type == ram.word(symbols.name_to_addr.at("type"))); + lg::info("finding types..."); + if (game_version == GameVersion::Jak1) { + u32 type_of_type = ram.word(s7 + jak1_symbols::FIX_SYM_TYPE_TYPE); + ASSERT(type_of_type == ram.word(symbols.name_to_addr.at("type"))); - for (const auto& [name, addr] : symbols.name_to_addr) { - u32 value = ram.word(addr); - if (ram.word_in_memory(value - 4) && ((value & 0x7) == BASIC_OFFSET)) { - if (ram.word(value - 4) == type_of_type) { - result[value] = name; + for (const auto& [name, addr] : symbols.name_to_addr) { + u32 value = ram.word(addr); + if (ram.word_in_memory(value - 4) && ((value & 0x7) == BASIC_OFFSET)) { + if (ram.word(value - 4) == type_of_type) { + result[value] = name; + } + } + } + } else { + u32 type_of_type = ram.word(s7 + jak2_symbols::FIX_SYM_TYPE_TYPE - 1); + ASSERT(type_of_type == ram.word(symbols.name_to_addr.at("type") - 1)); + + for (const auto& [name, addr] : symbols.name_to_addr) { + u32 value = ram.word(addr - 1); + if (ram.word_in_memory(value - 4) && ((value & 0x7) == BASIC_OFFSET)) { + if (ram.word(value - 4) == type_of_type) { + result[value] = name; + } } } } - fmt::print("found {} types\n", result.size()); + lg::info("found {} types", result.size()); return result; } +const std::vector ignored_types = {"symbol", "string", "function", "object", + "integer"}; + std::unordered_map> find_basics( const Ram& ram, const std::unordered_map& type_map) { - fmt::print("Scanning memory for objects. This may take a while...\n"); + lg::info("Scanning memory for objects. This may take a while..."); std::unordered_map> result; int total_objects = 0; @@ -188,14 +242,14 @@ std::unordered_map> find_basics( u32 tag = ram.word(addr); auto iter = type_map.find(tag); // ignore the stupid types. - if (iter != type_map.end() && iter->second != "symbol" && iter->second != "string" && - iter->second != "function" && iter->second != "object" && iter->second != "integer") { + if (iter != type_map.end() && std::find(ignored_types.begin(), ignored_types.end(), + iter->second) == ignored_types.end()) { result[iter->second].push_back(addr); total_objects++; } } - fmt::print("Got {} objects of {} unique types\n", total_objects, result.size()); + lg::info("Got {} objects of {} unique types\n", total_objects, result.size()); return result; } @@ -563,66 +617,92 @@ int main(int argc, char** argv) { exit(EXIT_FAILURE); } - fmt::print("MemoryDumpTool\n"); + fs::path dump_path; + fs::path output_path; + std::string game_name = "jak1"; - if (argc != 2 && argc != 3) { - fmt::print("usage: memory_dump_tool [output folder]\n"); + lg::initialize(); + + CLI::App app{"OpenGOAL Memory Dump Analyzer"}; + app.add_option("dump-path", dump_path, "The path to the dump file to analyze")->required(); + app.add_option("--output-path", output_path, + "Where the output files should be sent, defaults to current directory otherwise"); + app.add_option("-g,--game", game_name, "Specify the game name, defaults to 'jak1'"); + app.validate_positionals(); + CLI11_PARSE(app, argc, argv); + + auto ok = file_util::setup_project_path({}); + if (!ok) { + lg::error("couldn't setup project path, exiting"); + return 1; + } + lg::info("Loading type definitions from all-types.gc..."); + + auto game_version = game_name_to_version(game_name); + + decompiler::DecompilerTypeSystem dts(game_version); + + // TODO - this could be better + if (game_version == GameVersion::Jak1) { + dts.parse_type_defs({"decompiler", "config", "all-types.gc"}); + } else if (game_version == GameVersion::Jak2) { + dts.parse_type_defs({"decompiler", "config", "jak2", "all-types.gc"}); + } else { + lg::error("unsupported game version"); return 1; } - fmt::print("Loading type definitions from all-types.gc...\n"); - decompiler::DecompilerTypeSystem dts(kGameVersion); - dts.parse_type_defs({"decompiler", "config", "all-types.gc"}); - - std::string file_name = argv[1]; fs::path output_folder; - if (!fs::exists(output_folder) || argc < 3) { - fmt::print("Output folder not found, defaulting to current directory"); - output_folder = "."; - } else { - output_folder = argv[2]; + if (output_folder.empty() || !fs::exists(output_folder)) { + lg::warn("Output folder not found or not provided, defaulting to current directory"); + output_folder = "./"; } - if (ends_with(file_name, "p2s")) { - fmt::print("PS2 savestates are not directly supported. Please extract contents beforehand.\n"); + if (dump_path.extension() == "p2s") { + lg::error("PCSX2 savestates are not directly supported. Please extract contents beforehand"); return 1; } - fmt::print("Loading memory from '{}'\n", file_name); - auto data = file_util::read_binary_file(file_name); + lg::info("Loading memory from '{}'", dump_path.string()); + auto data = file_util::read_binary_file(dump_path); u32 one_mb = (1 << 20); if (data.size() == 32 * one_mb) { - fmt::print("Got 32MB file\n"); + lg::info("Got 32MB file"); } else if (data.size() == 128 * one_mb) { - fmt::print("Got 128MB file\n"); + lg::info("Got 128MB file"); } else if (data.size() == 127 * one_mb) { - fmt::print("Got a 127MB file. Assuming this is a dump with the first 1 MB missing.\n"); + lg::warn("Got a 127MB file. Assuming this is a dump with the first 1 MB missing.\n"); data.insert(data.begin(), one_mb, 0); - ASSERT(data.size() == 128 * one_mb); + if (data.size() != 128 * one_mb) { + lg::error("it was not!"); + return 1; + } } else { - fmt::print("Invalid size: {} bytes\n", data.size()); + lg::error("Invalid size: {} bytes", data.size()); + return 1; } Ram ram(data.data(), data.size()); - u32 s7 = scan_for_symbol_table(ram, one_mb, 2 * one_mb); + u32 s7 = scan_for_symbol_table(ram, game_version, one_mb, 2 * one_mb); if (!s7) { - fmt::print("Failed to find symbol table\n"); + lg::error("Failed to find symbol table"); return 1; } nlohmann::json results; - if (fs::exists(output_folder / "ee-results.json")) { - fmt::print("Found existing result file, appending results to it!\n"); - std::ifstream i(output_folder / "ee-results.json"); + auto json_path = fmt::format("ee-results-{}.json", game_name); + if (fs::exists(output_folder / json_path)) { + lg::info("Found existing result file, appending results to it!"); + std::ifstream i(output_folder / json_path); i >> results; } - auto symbol_map = build_symbol_map(ram, s7); - auto types = build_type_map(ram, symbol_map, s7); + auto symbol_map = build_symbol_map(game_version, ram, s7); + auto types = build_type_map(ram, symbol_map, game_version, s7); auto basics = find_basics(ram, types); follow_references_to_find_pointers(ram, dts.ts, basics, s7 + 0x100); @@ -631,10 +711,10 @@ int main(int argc, char** argv) { inspect_symbols(ram, types, symbol_map); inspect_process_self(ram, basics, types, dts.ts); - if (fs::exists(output_folder / "ee-results.json")) { - fs::remove(output_folder / "ee-results.json"); + if (fs::exists(output_folder / json_path)) { + fs::remove(output_folder / json_path); } - std::ofstream o(output_folder / "ee-results.json"); + std::ofstream o(output_folder / json_path); o << std::setw(2) << results << std::endl; return 0;