mirror of
https://github.com/open-goal/jak-project.git
synced 2024-10-19 14:47:49 -04:00
decompiler: Cleanup duplication in extractor/decompiler and make it easier to enable streamed audio ripping from CLI (#3560)
Some checks failed
Build / 🖥️ Windows (push) Has been cancelled
Build / 🐧 Linux (push) Has been cancelled
Build / 🍎 MacOS (push) Has been cancelled
Lint / 📝 Formatting (push) Has been cancelled
Lint / 📝 Required Checks (push) Has been cancelled
Lint / 📝 Optional Checks (push) Has been cancelled
Some checks failed
Build / 🖥️ Windows (push) Has been cancelled
Build / 🐧 Linux (push) Has been cancelled
Build / 🍎 MacOS (push) Has been cancelled
Lint / 📝 Formatting (push) Has been cancelled
Lint / 📝 Required Checks (push) Has been cancelled
Lint / 📝 Optional Checks (push) Has been cancelled
This centralizes the code that both `extractor` and the decompiler executes. In the past this code was partially-duplicated, meaning that the `extractor` could only do _some_ operations and not others (ie. could not extract the audio files). I also simplified the process to enable audio streaming in the configuration. This is to support a new feature in the launcher that allows you to enable these options for the decompiler: ![image](https://github.com/open-goal/jak-project/assets/13153231/8e6c20a1-8b5b-46f0-bceb-7644f713989f)
This commit is contained in:
parent
a485c236d6
commit
b4113dda67
|
@ -3,20 +3,29 @@
|
|||
#include "common/common_types.h"
|
||||
#include "common/log/log.h"
|
||||
|
||||
#ifdef __linux__
|
||||
|
||||
#ifdef _WIN32
|
||||
// clang-format off
|
||||
#define NOMINMAX
|
||||
#define WIN32_LEAN_AND_MEAN
|
||||
#include <Windows.h>
|
||||
#include <psapi.h>
|
||||
// clang-format on
|
||||
size_t get_peak_rss() {
|
||||
HANDLE hProcess = GetCurrentProcess();
|
||||
PROCESS_MEMORY_COUNTERS pmc;
|
||||
if (GetProcessMemoryInfo(hProcess, &pmc, sizeof(pmc))) {
|
||||
return pmc.PeakWorkingSetSize;
|
||||
} else {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
#else
|
||||
#include <sys/resource.h>
|
||||
|
||||
size_t get_peak_rss() {
|
||||
rusage x;
|
||||
getrusage(RUSAGE_SELF, &x);
|
||||
return x.ru_maxrss * 1024;
|
||||
}
|
||||
|
||||
#else
|
||||
size_t get_peak_rss() {
|
||||
return 0;
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifdef _WIN32
|
||||
|
|
|
@ -3,7 +3,6 @@
|
|||
#include <cstddef>
|
||||
#include <string>
|
||||
|
||||
// Note: these are not implemented on windows and will return zero.
|
||||
size_t get_peak_rss();
|
||||
void setup_cpu_info();
|
||||
|
||||
|
|
|
@ -89,7 +89,8 @@ add_library(
|
|||
VuDisasm/VuDisassembler.cpp
|
||||
VuDisasm/VuInstruction.cpp
|
||||
|
||||
config.cpp)
|
||||
config.cpp
|
||||
decompilation_process.cpp)
|
||||
|
||||
target_link_libraries(decomp
|
||||
lzokay
|
||||
|
|
|
@ -64,7 +64,6 @@ Config make_config_via_json(nlohmann::json& json) {
|
|||
inputs_json.at("str_art_file_names").get<std::vector<std::string>>();
|
||||
}
|
||||
|
||||
config.audio_dir_file_name = inputs_json.at("audio_dir_file_name").get<std::string>();
|
||||
config.streamed_audio_file_names =
|
||||
inputs_json.at("streamed_audio_file_names").get<std::vector<std::string>>();
|
||||
|
||||
|
@ -321,6 +320,9 @@ Config make_config_via_json(nlohmann::json& json) {
|
|||
if (json.contains("save_texture_pngs")) {
|
||||
config.save_texture_pngs = json.at("save_texture_pngs").get<bool>();
|
||||
}
|
||||
if (json.contains("rip_streamed_audio")) {
|
||||
config.rip_streamed_audio = json.at("rip_streamed_audio").get<bool>();
|
||||
}
|
||||
|
||||
if (inputs_json.contains("animated_textures")) {
|
||||
config.animated_textures =
|
||||
|
|
|
@ -102,8 +102,6 @@ struct Config {
|
|||
std::vector<std::string> str_file_names;
|
||||
std::vector<std::string> str_texture_file_names;
|
||||
std::vector<std::string> str_art_file_names;
|
||||
|
||||
std::string audio_dir_file_name;
|
||||
std::vector<std::string> streamed_audio_file_names;
|
||||
|
||||
std::string obj_file_name_map_file;
|
||||
|
@ -175,6 +173,7 @@ struct Config {
|
|||
std::vector<std::string> levels_to_extract;
|
||||
bool levels_extract;
|
||||
bool save_texture_pngs = false;
|
||||
bool rip_streamed_audio = false;
|
||||
|
||||
DecompileHacks hacks;
|
||||
|
||||
|
|
|
@ -122,6 +122,9 @@
|
|||
// save game textures as .png files to decompiler_out/<game>/textures
|
||||
"save_texture_pngs": false,
|
||||
|
||||
// whether or not to dump out streamed audio files to decompiler_out/<game>/audio
|
||||
"rip_streamed_audio": false,
|
||||
|
||||
////////////////////////////
|
||||
// PATCHING OPTIONS
|
||||
////////////////////////////
|
||||
|
|
|
@ -258,10 +258,6 @@
|
|||
"TEXT/6COMMON.TXT"
|
||||
],
|
||||
|
||||
// uncomment the next line to extract audio to wave files.
|
||||
//"audio_dir_file_name": "jak1/VAG",
|
||||
"audio_dir_file_name": "",
|
||||
|
||||
"streamed_audio_file_names": [
|
||||
"VAGWAD.ENG",
|
||||
"VAGWAD.FRE",
|
||||
|
|
|
@ -228,9 +228,5 @@
|
|||
"TEXT/6COMMON.TXT"
|
||||
],
|
||||
|
||||
// uncomment the next line to extract audio to wave files.
|
||||
//"audio_dir_file_name": "jak1/VAG",
|
||||
"audio_dir_file_name": "",
|
||||
|
||||
"streamed_audio_file_names": ["VAGWAD.ENG", "VAGWAD.JAP"]
|
||||
}
|
||||
|
|
|
@ -127,6 +127,9 @@
|
|||
"rip_collision": false,
|
||||
// save game textures as .png files to decompiler_out/<game>/textures
|
||||
"save_texture_pngs": false,
|
||||
|
||||
// whether or not to dump out streamed audio files to decompiler_out/<game>/audio
|
||||
"rip_streamed_audio": false,
|
||||
|
||||
////////////////////////////
|
||||
// PATCHING OPTIONS
|
||||
|
|
|
@ -441,10 +441,6 @@
|
|||
"TEXT/7COMMON.TXT"
|
||||
],
|
||||
|
||||
// uncomment the next line to extract audio to wave files.
|
||||
// "audio_dir_file_name": "jak2/VAG",
|
||||
"audio_dir_file_name": "",
|
||||
|
||||
"streamed_audio_file_names": [
|
||||
"VAGWAD.ENG",
|
||||
"VAGWAD.FRE",
|
||||
|
|
|
@ -127,6 +127,9 @@
|
|||
// save game textures as .png files to decompiler_out/<game>/textures
|
||||
"save_texture_pngs": false,
|
||||
|
||||
// whether or not to dump out streamed audio files to decompiler_out/<game>/audio
|
||||
"rip_streamed_audio": false,
|
||||
|
||||
////////////////////////////
|
||||
// PATCHING OPTIONS
|
||||
////////////////////////////
|
||||
|
|
|
@ -333,10 +333,6 @@
|
|||
"TEXT/11COMMON.TXT"
|
||||
],
|
||||
|
||||
// uncomment the next line to extract audio to wave files.
|
||||
// "audio_dir_file_name": "jak3/VAG",
|
||||
"audio_dir_file_name": "",
|
||||
|
||||
"streamed_audio_file_names": [
|
||||
"VAGWAD.ENG",
|
||||
"VAGWAD.FRE",
|
||||
|
|
284
decompiler/decompilation_process.cpp
Normal file
284
decompiler/decompilation_process.cpp
Normal file
|
@ -0,0 +1,284 @@
|
|||
#include "decompilation_process.h"
|
||||
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "config.h"
|
||||
|
||||
#include "common/log/log.h"
|
||||
#include "common/util/Timer.h"
|
||||
#include "common/util/os.h"
|
||||
|
||||
#include "Disasm/OpcodeInfo.h"
|
||||
#include "ObjectFile/ObjectFileDB.h"
|
||||
#include "data/streamed_audio.h"
|
||||
#include "level_extractor/extract_level.h"
|
||||
|
||||
int run_decompilation_process(decompiler::Config config,
|
||||
const fs::path& in_folder,
|
||||
const fs::path& out_folder,
|
||||
const bool minimal_for_extractor) {
|
||||
using namespace decompiler;
|
||||
Timer decomp_timer;
|
||||
|
||||
lg::info("[Mem] Top of main: {} MB\n", get_peak_rss() / (1024 * 1024));
|
||||
|
||||
init_opcode_info();
|
||||
|
||||
lg::info("[Mem] After init: {} MB\n", get_peak_rss() / (1024 * 1024));
|
||||
|
||||
std::vector<fs::path> dgos, objs, strs, tex_strs, art_strs;
|
||||
if (minimal_for_extractor) {
|
||||
// TODO - does this even matter, or can we just make the DGOs lazily loaded (does it already
|
||||
// happen?)
|
||||
for (const auto& dgo_name : config.dgo_names) {
|
||||
std::string common_name = "GAME.CGO";
|
||||
if (dgo_name.length() > 3 && dgo_name.substr(dgo_name.length() - 3) == "DGO") {
|
||||
// ends in DGO, it's a level
|
||||
dgos.push_back(in_folder / dgo_name);
|
||||
} else if (dgo_name.length() >= common_name.length() &&
|
||||
dgo_name.substr(dgo_name.length() - common_name.length()) == common_name) {
|
||||
// it's COMMON.CGO, we need that too.
|
||||
dgos.push_back(in_folder / dgo_name);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
for (const auto& dgo_name : config.dgo_names) {
|
||||
dgos.push_back(in_folder / dgo_name);
|
||||
}
|
||||
}
|
||||
|
||||
if (minimal_for_extractor) {
|
||||
// TODO - does this even matter, or can we just make the DGOs lazily loaded (does it already
|
||||
// happen?)
|
||||
for (const auto& obj_name : config.object_file_names) {
|
||||
if (obj_name.length() > 3 && obj_name.substr(obj_name.length() - 3) == "TXT") {
|
||||
// ends in TXT
|
||||
objs.push_back(in_folder / obj_name);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
for (const auto& obj_name : config.object_file_names) {
|
||||
objs.push_back(in_folder / obj_name);
|
||||
}
|
||||
}
|
||||
|
||||
if (!minimal_for_extractor) {
|
||||
for (const auto& str_name : config.str_file_names) {
|
||||
strs.push_back(in_folder / str_name);
|
||||
}
|
||||
}
|
||||
|
||||
for (const auto& str_name : config.str_texture_file_names) {
|
||||
tex_strs.push_back(in_folder / str_name);
|
||||
}
|
||||
|
||||
for (const auto& str_name : config.str_art_file_names) {
|
||||
art_strs.push_back(in_folder / str_name);
|
||||
}
|
||||
|
||||
lg::info("[Mem] After config read: {} MB", get_peak_rss() / (1024 * 1024));
|
||||
|
||||
// build file database
|
||||
lg::info("Setting up object file DB...");
|
||||
ObjectFileDB db(dgos, fs::path(config.obj_file_name_map_file), objs, strs, tex_strs, art_strs,
|
||||
config);
|
||||
|
||||
// Explicitly fail if a file in the 'allowed_objects' list wasn't found in the DB
|
||||
// as this is another silent error that can be confusing
|
||||
if (!config.allowed_objects.empty()) {
|
||||
for (const auto& expected_obj : config.allowed_objects) {
|
||||
if (db.obj_files_by_name.count(expected_obj) == 0) {
|
||||
// TODO - this is wrong for jak1, fix eventually as this is now done in 3 places
|
||||
lg::error(
|
||||
"Expected to find '{}' in the ObjectFileDB but did not. Check "
|
||||
"./decompiler/config/{}/inputs.jsonc",
|
||||
expected_obj, config.game_name);
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
lg::info("[Mem] After DB setup: {} MB", get_peak_rss() / (1024 * 1024));
|
||||
|
||||
// write out DGO file info
|
||||
file_util::create_dir_if_needed(out_folder);
|
||||
file_util::write_text_file(out_folder / "dgo.txt", db.generate_dgo_listing());
|
||||
// write out object file map (used for future decompilations, if desired)
|
||||
file_util::write_text_file(out_folder / "obj.txt",
|
||||
db.generate_obj_listing(config.merged_objects));
|
||||
|
||||
// dump raw objs
|
||||
if (config.dump_objs) {
|
||||
auto path = out_folder / "raw_obj";
|
||||
file_util::create_dir_if_needed(path);
|
||||
db.dump_raw_objects(path);
|
||||
}
|
||||
|
||||
// process files (required for all analysis)
|
||||
db.process_link_data(config);
|
||||
lg::info("[Mem] After link data: {} MB", get_peak_rss() / (1024 * 1024));
|
||||
db.find_code(config);
|
||||
db.process_labels();
|
||||
lg::info("[Mem] After code: {} MB", get_peak_rss() / (1024 * 1024));
|
||||
|
||||
// top level decompile (do this before printing asm so we get function names)
|
||||
if (config.find_functions) {
|
||||
db.ir2_top_level_pass(config);
|
||||
}
|
||||
|
||||
// print disassembly
|
||||
if (config.disassemble_code || config.disassemble_data) {
|
||||
db.write_disassembly(out_folder, config.disassemble_data, config.disassemble_code,
|
||||
config.write_hex_near_instructions);
|
||||
}
|
||||
|
||||
if (config.process_art_groups) {
|
||||
db.extract_art_info();
|
||||
// dump art info to json if requested
|
||||
if (config.dump_art_group_info) {
|
||||
auto ag_file_name = out_folder / "dump" / "art-group-info.min.json";
|
||||
nlohmann::json ag_json = db.dts.art_group_info;
|
||||
file_util::create_dir_if_needed_for_file(ag_file_name);
|
||||
file_util::write_text_file(ag_file_name, ag_json.dump(-1));
|
||||
lg::info("[DUMP] Dumped art group info to {}", ag_file_name.string());
|
||||
}
|
||||
if (config.dump_joint_geo_info) {
|
||||
auto jg_file_name = out_folder / "dump" / "joint-node-info.min.json";
|
||||
nlohmann::json jg_json = db.dts.jg_info;
|
||||
file_util::create_dir_if_needed_for_file(jg_file_name);
|
||||
file_util::write_text_file(jg_file_name, jg_json.dump(-1));
|
||||
lg::info("[DUMP] Dumped joint node info to {}", jg_file_name.string());
|
||||
}
|
||||
} else if (!config.art_group_info_dump.empty() || !config.jg_info_dump.empty()) {
|
||||
// process art groups (used in decompilation)
|
||||
// - if the config has a path to the art info dump, just use that
|
||||
// - otherwise (or if we want to dump it fresh) extract it
|
||||
if (!config.art_group_info_dump.empty()) {
|
||||
db.dts.art_group_info = config.art_group_info_dump;
|
||||
}
|
||||
if (!config.jg_info_dump.empty()) {
|
||||
db.dts.jg_info = config.jg_info_dump;
|
||||
}
|
||||
} else {
|
||||
lg::error("`process_art_groups` was false and no art-group-info dump was provided!");
|
||||
return 1;
|
||||
}
|
||||
|
||||
if (config.process_tpages && !config.texture_info_dump.empty()) {
|
||||
db.dts.textures = config.texture_info_dump;
|
||||
}
|
||||
|
||||
// main decompile.
|
||||
if (config.decompile_code) {
|
||||
db.analyze_functions_ir2(out_folder, config, {}, {}, {});
|
||||
}
|
||||
|
||||
if (config.generate_all_types) {
|
||||
ASSERT_MSG(config.decompile_code, "Must decompile code to generate all-types");
|
||||
db.ir2_analyze_all_types(out_folder / "new-all-types.gc", config.old_all_types_file,
|
||||
config.hacks.types_with_bad_inspect_methods);
|
||||
}
|
||||
|
||||
lg::info("[Mem] After decomp: {} MB", get_peak_rss() / (1024 * 1024));
|
||||
|
||||
// write out all symbols
|
||||
file_util::write_text_file(out_folder / "all-syms.gc", db.dts.dump_symbol_types());
|
||||
|
||||
// write art groups
|
||||
if (config.process_art_groups) {
|
||||
db.dump_art_info(out_folder);
|
||||
}
|
||||
|
||||
if (config.hexdump_code || config.hexdump_data) {
|
||||
db.write_object_file_words(out_folder, config.hexdump_data, config.hexdump_code);
|
||||
}
|
||||
|
||||
// data stuff
|
||||
if (config.write_scripts) {
|
||||
db.find_and_write_scripts(out_folder);
|
||||
}
|
||||
|
||||
// ensure asset dir exists
|
||||
file_util::create_dir_if_needed(out_folder / "assets");
|
||||
|
||||
if (config.process_game_text) {
|
||||
auto result = db.process_game_text_files(config);
|
||||
if (!result.empty()) {
|
||||
file_util::write_text_file(out_folder / "assets" / "game_text.txt", result);
|
||||
}
|
||||
}
|
||||
|
||||
lg::info("[Mem] After text: {} MB", get_peak_rss() / (1024 * 1024));
|
||||
|
||||
if (config.process_subtitle_text || config.process_subtitle_images) {
|
||||
auto result = db.process_all_spool_subtitles(
|
||||
config, config.process_subtitle_images ? out_folder / "assets" / "subtitle-images" : "");
|
||||
if (!result.empty()) {
|
||||
file_util::write_text_file(out_folder / "assets" / "game_subs.txt", result);
|
||||
}
|
||||
}
|
||||
|
||||
lg::info("[Mem] After spool handling: {} MB", get_peak_rss() / (1024 * 1024));
|
||||
|
||||
TextureDB tex_db;
|
||||
if (config.process_tpages || config.levels_extract) {
|
||||
auto textures_out = out_folder / "textures";
|
||||
auto dump_out = out_folder / "import";
|
||||
file_util::create_dir_if_needed(textures_out);
|
||||
auto result = db.process_tpages(tex_db, textures_out, config, dump_out);
|
||||
if (!result.empty() && config.process_tpages) {
|
||||
file_util::write_text_file(textures_out / "tpage-dir.txt", result);
|
||||
file_util::write_text_file(textures_out / "tex-remap.txt",
|
||||
tex_db.generate_texture_dest_adjustment_table());
|
||||
}
|
||||
if (config.dump_tex_info) {
|
||||
auto texture_file_name = out_folder / "dump" / "tex-info.min.json";
|
||||
nlohmann::json texture_json = db.dts.textures;
|
||||
file_util::create_dir_if_needed_for_file(texture_file_name);
|
||||
file_util::write_text_file(texture_file_name, texture_json.dump(-1));
|
||||
lg::info("[DUMP] Dumped texture info to {}", texture_file_name.string());
|
||||
}
|
||||
}
|
||||
|
||||
lg::info("[Mem] After textures: {} MB", get_peak_rss() / (1024 * 1024));
|
||||
|
||||
// Merge textures before replacing them, in other words, replacements take priority
|
||||
auto texture_merge_path = file_util::get_jak_project_dir() / "game" / "assets" /
|
||||
game_version_names[config.game_version] / "texture_merges";
|
||||
if (fs::exists(texture_merge_path)) {
|
||||
tex_db.merge_textures(texture_merge_path);
|
||||
}
|
||||
|
||||
auto replacements_path = file_util::get_jak_project_dir() / "custom_assets" /
|
||||
game_version_names[config.game_version] / "texture_replacements";
|
||||
if (fs::exists(replacements_path)) {
|
||||
tex_db.replace_textures(replacements_path);
|
||||
}
|
||||
|
||||
if (config.process_game_count) {
|
||||
auto result = db.process_game_count_file();
|
||||
if (!result.empty()) {
|
||||
file_util::write_text_file(out_folder / "assets" / "game_count.txt", result);
|
||||
}
|
||||
}
|
||||
|
||||
if (config.levels_extract) {
|
||||
auto level_out_path =
|
||||
file_util::get_jak_project_dir() / "out" / game_version_names[config.game_version] / "fr3";
|
||||
file_util::create_dir_if_needed(level_out_path);
|
||||
extract_all_levels(db, tex_db, config.levels_to_extract, "GAME.CGO", config, level_out_path);
|
||||
}
|
||||
|
||||
lg::info("[Mem] After extraction: {} MB", get_peak_rss() / (1024 * 1024));
|
||||
|
||||
if (config.rip_streamed_audio) {
|
||||
auto streaming_audio_out = out_folder / "audio";
|
||||
file_util::create_dir_if_needed(streaming_audio_out);
|
||||
process_streamed_audio(config, streaming_audio_out, in_folder,
|
||||
config.streamed_audio_file_names);
|
||||
}
|
||||
|
||||
lg::info("Decompiler has finished successfully in {:.2f} seconds.", decomp_timer.getSeconds());
|
||||
return 0;
|
||||
}
|
8
decompiler/decompilation_process.h
Normal file
8
decompiler/decompilation_process.h
Normal file
|
@ -0,0 +1,8 @@
|
|||
#pragma once
|
||||
|
||||
#include "config.h"
|
||||
|
||||
int run_decompilation_process(decompiler::Config config,
|
||||
const fs::path& in_folder,
|
||||
const fs::path& out_folder,
|
||||
const bool minimal_for_extractor);
|
|
@ -5,10 +5,9 @@
|
|||
#include "common/util/term_util.h"
|
||||
#include "common/util/unicode_util.h"
|
||||
|
||||
#include "decompiler/ObjectFile/ObjectFileDB.h"
|
||||
#include "decompiler/config.h"
|
||||
#include "decompiler/decompilation_process.h"
|
||||
#include "decompiler/extractor/extractor_util.h"
|
||||
#include "decompiler/level_extractor/extract_level.h"
|
||||
#include "goalc/compiler/Compiler.h"
|
||||
|
||||
#include "third-party/CLI11.hpp"
|
||||
|
@ -102,116 +101,24 @@ std::tuple<std::optional<ISOMetadata>, ExtractorErrorCode> validate(
|
|||
};
|
||||
}
|
||||
|
||||
// TODO - remove code duplication, most of this is copying what happens in decompiler's `main.cpp`
|
||||
void decompile(const fs::path& iso_data_path,
|
||||
const std::string& data_subfolder,
|
||||
const std::string& config_override) {
|
||||
using namespace decompiler;
|
||||
|
||||
ExtractorErrorCode decompile(const fs::path& in_folder,
|
||||
const std::string& data_subfolder,
|
||||
const std::string& config_override) {
|
||||
// Determine which config to use from the database
|
||||
const auto version_info = get_version_info_or_default(iso_data_path);
|
||||
const auto version_info = get_version_info_or_default(in_folder);
|
||||
|
||||
Config config = read_config_file(file_util::get_jak_project_dir() / "decompiler" / "config" /
|
||||
version_info.game_name /
|
||||
fmt::format("{}_config.jsonc", version_info.game_name),
|
||||
version_info.decomp_config_version, config_override);
|
||||
decompiler::Config config = decompiler::read_config_file(
|
||||
file_util::get_jak_project_dir() / "decompiler" / "config" / version_info.game_name /
|
||||
fmt::format("{}_config.jsonc", version_info.game_name),
|
||||
version_info.decomp_config_version, config_override);
|
||||
|
||||
std::vector<fs::path> dgos, objs, tex_strs, art_strs;
|
||||
|
||||
// grab all DGOS we need (level + common)
|
||||
// TODO - Jak 2 - jak 1 specific code?
|
||||
for (const auto& dgo_name : config.dgo_names) {
|
||||
std::string common_name = "GAME.CGO";
|
||||
if (dgo_name.length() > 3 && dgo_name.substr(dgo_name.length() - 3) == "DGO") {
|
||||
// ends in DGO, it's a level
|
||||
dgos.push_back(iso_data_path / dgo_name);
|
||||
} else if (dgo_name.length() >= common_name.length() &&
|
||||
dgo_name.substr(dgo_name.length() - common_name.length()) == common_name) {
|
||||
// it's COMMON.CGO, we need that too.
|
||||
dgos.push_back(iso_data_path / dgo_name);
|
||||
}
|
||||
}
|
||||
|
||||
// grab all the object files we need (just text)
|
||||
for (const auto& obj_name : config.object_file_names) {
|
||||
if (obj_name.length() > 3 && obj_name.substr(obj_name.length() - 3) == "TXT") {
|
||||
// ends in TXT
|
||||
objs.push_back(iso_data_path / obj_name);
|
||||
}
|
||||
}
|
||||
|
||||
for (const auto& str_name : config.str_texture_file_names) {
|
||||
tex_strs.push_back(iso_data_path / str_name);
|
||||
}
|
||||
|
||||
for (const auto& str_name : config.str_art_file_names) {
|
||||
art_strs.push_back(iso_data_path / str_name);
|
||||
}
|
||||
|
||||
// set up objects
|
||||
ObjectFileDB db(dgos, fs::path(config.obj_file_name_map_file), objs, {}, tex_strs, art_strs,
|
||||
config);
|
||||
|
||||
// save object files
|
||||
auto out_folder = file_util::get_jak_project_dir() / "decompiler_out" / data_subfolder;
|
||||
auto raw_obj_folder = out_folder / "raw_obj";
|
||||
file_util::create_dir_if_needed(raw_obj_folder);
|
||||
db.dump_raw_objects(raw_obj_folder);
|
||||
|
||||
// analyze object file link data
|
||||
db.process_link_data(config);
|
||||
db.find_code(config);
|
||||
db.process_labels();
|
||||
|
||||
// ensure asset dir exists
|
||||
file_util::create_dir_if_needed(out_folder / "assets");
|
||||
|
||||
// text files
|
||||
{
|
||||
auto result = db.process_game_text_files(config);
|
||||
if (!result.empty()) {
|
||||
file_util::write_text_file(out_folder / "assets" / "game_text.txt", result);
|
||||
}
|
||||
}
|
||||
|
||||
// textures
|
||||
decompiler::TextureDB tex_db;
|
||||
auto textures_out = out_folder / "textures";
|
||||
auto dump_out = out_folder / "import";
|
||||
file_util::create_dir_if_needed(textures_out);
|
||||
file_util::write_text_file(textures_out / "tpage-dir.txt",
|
||||
db.process_tpages(tex_db, textures_out, config, dump_out));
|
||||
|
||||
// texture merges
|
||||
// TODO - put all this stuff in somewhere common
|
||||
auto texture_merge_path = file_util::get_jak_project_dir() / "game" / "assets" /
|
||||
game_version_names[config.game_version] / "texture_merges";
|
||||
if (fs::exists(texture_merge_path)) {
|
||||
tex_db.merge_textures(texture_merge_path);
|
||||
}
|
||||
|
||||
// texture replacements
|
||||
auto replacements_path = file_util::get_jak_project_dir() / "custom_assets" /
|
||||
game_version_names[config.game_version] / "texture_replacements";
|
||||
if (fs::exists(replacements_path)) {
|
||||
tex_db.replace_textures(replacements_path);
|
||||
}
|
||||
|
||||
// game count
|
||||
{
|
||||
auto result = db.process_game_count_file();
|
||||
if (!result.empty()) {
|
||||
file_util::write_text_file(out_folder / "assets" / "game_count.txt", result);
|
||||
}
|
||||
}
|
||||
|
||||
// levels
|
||||
{
|
||||
auto level_out_path =
|
||||
file_util::get_jak_project_dir() / "out" / game_version_names[config.game_version] / "fr3";
|
||||
file_util::create_dir_if_needed(level_out_path);
|
||||
extract_all_levels(db, tex_db, config.levels_to_extract, "GAME.CGO", config, level_out_path);
|
||||
const auto result = run_decompilation_process(config, in_folder, out_folder, true);
|
||||
if (result != 0) {
|
||||
return ExtractorErrorCode::DECOMPILATION_GENERIC_ERROR;
|
||||
}
|
||||
return ExtractorErrorCode::SUCCESS;
|
||||
}
|
||||
|
||||
const std::unordered_map<std::string, GameIsoFlags> game_iso_flag_names = {
|
||||
|
@ -315,7 +222,6 @@ int main(int argc, char** argv) {
|
|||
}
|
||||
|
||||
// - SETUP
|
||||
decompiler::init_opcode_info();
|
||||
if (!project_path_override.empty()) {
|
||||
if (!fs::exists(project_path_override)) {
|
||||
lg::error("Error: project path override '{}' does not exist", project_path_override.string());
|
||||
|
@ -426,7 +332,11 @@ int main(int argc, char** argv) {
|
|||
// Get hash and file count
|
||||
const auto [hash, file_count] = calculate_extraction_hash(iso_data_path);
|
||||
// Validate
|
||||
auto [version_info, code] = validate(iso_data_path, hash, file_count);
|
||||
auto [version_info, validate_code] = validate(iso_data_path, hash, file_count);
|
||||
if (validate_code == ExtractorErrorCode::VALIDATION_BAD_EXTRACTION ||
|
||||
(flag_fail_on_validation && validate_code != ExtractorErrorCode::SUCCESS)) {
|
||||
return static_cast<int>(validate_code);
|
||||
}
|
||||
}
|
||||
|
||||
// write out a json file with some metadata for the game
|
||||
|
@ -455,7 +365,10 @@ int main(int argc, char** argv) {
|
|||
|
||||
if (flag_decompile) {
|
||||
try {
|
||||
decompile(iso_data_path, data_subfolder, decomp_config_override);
|
||||
const auto status_code = decompile(iso_data_path, data_subfolder, decomp_config_override);
|
||||
if (status_code != ExtractorErrorCode::SUCCESS) {
|
||||
return static_cast<int>(status_code);
|
||||
}
|
||||
} catch (std::exception& e) {
|
||||
lg::error("Error during decompile: {}", e.what());
|
||||
return static_cast<int>(ExtractorErrorCode::DECOMPILATION_GENERIC_ERROR);
|
||||
|
@ -463,7 +376,10 @@ int main(int argc, char** argv) {
|
|||
}
|
||||
|
||||
if (flag_compile) {
|
||||
compile(iso_data_path, data_subfolder);
|
||||
const auto status_code = compile(iso_data_path, data_subfolder);
|
||||
if (status_code != ExtractorErrorCode::SUCCESS) {
|
||||
return static_cast<int>(status_code);
|
||||
}
|
||||
}
|
||||
|
||||
if (flag_play) {
|
||||
|
|
|
@ -1,33 +1,15 @@
|
|||
#include <cstdio>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "config.h"
|
||||
#include "decompilation_process.h"
|
||||
|
||||
#include "common/log/log.h"
|
||||
#include "common/util/FileUtil.h"
|
||||
#include "common/util/Timer.h"
|
||||
#include "common/util/diff.h"
|
||||
#include "common/util/os.h"
|
||||
#include "common/util/set_util.h"
|
||||
#include "common/util/term_util.h"
|
||||
#include "common/util/unicode_util.h"
|
||||
#include "common/versions/versions.h"
|
||||
|
||||
#include "ObjectFile/ObjectFileDB.h"
|
||||
#include "decompiler/data/TextureDB.h"
|
||||
#include "decompiler/data/streamed_audio.h"
|
||||
#include "decompiler/level_extractor/extract_level.h"
|
||||
|
||||
#include "third-party/CLI11.hpp"
|
||||
#include "third-party/json.hpp"
|
||||
|
||||
template <typename... Args>
|
||||
static void mem_log(const std::string& format, Args&&... args) {
|
||||
#ifndef _WIN32
|
||||
lg::info("[Mem] " + format, std::forward<Args>(args)...);
|
||||
#endif
|
||||
}
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
ArgumentGuard u8_guard(argc, argv);
|
||||
|
@ -81,11 +63,9 @@ int main(int argc, char** argv) {
|
|||
return 1;
|
||||
}
|
||||
|
||||
using namespace decompiler;
|
||||
|
||||
Config config;
|
||||
decompiler::Config config;
|
||||
try {
|
||||
config = read_config_file(config_path, config_game_version, config_override);
|
||||
config = decompiler::read_config_file(config_path, config_game_version, config_override);
|
||||
} catch (const std::exception& e) {
|
||||
lg::error("Failed to parse config: {}", e.what());
|
||||
return 1;
|
||||
|
@ -115,7 +95,7 @@ int main(int argc, char** argv) {
|
|||
|
||||
// Warning message if expected ELF isn't found, user could be using bad assets / didn't extract
|
||||
// the ISO properly
|
||||
if (!config.expected_elf_name.empty() && !exists(in_folder / config.expected_elf_name)) {
|
||||
if (!config.expected_elf_name.empty() && !fs::exists(in_folder / config.expected_elf_name)) {
|
||||
lg::error(
|
||||
"WARNING - '{}' does not contain the expected ELF file '{}'. Was the ISO extracted "
|
||||
"properly or is there a version mismatch?",
|
||||
|
@ -123,235 +103,5 @@ int main(int argc, char** argv) {
|
|||
}
|
||||
|
||||
// -- Begin the Decompilation!
|
||||
|
||||
Timer decomp_timer;
|
||||
|
||||
mem_log("Top of main: {} MB\n", get_peak_rss() / (1024 * 1024));
|
||||
|
||||
init_opcode_info();
|
||||
|
||||
mem_log("After init: {} MB\n", get_peak_rss() / (1024 * 1024));
|
||||
|
||||
std::vector<fs::path> dgos, objs, strs, tex_strs, art_strs;
|
||||
for (const auto& dgo_name : config.dgo_names) {
|
||||
dgos.push_back(in_folder / dgo_name);
|
||||
}
|
||||
|
||||
for (const auto& obj_name : config.object_file_names) {
|
||||
objs.push_back(in_folder / obj_name);
|
||||
}
|
||||
|
||||
for (const auto& str_name : config.str_file_names) {
|
||||
strs.push_back(in_folder / str_name);
|
||||
}
|
||||
|
||||
for (const auto& str_name : config.str_texture_file_names) {
|
||||
tex_strs.push_back(in_folder / str_name);
|
||||
}
|
||||
|
||||
for (const auto& str_name : config.str_art_file_names) {
|
||||
art_strs.push_back(in_folder / str_name);
|
||||
}
|
||||
|
||||
mem_log("After config read: {} MB", get_peak_rss() / (1024 * 1024));
|
||||
|
||||
// build file database
|
||||
lg::info("Setting up object file DB...");
|
||||
ObjectFileDB db(dgos, fs::path(config.obj_file_name_map_file), objs, strs, tex_strs, art_strs,
|
||||
config);
|
||||
|
||||
// Explicitly fail if a file in the 'allowed_objects' list wasn't found in the DB
|
||||
// as this is another silent error that can be confusing
|
||||
if (!config.allowed_objects.empty()) {
|
||||
for (const auto& expected_obj : config.allowed_objects) {
|
||||
if (db.obj_files_by_name.count(expected_obj) == 0) {
|
||||
// TODO - this is wrong for jak1, fix eventually as this is now done in 3 places
|
||||
lg::error(
|
||||
"Expected to find '{}' in the ObjectFileDB but did not. Check "
|
||||
"./decompiler/config/{}/inputs.jsonc",
|
||||
expected_obj, config.game_name);
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
mem_log("After DB setup: {} MB", get_peak_rss() / (1024 * 1024));
|
||||
|
||||
// write out DGO file info
|
||||
file_util::write_text_file(out_folder / "dgo.txt", db.generate_dgo_listing());
|
||||
// write out object file map (used for future decompilations, if desired)
|
||||
file_util::write_text_file(out_folder / "obj.txt",
|
||||
db.generate_obj_listing(config.merged_objects));
|
||||
|
||||
// dump raw objs
|
||||
if (config.dump_objs) {
|
||||
auto path = out_folder / "raw_obj";
|
||||
file_util::create_dir_if_needed(path);
|
||||
db.dump_raw_objects(path);
|
||||
}
|
||||
|
||||
// process files (required for all analysis)
|
||||
db.process_link_data(config);
|
||||
mem_log("After link data: {} MB", get_peak_rss() / (1024 * 1024));
|
||||
db.find_code(config);
|
||||
db.process_labels();
|
||||
mem_log("After code: {} MB", get_peak_rss() / (1024 * 1024));
|
||||
|
||||
// top level decompile (do this before printing asm so we get function names)
|
||||
if (config.find_functions) {
|
||||
db.ir2_top_level_pass(config);
|
||||
}
|
||||
|
||||
// print disassembly
|
||||
if (config.disassemble_code || config.disassemble_data) {
|
||||
db.write_disassembly(out_folder, config.disassemble_data, config.disassemble_code,
|
||||
config.write_hex_near_instructions);
|
||||
}
|
||||
|
||||
if (config.process_art_groups) {
|
||||
db.extract_art_info();
|
||||
// dump art info to json if requested
|
||||
if (config.dump_art_group_info) {
|
||||
auto ag_file_name = out_folder / "dump" / "art-group-info.min.json";
|
||||
nlohmann::json ag_json = db.dts.art_group_info;
|
||||
file_util::create_dir_if_needed_for_file(ag_file_name);
|
||||
file_util::write_text_file(ag_file_name, ag_json.dump(-1));
|
||||
lg::info("[DUMP] Dumped art group info to {}", ag_file_name.string());
|
||||
}
|
||||
if (config.dump_joint_geo_info) {
|
||||
auto jg_file_name = out_folder / "dump" / "joint-node-info.min.json";
|
||||
nlohmann::json jg_json = db.dts.jg_info;
|
||||
file_util::create_dir_if_needed_for_file(jg_file_name);
|
||||
file_util::write_text_file(jg_file_name, jg_json.dump(-1));
|
||||
lg::info("[DUMP] Dumped joint node info to {}", jg_file_name.string());
|
||||
}
|
||||
} else if (!config.art_group_info_dump.empty() || !config.jg_info_dump.empty()) {
|
||||
// process art groups (used in decompilation)
|
||||
// - if the config has a path to the art info dump, just use that
|
||||
// - otherwise (or if we want to dump it fresh) extract it
|
||||
if (!config.art_group_info_dump.empty()) {
|
||||
db.dts.art_group_info = config.art_group_info_dump;
|
||||
}
|
||||
if (!config.jg_info_dump.empty()) {
|
||||
db.dts.jg_info = config.jg_info_dump;
|
||||
}
|
||||
} else {
|
||||
lg::error("`process_art_groups` was false and no art-group-info dump was provided!");
|
||||
return 1;
|
||||
}
|
||||
|
||||
if (config.process_tpages && !config.texture_info_dump.empty()) {
|
||||
db.dts.textures = config.texture_info_dump;
|
||||
}
|
||||
|
||||
// main decompile.
|
||||
if (config.decompile_code) {
|
||||
db.analyze_functions_ir2(out_folder, config, {}, {}, {});
|
||||
}
|
||||
|
||||
if (config.generate_all_types) {
|
||||
ASSERT_MSG(config.decompile_code, "Must decompile code to generate all-types");
|
||||
db.ir2_analyze_all_types(out_folder / "new-all-types.gc", config.old_all_types_file,
|
||||
config.hacks.types_with_bad_inspect_methods);
|
||||
}
|
||||
|
||||
mem_log("After decomp: {} MB", get_peak_rss() / (1024 * 1024));
|
||||
|
||||
// write out all symbols
|
||||
file_util::write_text_file(out_folder / "all-syms.gc", db.dts.dump_symbol_types());
|
||||
|
||||
// write art groups
|
||||
if (config.process_art_groups) {
|
||||
db.dump_art_info(out_folder);
|
||||
}
|
||||
|
||||
if (config.hexdump_code || config.hexdump_data) {
|
||||
db.write_object_file_words(out_folder, config.hexdump_data, config.hexdump_code);
|
||||
}
|
||||
|
||||
// data stuff
|
||||
if (config.write_scripts) {
|
||||
db.find_and_write_scripts(out_folder);
|
||||
}
|
||||
|
||||
if (config.process_game_text) {
|
||||
auto result = db.process_game_text_files(config);
|
||||
if (!result.empty()) {
|
||||
file_util::write_text_file(out_folder / "assets" / "game_text.txt", result);
|
||||
}
|
||||
}
|
||||
|
||||
mem_log("After text: {} MB", get_peak_rss() / (1024 * 1024));
|
||||
|
||||
if (config.process_subtitle_text || config.process_subtitle_images) {
|
||||
auto result = db.process_all_spool_subtitles(
|
||||
config, config.process_subtitle_images ? out_folder / "assets" / "subtitle-images" : "");
|
||||
if (!result.empty()) {
|
||||
file_util::write_text_file(out_folder / "assets" / "game_subs.txt", result);
|
||||
}
|
||||
}
|
||||
|
||||
mem_log("After spool handling: {} MB", get_peak_rss() / (1024 * 1024));
|
||||
|
||||
TextureDB tex_db;
|
||||
if (config.process_tpages || config.levels_extract) {
|
||||
auto textures_out = out_folder / "textures";
|
||||
auto dump_out = out_folder / "import";
|
||||
file_util::create_dir_if_needed(textures_out);
|
||||
auto result = db.process_tpages(tex_db, textures_out, config, dump_out);
|
||||
if (!result.empty() && config.process_tpages) {
|
||||
file_util::write_text_file(textures_out / "tpage-dir.txt", result);
|
||||
file_util::write_text_file(textures_out / "tex-remap.txt",
|
||||
tex_db.generate_texture_dest_adjustment_table());
|
||||
}
|
||||
if (config.dump_tex_info) {
|
||||
auto texture_file_name = out_folder / "dump" / "tex-info.min.json";
|
||||
nlohmann::json texture_json = db.dts.textures;
|
||||
file_util::create_dir_if_needed_for_file(texture_file_name);
|
||||
file_util::write_text_file(texture_file_name, texture_json.dump(-1));
|
||||
lg::info("[DUMP] Dumped texture info to {}", texture_file_name.string());
|
||||
}
|
||||
}
|
||||
|
||||
mem_log("After textures: {} MB", get_peak_rss() / (1024 * 1024));
|
||||
|
||||
// Merge textures before replacing them, in other words, replacements take priority
|
||||
auto texture_merge_path = file_util::get_jak_project_dir() / "game" / "assets" /
|
||||
game_version_names[config.game_version] / "texture_merges";
|
||||
if (fs::exists(texture_merge_path)) {
|
||||
tex_db.merge_textures(texture_merge_path);
|
||||
}
|
||||
|
||||
auto replacements_path = file_util::get_jak_project_dir() / "custom_assets" /
|
||||
game_version_names[config.game_version] / "texture_replacements";
|
||||
if (fs::exists(replacements_path)) {
|
||||
tex_db.replace_textures(replacements_path);
|
||||
}
|
||||
|
||||
if (config.process_game_count) {
|
||||
auto result = db.process_game_count_file();
|
||||
if (!result.empty()) {
|
||||
file_util::write_text_file(out_folder / "assets" / "game_count.txt", result);
|
||||
}
|
||||
}
|
||||
|
||||
if (config.levels_extract) {
|
||||
auto level_out_path =
|
||||
file_util::get_jak_project_dir() / "out" / game_version_names[config.game_version] / "fr3";
|
||||
file_util::create_dir_if_needed(level_out_path);
|
||||
extract_all_levels(db, tex_db, config.levels_to_extract, "GAME.CGO", config, level_out_path);
|
||||
}
|
||||
|
||||
mem_log("After extraction: {} MB", get_peak_rss() / (1024 * 1024));
|
||||
|
||||
if (!config.audio_dir_file_name.empty()) {
|
||||
auto streaming_audio_in = in_folder / "VAG";
|
||||
auto streaming_audio_out = out_folder / "assets" / "streaming_audio";
|
||||
file_util::create_dir_if_needed(streaming_audio_out);
|
||||
process_streamed_audio(config, streaming_audio_out, in_folder,
|
||||
config.streamed_audio_file_names);
|
||||
}
|
||||
|
||||
lg::info("Decompiler has finished successfully in {:.2f} seconds.", decomp_timer.getSeconds());
|
||||
return 0;
|
||||
return run_decompilation_process(config, in_folder, out_folder, false);
|
||||
}
|
||||
|
|
Loading…
Reference in a new issue