diff --git a/common/util/os.cpp b/common/util/os.cpp index 0b49c25d4..397a0b62f 100644 --- a/common/util/os.cpp +++ b/common/util/os.cpp @@ -3,20 +3,29 @@ #include "common/common_types.h" #include "common/log/log.h" -#ifdef __linux__ - +#ifdef _WIN32 +// clang-format off +#define NOMINMAX +#define WIN32_LEAN_AND_MEAN +#include +#include +// clang-format on +size_t get_peak_rss() { + HANDLE hProcess = GetCurrentProcess(); + PROCESS_MEMORY_COUNTERS pmc; + if (GetProcessMemoryInfo(hProcess, &pmc, sizeof(pmc))) { + return pmc.PeakWorkingSetSize; + } else { + return 0; + } +} +#else #include - size_t get_peak_rss() { rusage x; getrusage(RUSAGE_SELF, &x); return x.ru_maxrss * 1024; } - -#else -size_t get_peak_rss() { - return 0; -} #endif #ifdef _WIN32 diff --git a/common/util/os.h b/common/util/os.h index 7f745e66b..de779ee67 100644 --- a/common/util/os.h +++ b/common/util/os.h @@ -3,7 +3,6 @@ #include #include -// Note: these are not implemented on windows and will return zero. size_t get_peak_rss(); void setup_cpu_info(); diff --git a/decompiler/CMakeLists.txt b/decompiler/CMakeLists.txt index 00ef05461..7532391a8 100644 --- a/decompiler/CMakeLists.txt +++ b/decompiler/CMakeLists.txt @@ -89,7 +89,8 @@ add_library( VuDisasm/VuDisassembler.cpp VuDisasm/VuInstruction.cpp - config.cpp) + config.cpp + decompilation_process.cpp) target_link_libraries(decomp lzokay diff --git a/decompiler/config.cpp b/decompiler/config.cpp index d0d7d9127..a47df7c4b 100644 --- a/decompiler/config.cpp +++ b/decompiler/config.cpp @@ -64,7 +64,6 @@ Config make_config_via_json(nlohmann::json& json) { inputs_json.at("str_art_file_names").get>(); } - config.audio_dir_file_name = inputs_json.at("audio_dir_file_name").get(); config.streamed_audio_file_names = inputs_json.at("streamed_audio_file_names").get>(); @@ -321,6 +320,9 @@ Config make_config_via_json(nlohmann::json& json) { if (json.contains("save_texture_pngs")) { config.save_texture_pngs = json.at("save_texture_pngs").get(); } + if (json.contains("rip_streamed_audio")) { + config.rip_streamed_audio = json.at("rip_streamed_audio").get(); + } if (inputs_json.contains("animated_textures")) { config.animated_textures = diff --git a/decompiler/config.h b/decompiler/config.h index 9516a212a..7ac7793a2 100644 --- a/decompiler/config.h +++ b/decompiler/config.h @@ -102,8 +102,6 @@ struct Config { std::vector str_file_names; std::vector str_texture_file_names; std::vector str_art_file_names; - - std::string audio_dir_file_name; std::vector streamed_audio_file_names; std::string obj_file_name_map_file; @@ -175,6 +173,7 @@ struct Config { std::vector levels_to_extract; bool levels_extract; bool save_texture_pngs = false; + bool rip_streamed_audio = false; DecompileHacks hacks; diff --git a/decompiler/config/jak1/jak1_config.jsonc b/decompiler/config/jak1/jak1_config.jsonc index 85e7c3edf..ed1be8159 100644 --- a/decompiler/config/jak1/jak1_config.jsonc +++ b/decompiler/config/jak1/jak1_config.jsonc @@ -122,6 +122,9 @@ // save game textures as .png files to decompiler_out//textures "save_texture_pngs": false, + // whether or not to dump out streamed audio files to decompiler_out//audio + "rip_streamed_audio": false, + //////////////////////////// // PATCHING OPTIONS //////////////////////////// diff --git a/decompiler/config/jak1/ntsc_v1/inputs.jsonc b/decompiler/config/jak1/ntsc_v1/inputs.jsonc index afe79b72f..4e9b44f5c 100644 --- a/decompiler/config/jak1/ntsc_v1/inputs.jsonc +++ b/decompiler/config/jak1/ntsc_v1/inputs.jsonc @@ -258,10 +258,6 @@ "TEXT/6COMMON.TXT" ], - // uncomment the next line to extract audio to wave files. - //"audio_dir_file_name": "jak1/VAG", - "audio_dir_file_name": "", - "streamed_audio_file_names": [ "VAGWAD.ENG", "VAGWAD.FRE", diff --git a/decompiler/config/jak1_demo/default/inputs.jsonc b/decompiler/config/jak1_demo/default/inputs.jsonc index 4438a5472..e78766381 100644 --- a/decompiler/config/jak1_demo/default/inputs.jsonc +++ b/decompiler/config/jak1_demo/default/inputs.jsonc @@ -228,9 +228,5 @@ "TEXT/6COMMON.TXT" ], - // uncomment the next line to extract audio to wave files. - //"audio_dir_file_name": "jak1/VAG", - "audio_dir_file_name": "", - "streamed_audio_file_names": ["VAGWAD.ENG", "VAGWAD.JAP"] } diff --git a/decompiler/config/jak2/jak2_config.jsonc b/decompiler/config/jak2/jak2_config.jsonc index 460731425..2965d2de5 100644 --- a/decompiler/config/jak2/jak2_config.jsonc +++ b/decompiler/config/jak2/jak2_config.jsonc @@ -127,6 +127,9 @@ "rip_collision": false, // save game textures as .png files to decompiler_out//textures "save_texture_pngs": false, + + // whether or not to dump out streamed audio files to decompiler_out//audio + "rip_streamed_audio": false, //////////////////////////// // PATCHING OPTIONS diff --git a/decompiler/config/jak2/ntsc_v1/inputs.jsonc b/decompiler/config/jak2/ntsc_v1/inputs.jsonc index 8d9da73f7..d6876bccd 100644 --- a/decompiler/config/jak2/ntsc_v1/inputs.jsonc +++ b/decompiler/config/jak2/ntsc_v1/inputs.jsonc @@ -441,10 +441,6 @@ "TEXT/7COMMON.TXT" ], - // uncomment the next line to extract audio to wave files. - // "audio_dir_file_name": "jak2/VAG", - "audio_dir_file_name": "", - "streamed_audio_file_names": [ "VAGWAD.ENG", "VAGWAD.FRE", diff --git a/decompiler/config/jak3/jak3_config.jsonc b/decompiler/config/jak3/jak3_config.jsonc index 3377deaca..710820c0c 100644 --- a/decompiler/config/jak3/jak3_config.jsonc +++ b/decompiler/config/jak3/jak3_config.jsonc @@ -127,6 +127,9 @@ // save game textures as .png files to decompiler_out//textures "save_texture_pngs": false, + // whether or not to dump out streamed audio files to decompiler_out//audio + "rip_streamed_audio": false, + //////////////////////////// // PATCHING OPTIONS //////////////////////////// diff --git a/decompiler/config/jak3/ntsc_v1/inputs.jsonc b/decompiler/config/jak3/ntsc_v1/inputs.jsonc index fa1f68915..44f630845 100644 --- a/decompiler/config/jak3/ntsc_v1/inputs.jsonc +++ b/decompiler/config/jak3/ntsc_v1/inputs.jsonc @@ -333,10 +333,6 @@ "TEXT/11COMMON.TXT" ], - // uncomment the next line to extract audio to wave files. - // "audio_dir_file_name": "jak3/VAG", - "audio_dir_file_name": "", - "streamed_audio_file_names": [ "VAGWAD.ENG", "VAGWAD.FRE", diff --git a/decompiler/decompilation_process.cpp b/decompiler/decompilation_process.cpp new file mode 100644 index 000000000..89b3c1b6f --- /dev/null +++ b/decompiler/decompilation_process.cpp @@ -0,0 +1,284 @@ +#include "decompilation_process.h" + +#include +#include + +#include "config.h" + +#include "common/log/log.h" +#include "common/util/Timer.h" +#include "common/util/os.h" + +#include "Disasm/OpcodeInfo.h" +#include "ObjectFile/ObjectFileDB.h" +#include "data/streamed_audio.h" +#include "level_extractor/extract_level.h" + +int run_decompilation_process(decompiler::Config config, + const fs::path& in_folder, + const fs::path& out_folder, + const bool minimal_for_extractor) { + using namespace decompiler; + Timer decomp_timer; + + lg::info("[Mem] Top of main: {} MB\n", get_peak_rss() / (1024 * 1024)); + + init_opcode_info(); + + lg::info("[Mem] After init: {} MB\n", get_peak_rss() / (1024 * 1024)); + + std::vector dgos, objs, strs, tex_strs, art_strs; + if (minimal_for_extractor) { + // TODO - does this even matter, or can we just make the DGOs lazily loaded (does it already + // happen?) + for (const auto& dgo_name : config.dgo_names) { + std::string common_name = "GAME.CGO"; + if (dgo_name.length() > 3 && dgo_name.substr(dgo_name.length() - 3) == "DGO") { + // ends in DGO, it's a level + dgos.push_back(in_folder / dgo_name); + } else if (dgo_name.length() >= common_name.length() && + dgo_name.substr(dgo_name.length() - common_name.length()) == common_name) { + // it's COMMON.CGO, we need that too. + dgos.push_back(in_folder / dgo_name); + } + } + } else { + for (const auto& dgo_name : config.dgo_names) { + dgos.push_back(in_folder / dgo_name); + } + } + + if (minimal_for_extractor) { + // TODO - does this even matter, or can we just make the DGOs lazily loaded (does it already + // happen?) + for (const auto& obj_name : config.object_file_names) { + if (obj_name.length() > 3 && obj_name.substr(obj_name.length() - 3) == "TXT") { + // ends in TXT + objs.push_back(in_folder / obj_name); + } + } + } else { + for (const auto& obj_name : config.object_file_names) { + objs.push_back(in_folder / obj_name); + } + } + + if (!minimal_for_extractor) { + for (const auto& str_name : config.str_file_names) { + strs.push_back(in_folder / str_name); + } + } + + for (const auto& str_name : config.str_texture_file_names) { + tex_strs.push_back(in_folder / str_name); + } + + for (const auto& str_name : config.str_art_file_names) { + art_strs.push_back(in_folder / str_name); + } + + lg::info("[Mem] After config read: {} MB", get_peak_rss() / (1024 * 1024)); + + // build file database + lg::info("Setting up object file DB..."); + ObjectFileDB db(dgos, fs::path(config.obj_file_name_map_file), objs, strs, tex_strs, art_strs, + config); + + // Explicitly fail if a file in the 'allowed_objects' list wasn't found in the DB + // as this is another silent error that can be confusing + if (!config.allowed_objects.empty()) { + for (const auto& expected_obj : config.allowed_objects) { + if (db.obj_files_by_name.count(expected_obj) == 0) { + // TODO - this is wrong for jak1, fix eventually as this is now done in 3 places + lg::error( + "Expected to find '{}' in the ObjectFileDB but did not. Check " + "./decompiler/config/{}/inputs.jsonc", + expected_obj, config.game_name); + return 1; + } + } + } + + lg::info("[Mem] After DB setup: {} MB", get_peak_rss() / (1024 * 1024)); + + // write out DGO file info + file_util::create_dir_if_needed(out_folder); + file_util::write_text_file(out_folder / "dgo.txt", db.generate_dgo_listing()); + // write out object file map (used for future decompilations, if desired) + file_util::write_text_file(out_folder / "obj.txt", + db.generate_obj_listing(config.merged_objects)); + + // dump raw objs + if (config.dump_objs) { + auto path = out_folder / "raw_obj"; + file_util::create_dir_if_needed(path); + db.dump_raw_objects(path); + } + + // process files (required for all analysis) + db.process_link_data(config); + lg::info("[Mem] After link data: {} MB", get_peak_rss() / (1024 * 1024)); + db.find_code(config); + db.process_labels(); + lg::info("[Mem] After code: {} MB", get_peak_rss() / (1024 * 1024)); + + // top level decompile (do this before printing asm so we get function names) + if (config.find_functions) { + db.ir2_top_level_pass(config); + } + + // print disassembly + if (config.disassemble_code || config.disassemble_data) { + db.write_disassembly(out_folder, config.disassemble_data, config.disassemble_code, + config.write_hex_near_instructions); + } + + if (config.process_art_groups) { + db.extract_art_info(); + // dump art info to json if requested + if (config.dump_art_group_info) { + auto ag_file_name = out_folder / "dump" / "art-group-info.min.json"; + nlohmann::json ag_json = db.dts.art_group_info; + file_util::create_dir_if_needed_for_file(ag_file_name); + file_util::write_text_file(ag_file_name, ag_json.dump(-1)); + lg::info("[DUMP] Dumped art group info to {}", ag_file_name.string()); + } + if (config.dump_joint_geo_info) { + auto jg_file_name = out_folder / "dump" / "joint-node-info.min.json"; + nlohmann::json jg_json = db.dts.jg_info; + file_util::create_dir_if_needed_for_file(jg_file_name); + file_util::write_text_file(jg_file_name, jg_json.dump(-1)); + lg::info("[DUMP] Dumped joint node info to {}", jg_file_name.string()); + } + } else if (!config.art_group_info_dump.empty() || !config.jg_info_dump.empty()) { + // process art groups (used in decompilation) + // - if the config has a path to the art info dump, just use that + // - otherwise (or if we want to dump it fresh) extract it + if (!config.art_group_info_dump.empty()) { + db.dts.art_group_info = config.art_group_info_dump; + } + if (!config.jg_info_dump.empty()) { + db.dts.jg_info = config.jg_info_dump; + } + } else { + lg::error("`process_art_groups` was false and no art-group-info dump was provided!"); + return 1; + } + + if (config.process_tpages && !config.texture_info_dump.empty()) { + db.dts.textures = config.texture_info_dump; + } + + // main decompile. + if (config.decompile_code) { + db.analyze_functions_ir2(out_folder, config, {}, {}, {}); + } + + if (config.generate_all_types) { + ASSERT_MSG(config.decompile_code, "Must decompile code to generate all-types"); + db.ir2_analyze_all_types(out_folder / "new-all-types.gc", config.old_all_types_file, + config.hacks.types_with_bad_inspect_methods); + } + + lg::info("[Mem] After decomp: {} MB", get_peak_rss() / (1024 * 1024)); + + // write out all symbols + file_util::write_text_file(out_folder / "all-syms.gc", db.dts.dump_symbol_types()); + + // write art groups + if (config.process_art_groups) { + db.dump_art_info(out_folder); + } + + if (config.hexdump_code || config.hexdump_data) { + db.write_object_file_words(out_folder, config.hexdump_data, config.hexdump_code); + } + + // data stuff + if (config.write_scripts) { + db.find_and_write_scripts(out_folder); + } + + // ensure asset dir exists + file_util::create_dir_if_needed(out_folder / "assets"); + + if (config.process_game_text) { + auto result = db.process_game_text_files(config); + if (!result.empty()) { + file_util::write_text_file(out_folder / "assets" / "game_text.txt", result); + } + } + + lg::info("[Mem] After text: {} MB", get_peak_rss() / (1024 * 1024)); + + if (config.process_subtitle_text || config.process_subtitle_images) { + auto result = db.process_all_spool_subtitles( + config, config.process_subtitle_images ? out_folder / "assets" / "subtitle-images" : ""); + if (!result.empty()) { + file_util::write_text_file(out_folder / "assets" / "game_subs.txt", result); + } + } + + lg::info("[Mem] After spool handling: {} MB", get_peak_rss() / (1024 * 1024)); + + TextureDB tex_db; + if (config.process_tpages || config.levels_extract) { + auto textures_out = out_folder / "textures"; + auto dump_out = out_folder / "import"; + file_util::create_dir_if_needed(textures_out); + auto result = db.process_tpages(tex_db, textures_out, config, dump_out); + if (!result.empty() && config.process_tpages) { + file_util::write_text_file(textures_out / "tpage-dir.txt", result); + file_util::write_text_file(textures_out / "tex-remap.txt", + tex_db.generate_texture_dest_adjustment_table()); + } + if (config.dump_tex_info) { + auto texture_file_name = out_folder / "dump" / "tex-info.min.json"; + nlohmann::json texture_json = db.dts.textures; + file_util::create_dir_if_needed_for_file(texture_file_name); + file_util::write_text_file(texture_file_name, texture_json.dump(-1)); + lg::info("[DUMP] Dumped texture info to {}", texture_file_name.string()); + } + } + + lg::info("[Mem] After textures: {} MB", get_peak_rss() / (1024 * 1024)); + + // Merge textures before replacing them, in other words, replacements take priority + auto texture_merge_path = file_util::get_jak_project_dir() / "game" / "assets" / + game_version_names[config.game_version] / "texture_merges"; + if (fs::exists(texture_merge_path)) { + tex_db.merge_textures(texture_merge_path); + } + + auto replacements_path = file_util::get_jak_project_dir() / "custom_assets" / + game_version_names[config.game_version] / "texture_replacements"; + if (fs::exists(replacements_path)) { + tex_db.replace_textures(replacements_path); + } + + if (config.process_game_count) { + auto result = db.process_game_count_file(); + if (!result.empty()) { + file_util::write_text_file(out_folder / "assets" / "game_count.txt", result); + } + } + + if (config.levels_extract) { + auto level_out_path = + file_util::get_jak_project_dir() / "out" / game_version_names[config.game_version] / "fr3"; + file_util::create_dir_if_needed(level_out_path); + extract_all_levels(db, tex_db, config.levels_to_extract, "GAME.CGO", config, level_out_path); + } + + lg::info("[Mem] After extraction: {} MB", get_peak_rss() / (1024 * 1024)); + + if (config.rip_streamed_audio) { + auto streaming_audio_out = out_folder / "audio"; + file_util::create_dir_if_needed(streaming_audio_out); + process_streamed_audio(config, streaming_audio_out, in_folder, + config.streamed_audio_file_names); + } + + lg::info("Decompiler has finished successfully in {:.2f} seconds.", decomp_timer.getSeconds()); + return 0; +} diff --git a/decompiler/decompilation_process.h b/decompiler/decompilation_process.h new file mode 100644 index 000000000..c7efd4a8e --- /dev/null +++ b/decompiler/decompilation_process.h @@ -0,0 +1,8 @@ +#pragma once + +#include "config.h" + +int run_decompilation_process(decompiler::Config config, + const fs::path& in_folder, + const fs::path& out_folder, + const bool minimal_for_extractor); diff --git a/decompiler/extractor/main.cpp b/decompiler/extractor/main.cpp index 879671b85..241a6cc0f 100644 --- a/decompiler/extractor/main.cpp +++ b/decompiler/extractor/main.cpp @@ -5,10 +5,9 @@ #include "common/util/term_util.h" #include "common/util/unicode_util.h" -#include "decompiler/ObjectFile/ObjectFileDB.h" #include "decompiler/config.h" +#include "decompiler/decompilation_process.h" #include "decompiler/extractor/extractor_util.h" -#include "decompiler/level_extractor/extract_level.h" #include "goalc/compiler/Compiler.h" #include "third-party/CLI11.hpp" @@ -102,116 +101,24 @@ std::tuple, ExtractorErrorCode> validate( }; } -// TODO - remove code duplication, most of this is copying what happens in decompiler's `main.cpp` -void decompile(const fs::path& iso_data_path, - const std::string& data_subfolder, - const std::string& config_override) { - using namespace decompiler; - +ExtractorErrorCode decompile(const fs::path& in_folder, + const std::string& data_subfolder, + const std::string& config_override) { // Determine which config to use from the database - const auto version_info = get_version_info_or_default(iso_data_path); + const auto version_info = get_version_info_or_default(in_folder); - Config config = read_config_file(file_util::get_jak_project_dir() / "decompiler" / "config" / - version_info.game_name / - fmt::format("{}_config.jsonc", version_info.game_name), - version_info.decomp_config_version, config_override); + decompiler::Config config = decompiler::read_config_file( + file_util::get_jak_project_dir() / "decompiler" / "config" / version_info.game_name / + fmt::format("{}_config.jsonc", version_info.game_name), + version_info.decomp_config_version, config_override); - std::vector dgos, objs, tex_strs, art_strs; - - // grab all DGOS we need (level + common) - // TODO - Jak 2 - jak 1 specific code? - for (const auto& dgo_name : config.dgo_names) { - std::string common_name = "GAME.CGO"; - if (dgo_name.length() > 3 && dgo_name.substr(dgo_name.length() - 3) == "DGO") { - // ends in DGO, it's a level - dgos.push_back(iso_data_path / dgo_name); - } else if (dgo_name.length() >= common_name.length() && - dgo_name.substr(dgo_name.length() - common_name.length()) == common_name) { - // it's COMMON.CGO, we need that too. - dgos.push_back(iso_data_path / dgo_name); - } - } - - // grab all the object files we need (just text) - for (const auto& obj_name : config.object_file_names) { - if (obj_name.length() > 3 && obj_name.substr(obj_name.length() - 3) == "TXT") { - // ends in TXT - objs.push_back(iso_data_path / obj_name); - } - } - - for (const auto& str_name : config.str_texture_file_names) { - tex_strs.push_back(iso_data_path / str_name); - } - - for (const auto& str_name : config.str_art_file_names) { - art_strs.push_back(iso_data_path / str_name); - } - - // set up objects - ObjectFileDB db(dgos, fs::path(config.obj_file_name_map_file), objs, {}, tex_strs, art_strs, - config); - - // save object files auto out_folder = file_util::get_jak_project_dir() / "decompiler_out" / data_subfolder; - auto raw_obj_folder = out_folder / "raw_obj"; - file_util::create_dir_if_needed(raw_obj_folder); - db.dump_raw_objects(raw_obj_folder); - // analyze object file link data - db.process_link_data(config); - db.find_code(config); - db.process_labels(); - - // ensure asset dir exists - file_util::create_dir_if_needed(out_folder / "assets"); - - // text files - { - auto result = db.process_game_text_files(config); - if (!result.empty()) { - file_util::write_text_file(out_folder / "assets" / "game_text.txt", result); - } - } - - // textures - decompiler::TextureDB tex_db; - auto textures_out = out_folder / "textures"; - auto dump_out = out_folder / "import"; - file_util::create_dir_if_needed(textures_out); - file_util::write_text_file(textures_out / "tpage-dir.txt", - db.process_tpages(tex_db, textures_out, config, dump_out)); - - // texture merges - // TODO - put all this stuff in somewhere common - auto texture_merge_path = file_util::get_jak_project_dir() / "game" / "assets" / - game_version_names[config.game_version] / "texture_merges"; - if (fs::exists(texture_merge_path)) { - tex_db.merge_textures(texture_merge_path); - } - - // texture replacements - auto replacements_path = file_util::get_jak_project_dir() / "custom_assets" / - game_version_names[config.game_version] / "texture_replacements"; - if (fs::exists(replacements_path)) { - tex_db.replace_textures(replacements_path); - } - - // game count - { - auto result = db.process_game_count_file(); - if (!result.empty()) { - file_util::write_text_file(out_folder / "assets" / "game_count.txt", result); - } - } - - // levels - { - auto level_out_path = - file_util::get_jak_project_dir() / "out" / game_version_names[config.game_version] / "fr3"; - file_util::create_dir_if_needed(level_out_path); - extract_all_levels(db, tex_db, config.levels_to_extract, "GAME.CGO", config, level_out_path); + const auto result = run_decompilation_process(config, in_folder, out_folder, true); + if (result != 0) { + return ExtractorErrorCode::DECOMPILATION_GENERIC_ERROR; } + return ExtractorErrorCode::SUCCESS; } const std::unordered_map game_iso_flag_names = { @@ -315,7 +222,6 @@ int main(int argc, char** argv) { } // - SETUP - decompiler::init_opcode_info(); if (!project_path_override.empty()) { if (!fs::exists(project_path_override)) { lg::error("Error: project path override '{}' does not exist", project_path_override.string()); @@ -426,7 +332,11 @@ int main(int argc, char** argv) { // Get hash and file count const auto [hash, file_count] = calculate_extraction_hash(iso_data_path); // Validate - auto [version_info, code] = validate(iso_data_path, hash, file_count); + auto [version_info, validate_code] = validate(iso_data_path, hash, file_count); + if (validate_code == ExtractorErrorCode::VALIDATION_BAD_EXTRACTION || + (flag_fail_on_validation && validate_code != ExtractorErrorCode::SUCCESS)) { + return static_cast(validate_code); + } } // write out a json file with some metadata for the game @@ -455,7 +365,10 @@ int main(int argc, char** argv) { if (flag_decompile) { try { - decompile(iso_data_path, data_subfolder, decomp_config_override); + const auto status_code = decompile(iso_data_path, data_subfolder, decomp_config_override); + if (status_code != ExtractorErrorCode::SUCCESS) { + return static_cast(status_code); + } } catch (std::exception& e) { lg::error("Error during decompile: {}", e.what()); return static_cast(ExtractorErrorCode::DECOMPILATION_GENERIC_ERROR); @@ -463,7 +376,10 @@ int main(int argc, char** argv) { } if (flag_compile) { - compile(iso_data_path, data_subfolder); + const auto status_code = compile(iso_data_path, data_subfolder); + if (status_code != ExtractorErrorCode::SUCCESS) { + return static_cast(status_code); + } } if (flag_play) { diff --git a/decompiler/main.cpp b/decompiler/main.cpp index 5586970e0..1ce9d89c5 100644 --- a/decompiler/main.cpp +++ b/decompiler/main.cpp @@ -1,33 +1,15 @@ -#include #include -#include #include "config.h" +#include "decompilation_process.h" #include "common/log/log.h" #include "common/util/FileUtil.h" -#include "common/util/Timer.h" -#include "common/util/diff.h" -#include "common/util/os.h" #include "common/util/set_util.h" #include "common/util/term_util.h" #include "common/util/unicode_util.h" -#include "common/versions/versions.h" - -#include "ObjectFile/ObjectFileDB.h" -#include "decompiler/data/TextureDB.h" -#include "decompiler/data/streamed_audio.h" -#include "decompiler/level_extractor/extract_level.h" #include "third-party/CLI11.hpp" -#include "third-party/json.hpp" - -template -static void mem_log(const std::string& format, Args&&... args) { -#ifndef _WIN32 - lg::info("[Mem] " + format, std::forward(args)...); -#endif -} int main(int argc, char** argv) { ArgumentGuard u8_guard(argc, argv); @@ -81,11 +63,9 @@ int main(int argc, char** argv) { return 1; } - using namespace decompiler; - - Config config; + decompiler::Config config; try { - config = read_config_file(config_path, config_game_version, config_override); + config = decompiler::read_config_file(config_path, config_game_version, config_override); } catch (const std::exception& e) { lg::error("Failed to parse config: {}", e.what()); return 1; @@ -115,7 +95,7 @@ int main(int argc, char** argv) { // Warning message if expected ELF isn't found, user could be using bad assets / didn't extract // the ISO properly - if (!config.expected_elf_name.empty() && !exists(in_folder / config.expected_elf_name)) { + if (!config.expected_elf_name.empty() && !fs::exists(in_folder / config.expected_elf_name)) { lg::error( "WARNING - '{}' does not contain the expected ELF file '{}'. Was the ISO extracted " "properly or is there a version mismatch?", @@ -123,235 +103,5 @@ int main(int argc, char** argv) { } // -- Begin the Decompilation! - - Timer decomp_timer; - - mem_log("Top of main: {} MB\n", get_peak_rss() / (1024 * 1024)); - - init_opcode_info(); - - mem_log("After init: {} MB\n", get_peak_rss() / (1024 * 1024)); - - std::vector dgos, objs, strs, tex_strs, art_strs; - for (const auto& dgo_name : config.dgo_names) { - dgos.push_back(in_folder / dgo_name); - } - - for (const auto& obj_name : config.object_file_names) { - objs.push_back(in_folder / obj_name); - } - - for (const auto& str_name : config.str_file_names) { - strs.push_back(in_folder / str_name); - } - - for (const auto& str_name : config.str_texture_file_names) { - tex_strs.push_back(in_folder / str_name); - } - - for (const auto& str_name : config.str_art_file_names) { - art_strs.push_back(in_folder / str_name); - } - - mem_log("After config read: {} MB", get_peak_rss() / (1024 * 1024)); - - // build file database - lg::info("Setting up object file DB..."); - ObjectFileDB db(dgos, fs::path(config.obj_file_name_map_file), objs, strs, tex_strs, art_strs, - config); - - // Explicitly fail if a file in the 'allowed_objects' list wasn't found in the DB - // as this is another silent error that can be confusing - if (!config.allowed_objects.empty()) { - for (const auto& expected_obj : config.allowed_objects) { - if (db.obj_files_by_name.count(expected_obj) == 0) { - // TODO - this is wrong for jak1, fix eventually as this is now done in 3 places - lg::error( - "Expected to find '{}' in the ObjectFileDB but did not. Check " - "./decompiler/config/{}/inputs.jsonc", - expected_obj, config.game_name); - return 1; - } - } - } - - mem_log("After DB setup: {} MB", get_peak_rss() / (1024 * 1024)); - - // write out DGO file info - file_util::write_text_file(out_folder / "dgo.txt", db.generate_dgo_listing()); - // write out object file map (used for future decompilations, if desired) - file_util::write_text_file(out_folder / "obj.txt", - db.generate_obj_listing(config.merged_objects)); - - // dump raw objs - if (config.dump_objs) { - auto path = out_folder / "raw_obj"; - file_util::create_dir_if_needed(path); - db.dump_raw_objects(path); - } - - // process files (required for all analysis) - db.process_link_data(config); - mem_log("After link data: {} MB", get_peak_rss() / (1024 * 1024)); - db.find_code(config); - db.process_labels(); - mem_log("After code: {} MB", get_peak_rss() / (1024 * 1024)); - - // top level decompile (do this before printing asm so we get function names) - if (config.find_functions) { - db.ir2_top_level_pass(config); - } - - // print disassembly - if (config.disassemble_code || config.disassemble_data) { - db.write_disassembly(out_folder, config.disassemble_data, config.disassemble_code, - config.write_hex_near_instructions); - } - - if (config.process_art_groups) { - db.extract_art_info(); - // dump art info to json if requested - if (config.dump_art_group_info) { - auto ag_file_name = out_folder / "dump" / "art-group-info.min.json"; - nlohmann::json ag_json = db.dts.art_group_info; - file_util::create_dir_if_needed_for_file(ag_file_name); - file_util::write_text_file(ag_file_name, ag_json.dump(-1)); - lg::info("[DUMP] Dumped art group info to {}", ag_file_name.string()); - } - if (config.dump_joint_geo_info) { - auto jg_file_name = out_folder / "dump" / "joint-node-info.min.json"; - nlohmann::json jg_json = db.dts.jg_info; - file_util::create_dir_if_needed_for_file(jg_file_name); - file_util::write_text_file(jg_file_name, jg_json.dump(-1)); - lg::info("[DUMP] Dumped joint node info to {}", jg_file_name.string()); - } - } else if (!config.art_group_info_dump.empty() || !config.jg_info_dump.empty()) { - // process art groups (used in decompilation) - // - if the config has a path to the art info dump, just use that - // - otherwise (or if we want to dump it fresh) extract it - if (!config.art_group_info_dump.empty()) { - db.dts.art_group_info = config.art_group_info_dump; - } - if (!config.jg_info_dump.empty()) { - db.dts.jg_info = config.jg_info_dump; - } - } else { - lg::error("`process_art_groups` was false and no art-group-info dump was provided!"); - return 1; - } - - if (config.process_tpages && !config.texture_info_dump.empty()) { - db.dts.textures = config.texture_info_dump; - } - - // main decompile. - if (config.decompile_code) { - db.analyze_functions_ir2(out_folder, config, {}, {}, {}); - } - - if (config.generate_all_types) { - ASSERT_MSG(config.decompile_code, "Must decompile code to generate all-types"); - db.ir2_analyze_all_types(out_folder / "new-all-types.gc", config.old_all_types_file, - config.hacks.types_with_bad_inspect_methods); - } - - mem_log("After decomp: {} MB", get_peak_rss() / (1024 * 1024)); - - // write out all symbols - file_util::write_text_file(out_folder / "all-syms.gc", db.dts.dump_symbol_types()); - - // write art groups - if (config.process_art_groups) { - db.dump_art_info(out_folder); - } - - if (config.hexdump_code || config.hexdump_data) { - db.write_object_file_words(out_folder, config.hexdump_data, config.hexdump_code); - } - - // data stuff - if (config.write_scripts) { - db.find_and_write_scripts(out_folder); - } - - if (config.process_game_text) { - auto result = db.process_game_text_files(config); - if (!result.empty()) { - file_util::write_text_file(out_folder / "assets" / "game_text.txt", result); - } - } - - mem_log("After text: {} MB", get_peak_rss() / (1024 * 1024)); - - if (config.process_subtitle_text || config.process_subtitle_images) { - auto result = db.process_all_spool_subtitles( - config, config.process_subtitle_images ? out_folder / "assets" / "subtitle-images" : ""); - if (!result.empty()) { - file_util::write_text_file(out_folder / "assets" / "game_subs.txt", result); - } - } - - mem_log("After spool handling: {} MB", get_peak_rss() / (1024 * 1024)); - - TextureDB tex_db; - if (config.process_tpages || config.levels_extract) { - auto textures_out = out_folder / "textures"; - auto dump_out = out_folder / "import"; - file_util::create_dir_if_needed(textures_out); - auto result = db.process_tpages(tex_db, textures_out, config, dump_out); - if (!result.empty() && config.process_tpages) { - file_util::write_text_file(textures_out / "tpage-dir.txt", result); - file_util::write_text_file(textures_out / "tex-remap.txt", - tex_db.generate_texture_dest_adjustment_table()); - } - if (config.dump_tex_info) { - auto texture_file_name = out_folder / "dump" / "tex-info.min.json"; - nlohmann::json texture_json = db.dts.textures; - file_util::create_dir_if_needed_for_file(texture_file_name); - file_util::write_text_file(texture_file_name, texture_json.dump(-1)); - lg::info("[DUMP] Dumped texture info to {}", texture_file_name.string()); - } - } - - mem_log("After textures: {} MB", get_peak_rss() / (1024 * 1024)); - - // Merge textures before replacing them, in other words, replacements take priority - auto texture_merge_path = file_util::get_jak_project_dir() / "game" / "assets" / - game_version_names[config.game_version] / "texture_merges"; - if (fs::exists(texture_merge_path)) { - tex_db.merge_textures(texture_merge_path); - } - - auto replacements_path = file_util::get_jak_project_dir() / "custom_assets" / - game_version_names[config.game_version] / "texture_replacements"; - if (fs::exists(replacements_path)) { - tex_db.replace_textures(replacements_path); - } - - if (config.process_game_count) { - auto result = db.process_game_count_file(); - if (!result.empty()) { - file_util::write_text_file(out_folder / "assets" / "game_count.txt", result); - } - } - - if (config.levels_extract) { - auto level_out_path = - file_util::get_jak_project_dir() / "out" / game_version_names[config.game_version] / "fr3"; - file_util::create_dir_if_needed(level_out_path); - extract_all_levels(db, tex_db, config.levels_to_extract, "GAME.CGO", config, level_out_path); - } - - mem_log("After extraction: {} MB", get_peak_rss() / (1024 * 1024)); - - if (!config.audio_dir_file_name.empty()) { - auto streaming_audio_in = in_folder / "VAG"; - auto streaming_audio_out = out_folder / "assets" / "streaming_audio"; - file_util::create_dir_if_needed(streaming_audio_out); - process_streamed_audio(config, streaming_audio_out, in_folder, - config.streamed_audio_file_names); - } - - lg::info("Decompiler has finished successfully in {:.2f} seconds.", decomp_timer.getSeconds()); - return 0; + return run_decompilation_process(config, in_folder, out_folder, false); }