Rip collision based on config flag (#3348)

Added a new `rip_collision` flag to the decompiler config and wired it
up. Cleaned up the comments around these fields in the JSON too for
clarity

Also refactored a bunch of the related extract functions to read from
`config` object rather than pass down a bunch of booleans

Taskfile.yml

@@ -90,7 +90,6 @@ tasks:
       - cmd: python ./scripts/cpp/format-includes.py
       - cmd: python ./third-party/run-clang-format/run-clang-format.py -r common decompiler game goalc test tools lsp -i
       - task: format-json
-
   # DECOMPILING
   decomp:
     cmds:
@@ -115,6 +114,16 @@ tasks:
       - task: decomp-file
       - python ./scripts/gsrc/update-from-decomp.py --game "{{.GAME}}" --file {{.FILE}}
       - task: lint-gsrc-file
+  # ASSET RIPPING
+  rip-textures:
+    cmds:
+      - "{{.DECOMP_BIN_RELEASE_DIR}}/decompiler \"./decompiler/config/{{.DECOMP_CONFIG}}\" \"./iso_data\" \"./decompiler_out\" --version \"{{.DECOMP_CONFIG_VERSION}}\" --config-override '{\"decompile_code\": false, \"levels_extract\": true, \"save_texture_pngs\": true}'"
+  rip-levels:
+    cmds:
+      - "{{.DECOMP_BIN_RELEASE_DIR}}/decompiler \"./decompiler/config/{{.DECOMP_CONFIG}}\" \"./iso_data\" \"./decompiler_out\" --version \"{{.DECOMP_CONFIG_VERSION}}\" --config-override '{\"decompile_code\": false, \"levels_extract\": true, \"rip_levels\": true}'"
+  rip-collision:
+    cmds:
+      - "{{.DECOMP_BIN_RELEASE_DIR}}/decompiler \"./decompiler/config/{{.DECOMP_CONFIG}}\" \"./iso_data\" \"./decompiler_out\" --version \"{{.DECOMP_CONFIG_VERSION}}\" --config-override '{\"decompile_code\": false, \"levels_extract\": true, \"extract_collision\": true, \"rip_collision\": true}'"
   # TOOLS
   analyze-ee-memory:
     cmds:

decompiler/config.cpp

@@ -314,6 +314,10 @@ Config make_config_via_json(nlohmann::json& json) {
   config.import_deps_by_file =
       import_deps.get<std::unordered_map<std::string, std::vector<std::string>>>();
 
+  if (json.contains("rip_collision")) {
+    config.rip_collision = json.at("rip_collision").get<bool>();
+  }
+
   config.write_patches = json.at("write_patches").get<bool>();
   config.apply_patches = json.at("apply_patches").get<bool>();
   const auto& object_patches = json.at("object_patches");

decompiler/config.h

@@ -179,6 +179,8 @@ struct Config {
   std::unordered_map<std::string, std::string> joint_node_hacks;
 
   std::unordered_map<std::string, std::vector<std::string>> import_deps_by_file;
+
+  bool rip_collision = false;
 };
 
 Config read_config_file(const fs::path& path_to_config_file,

decompiler/config/jak1/jak1_config.jsonc

@@ -56,7 +56,7 @@
   "generate_symbol_definition_map": false,
 
   // genreate the all-types file
-  "generate_all_types" : false,
+  "generate_all_types": false,
 
   // debug option for instruction decoder
   "write_hex_near_instructions": false,
@@ -100,14 +100,16 @@
   // LEVEL EXTRACTION
   ////////////////////////////
 
-  // turn this on to extract level background graphics data
+  // turn this on to extract level background graphics data as .fr3 files in out/<game>/fr3
   "levels_extract": true,
-  // turn this on if you want extracted levels to be saved out as .obj files
+  // turn this on if you want extracted levels to be saved out as .glb files in glb_out/<game>
   "rip_levels": false,
-  // should we extract collision meshes?
-  // these can be displayed in game, but makes the .fr3 files slightly larger
+  // should we also extract collision meshes to the .fr3 files?
+  // these can be displayed in-game with the OpenGOAL collision renderer
   "extract_collision": true,
-  // save game textures as .png files.
+  // turn this on if you want extracted level collision to be saved as .obj files in debug_out/<game>
+  "rip_collision": false,
+  // save game textures as .png files to decompiler_out/<game>/textures
   "save_texture_pngs": false,
 
   ////////////////////////////

decompiler/config/jak2/jak2_config.jsonc

@@ -110,14 +110,16 @@
   // LEVEL EXTRACTION
   ////////////////////////////
 
-  // turn this on to extract level background graphics data
+  // turn this on to extract level background graphics data as .fr3 files in out/<game>/fr3
   "levels_extract": true,
-  // turn this on if you want extracted levels to be saved out as .obj files
+  // turn this on if you want extracted levels to be saved out as .glb files in glb_out/<game>
   "rip_levels": false,
-  // should we extract collision meshes?
-  // these can be displayed in game, but makes the .fr3 files slightly larger
+  // should we also extract collision meshes to the .fr3 files?
+  // these can be displayed in-game with the OpenGOAL collision renderer
   "extract_collision": true,
-
+  // turn this on if you want extracted level collision to be saved as .obj files in debug_out/<game>
+  "rip_collision": false,
+  // save game textures as .png files to decompiler_out/<game>/textures
   "save_texture_pngs": false,
 
   ////////////////////////////

decompiler/config/jak3/jak3_config.jsonc

@@ -102,14 +102,16 @@
   // LEVEL EXTRACTION
   ////////////////////////////
 
-  // turn this on to extract level background graphics data
+  // turn this on to extract level background graphics data as .fr3 files in out/<game>/fr3
   "levels_extract": false,
-  // turn this on if you want extracted levels to be saved out as .obj files
+  // turn this on if you want extracted levels to be saved out as .glb files in glb_out/<game>
   "rip_levels": false,
-  // should we extract collision meshes?
-  // these can be displayed in game, but makes the .fr3 files slightly larger
+  // should we also extract collision meshes to the .fr3 files?
+  // these can be displayed in-game with the OpenGOAL collision renderer
   "extract_collision": false,
-
+  // turn this on if you want extracted level collision to be saved as .obj files in debug_out/<game>
+  "rip_collision": false,
+  // save game textures as .png files to decompiler_out/<game>/textures
   "save_texture_pngs": false,
 
   ////////////////////////////

decompiler/extractor/main.cpp

@@ -200,8 +200,7 @@ void decompile(const fs::path& iso_data_path, const std::string& data_subfolder)
     auto level_out_path =
         file_util::get_jak_project_dir() / "out" / game_version_names[config.game_version] / "fr3";
     file_util::create_dir_if_needed(level_out_path);
-    extract_all_levels(db, tex_db, config.levels_to_extract, "GAME.CGO", config, config.rip_levels,
-                       config.extract_collision, level_out_path);
+    extract_all_levels(db, tex_db, config.levels_to_extract, "GAME.CGO", config, level_out_path);
   }
 }
 

decompiler/level_extractor/extract_collide_frags.cpp

@@ -233,8 +233,7 @@ void extract_collide_frags(const level_tools::DrawableTreeCollideFragment* tree,
                            const std::vector<const level_tools::DrawableTreeInstanceTie*>& ties,
                            const Config& config,
                            const std::string& debug_name,
-                           tfrag3::Level& out,
-                           bool dump_level) {
+                           tfrag3::Level& out) {
   // in-game, the broad-phase collision builds a list of fragments, then unpacks them with:
   /*
    *(dotimes (i (-> *collide-list* num-items))
@@ -260,7 +259,7 @@ void extract_collide_frags(const level_tools::DrawableTreeCollideFragment* tree,
     total_faces += frag.unpacked.faces.size();
   }
 
-  if (dump_level) {
+  if (config.rip_collision) {
     auto debug_out = debug_dump_to_obj(all_frags);
     auto file_path = file_util::get_file_path(
         {fmt::format("debug_out/{}", config.game_name), fmt::format("collide-{}.obj", debug_name)});
@@ -442,8 +441,7 @@ void extract_collide_frags(const level_tools::CollideHash& chash,
                            const Config& config,
                            const std::string& debug_name,
                            const decompiler::DecompilerTypeSystem& dts,
-                           tfrag3::Level& out,
-                           bool dump_level) {
+                           tfrag3::Level& out) {
   // We need to find all collide-hash-fragments, but we can't just scan through the entire file.
   // for collide-hash-fragments, we need to figure out which TIEs they belong to, to apply the
   // instance transformation matrix.
@@ -507,7 +505,7 @@ void extract_collide_frags(const level_tools::CollideHash& chash,
     }
   }
 
-  if (dump_level) {
+  if (config.rip_collision) {
     // out.collision.vertices every 3 vertices make a face, so it duplicates vertices in many cases
     // for now debug_dump_to_obj isn't smart and doesn't hash these to save space or anything
     auto debug_out = debug_dump_to_obj(out.collision.vertices);

decompiler/level_extractor/extract_collide_frags.h

@@ -10,15 +10,13 @@ void extract_collide_frags(const level_tools::DrawableTreeCollideFragment* tree,
                            const std::vector<const level_tools::DrawableTreeInstanceTie*>& ties,
                            const Config& config,
                            const std::string& debug_name,
-                           tfrag3::Level& out,
-                           bool dump_level);
+                           tfrag3::Level& out);
 
 void extract_collide_frags(const level_tools::CollideHash& chash,
                            const std::vector<const level_tools::DrawableTreeInstanceTie*>& ties,
                            const Config& config,
                            const std::string& debug_name,
                            const decompiler::DecompilerTypeSystem& dts,
-                           tfrag3::Level& out,
-                           bool dump_level);
+                           tfrag3::Level& out);
 
 }  // namespace decompiler

decompiler/level_extractor/extract_level.cpp

@@ -160,7 +160,6 @@ level_tools::BspHeader extract_bsp_from_level(const ObjectFileDB& db,
                                               const TextureDB& tex_db,
                                               const std::string& dgo_name,
                                               const Config& config,
-                                              bool extract_collision,
                                               tfrag3::Level& level_data) {
   auto hacks = config.hacks;
   auto bsp_rec = get_bsp_file(db.obj_files_by_dgo.at(dgo_name), dgo_name);
@@ -231,14 +230,15 @@ level_tools::BspHeader extract_bsp_from_level(const ObjectFileDB& db,
       ASSERT(as_shrub_tree);
       extract_shrub(as_shrub_tree, fmt::format("{}-{}-shrub", dgo_name, i++),
                     bsp_header.texture_remap_table, tex_db, {}, level_data, false, db.version());
-    } else if (draw_tree->my_type() == "drawable-tree-collide-fragment" && extract_collision) {
+    } else if (draw_tree->my_type() == "drawable-tree-collide-fragment" &&
+               config.extract_collision) {
       auto as_collide_frags =
           dynamic_cast<level_tools::DrawableTreeCollideFragment*>(draw_tree.get());
       ASSERT(as_collide_frags);
       ASSERT(!got_collide);
       got_collide = true;
       extract_collide_frags(as_collide_frags, all_ties, config,
-                            fmt::format("{}-{}-collide", dgo_name, i++), level_data, false);
+                            fmt::format("{}-{}-collide", dgo_name, i++), level_data);
     } else {
       lg::print("  unsupported tree {}\n", draw_tree->my_type());
     }
@@ -247,7 +247,7 @@ level_tools::BspHeader extract_bsp_from_level(const ObjectFileDB& db,
   if (bsp_header.collide_hash.num_items) {
     ASSERT(!got_collide);
     extract_collide_frags(bsp_header.collide_hash, all_ties, config,
-                          fmt::format("{}-{}-collide", dgo_name, i++), db.dts, level_data, false);
+                          fmt::format("{}-{}-collide", dgo_name, i++), db.dts, level_data);
   }
   level_data.level_name = level_name;
 
@@ -262,7 +262,6 @@ level_tools::BspHeader extract_bsp_from_level(const ObjectFileDB& db,
 void extract_common(const ObjectFileDB& db,
                     const TextureDB& tex_db,
                     const std::string& dgo_name,
-                    bool dump_levels,
                     const fs::path& output_folder,
                     const Config& config) {
   if (db.obj_files_by_dgo.count(dgo_name) == 0) {
@@ -325,7 +324,7 @@ void extract_common(const ObjectFileDB& db,
       output_folder / fmt::format("{}.fr3", dgo_name.substr(0, dgo_name.length() - 4)),
       compressed.data(), compressed.size());
 
-  if (dump_levels) {
+  if (config.rip_levels) {
     auto file_path = file_util::get_jak_project_dir() / "glb_out" / "common.glb";
     file_util::create_dir_if_needed_for_file(file_path);
     save_level_foreground_as_gltf(tfrag_level, art_group_data, file_path);
@@ -336,8 +335,6 @@ void extract_from_level(const ObjectFileDB& db,
                         const TextureDB& tex_db,
                         const std::string& dgo_name,
                         const Config& config,
-                        bool dump_level,
-                        bool extract_collision,
                         const fs::path& output_folder,
                         const fs::path& entities_folder) {
   if (db.obj_files_by_dgo.count(dgo_name) == 0) {
@@ -349,8 +346,7 @@ void extract_from_level(const ObjectFileDB& db,
   add_all_textures_from_level(level_data, dgo_name, tex_db);
 
   // the bsp header file data
-  auto bsp_header =
-      extract_bsp_from_level(db, tex_db, dgo_name, config, extract_collision, level_data);
+  auto bsp_header = extract_bsp_from_level(db, tex_db, dgo_name, config, level_data);
   extract_art_groups_from_level(db, tex_db, bsp_header.texture_remap_table, dgo_name, level_data,
                                 art_group_data);
 
@@ -366,7 +362,7 @@ void extract_from_level(const ObjectFileDB& db,
       output_folder / fmt::format("{}.fr3", dgo_name.substr(0, dgo_name.length() - 4)),
       compressed.data(), compressed.size());
 
-  if (dump_level) {
+  if (config.rip_levels) {
     auto back_file_path = file_util::get_jak_project_dir() / "glb_out" /
                           fmt::format("{}_background.glb", level_data.level_name);
     file_util::create_dir_if_needed_for_file(back_file_path);
@@ -385,18 +381,15 @@ void extract_all_levels(const ObjectFileDB& db,
                         const std::vector<std::string>& dgo_names,
                         const std::string& common_name,
                         const Config& config,
-                        bool debug_dump_level,
-                        bool extract_collision,
                         const fs::path& output_path) {
-  extract_common(db, tex_db, common_name, debug_dump_level, output_path, config);
+  extract_common(db, tex_db, common_name, output_path, config);
   auto entities_dir = file_util::get_jak_project_dir() / "decompiler_out" /
                       game_version_names[config.game_version] / "entities";
   file_util::create_dir_if_needed(entities_dir);
   SimpleThreadGroup threads;
   threads.run(
       [&](int idx) {
-        extract_from_level(db, tex_db, dgo_names[idx], config, debug_dump_level, extract_collision,
-                           output_path, entities_dir);
+        extract_from_level(db, tex_db, dgo_names[idx], config, output_path, entities_dir);
       },
       dgo_names.size());
   threads.join();

decompiler/level_extractor/extract_level.h

@@ -16,8 +16,6 @@ void extract_all_levels(const ObjectFileDB& db,
                         const std::vector<std::string>& dgo_names,
                         const std::string& common_name,
                         const Config& config,
-                        bool debug_dump_level,
-                        bool extract_collision,
                         const fs::path& path);
 void add_all_textures_from_level(tfrag3::Level& lev,
                                  const std::string& level_name,

decompiler/main.cpp

@@ -321,8 +321,7 @@ int main(int argc, char** argv) {
     auto level_out_path =
         file_util::get_jak_project_dir() / "out" / game_version_names[config.game_version] / "fr3";
     file_util::create_dir_if_needed(level_out_path);
-    extract_all_levels(db, tex_db, config.levels_to_extract, "GAME.CGO", config, config.rip_levels,
-                       config.extract_collision, level_out_path);
+    extract_all_levels(db, tex_db, config.levels_to_extract, "GAME.CGO", config, level_out_path);
   }
 
   mem_log("After extraction: {} MB", get_peak_rss() / (1024 * 1024));