extractor: support extracting using a folder path (#3422)

Patching up the extractor while working on the launcher, fixes:
- makes it so you can compile successfully given a folder path
(currently assumes your project path contains `iso_data`)
- ignore `buildinfo.json` from validation code.
- fixes an edge-case that could recursively fill up your entire
hard-drive!
- allows overriding the decompilation configuration via flag
- adds a way to specify where the ISO should be extracted to

common/util/FileUtil.cpp

@@ -110,6 +110,8 @@ struct {
   fs::path path_to_data;
 } gFilePathInfo;
 
+fs::path g_iso_data_directory = "";
+
 /*!
  * Get the path to the current executable.
  */
@@ -207,6 +209,28 @@ fs::path get_jak_project_dir() {
   return gFilePathInfo.path_to_data;
 }
 
+fs::path get_iso_dir_for_game(GameVersion game_version) {
+  if (!g_iso_data_directory.empty()) {
+    return g_iso_data_directory;
+  }
+  // Find the location based on the game version
+  std::string expected_subdir = "jak1";
+  if (game_version == GameVersion::Jak2) {
+    expected_subdir = "jak2";
+  } else if (game_version == GameVersion::Jak3) {
+    expected_subdir = "jak3";
+  }
+  const auto temp_dir = get_jak_project_dir() / "iso_data" / expected_subdir;
+  if (fs::exists(temp_dir)) {
+    g_iso_data_directory = temp_dir;
+  }
+  return g_iso_data_directory;
+}
+
+void set_iso_data_dir(const fs::path& directory) {
+  g_iso_data_directory = directory;
+}
+
 std::string get_file_path(const std::vector<std::string>& input) {
   // TODO - clean this behaviour up, it causes unexpected behaviour when working with files
   // the project path should be explicitly provided by whatever if needed
@@ -757,4 +781,9 @@ std::pair<int, std::string> get_majority_file_line_endings_and_count(
   return {lf_count + crlf_count, "\n"};
 }
 
+bool is_dir_in_dir(const fs::path& parent, const fs::path& child) {
+  // Check if the parent path is a prefix of the child path
+  return child.has_parent_path() && child.parent_path().lexically_relative(parent) == fs::path(".");
+}
+
 }  // namespace file_util

common/util/FileUtil.h

@@ -35,6 +35,8 @@ fs::path get_user_screenshots_dir(GameVersion game_version);
 fs::path get_user_misc_dir(GameVersion game_version);
 fs::path get_user_features_dir(GameVersion game_version);
 fs::path get_jak_project_dir();
+fs::path get_iso_dir_for_game(GameVersion game_version);
+void set_iso_data_dir(const fs::path& directory);
 
 bool create_dir_if_needed(const fs::path& path);
 bool create_dir_if_needed_for_file(const std::string& path);
@@ -74,4 +76,5 @@ std::string make_screenshot_filepath(const GameVersion game_version, const std::
 std::string get_majority_file_line_endings(const std::string& file_contents);
 std::pair<int, std::string> get_majority_file_line_endings_and_count(
     const std::string& file_contents);
+bool is_dir_in_dir(const fs::path& parent, const fs::path& child);
 }  // namespace file_util

decompiler/ObjectFile/ObjectFileDB.h

@@ -332,7 +332,7 @@ class ObjectFileDB {
   void for_each_function_def_order(Func f) {
     for_each_obj([&](ObjectFileData& data) {
       for (int i = 0; i < int(data.linked_data.segments); i++) {
-        int fn = 0;
+        [[maybe_unused]] int fn = 0;
         for (size_t j = data.linked_data.functions_by_seg.at(i).size(); j-- > 0;) {
           f(data.linked_data.functions_by_seg.at(i).at(j), i, data);
           fn++;
@@ -355,7 +355,7 @@ class ObjectFileDB {
   template <typename Func>
   void for_each_function_in_seg(int seg, Func f) {
     for_each_obj([&](ObjectFileData& data) {
-      int fn = 0;
+      [[maybe_unused]] int fn = 0;
       if (data.linked_data.segments == 3) {
         for (size_t j = data.linked_data.functions_by_seg.at(seg).size(); j-- > 0;) {
           f(data.linked_data.functions_by_seg.at(seg).at(j), data);

decompiler/extractor/extractor_util.cpp

@@ -267,6 +267,11 @@ std::tuple<uint64_t, int> calculate_extraction_hash(const fs::path& extracted_is
   int filec = 0;
   for (auto const& dir_entry : fs::recursive_directory_iterator(extracted_iso_path)) {
     if (dir_entry.is_regular_file()) {
+      // skip the `buildinfo.json` file, we make that -- not relevant!
+      if (dir_entry.path().filename() == "buildinfo.json") {
+        lg::warn("skipping buildinfo.json, that is a file our tools generate");
+        continue;
+      }
       auto buffer = file_util::read_binary_file(dir_entry.path().string());
       auto hash = XXH64(buffer.data(), buffer.size(), 0);
       combined_hash ^= hash;

decompiler/extractor/main.cpp

@@ -102,7 +102,10 @@ std::tuple<std::optional<ISOMetadata>, ExtractorErrorCode> validate(
   };
 }
 
-void decompile(const fs::path& iso_data_path, const std::string& data_subfolder) {
+// TODO - remove code duplication, most of this is copying what happens in decompiler's `main.cpp`
+void decompile(const fs::path& iso_data_path,
+               const std::string& data_subfolder,
+               const std::string& config_override) {
   using namespace decompiler;
 
   // Determine which config to use from the database
@@ -111,7 +114,7 @@ void decompile(const fs::path& iso_data_path, const std::string& data_subfolder)
   Config config = read_config_file(file_util::get_jak_project_dir() / "decompiler" / "config" /
                                        version_info.game_name /
                                        fmt::format("{}_config.jsonc", version_info.game_name),
-                                   version_info.decomp_config_version);
+                                   version_info.decomp_config_version, config_override);
 
   std::vector<fs::path> dgos, objs, tex_strs, art_strs;
 
@@ -220,6 +223,8 @@ ExtractorErrorCode compile(const fs::path& iso_data_path, const std::string& dat
   Compiler compiler(game_name_to_version(version_info.game_name));
   compiler.make_system().set_constant("*iso-data*", absolute(iso_data_path).string());
   compiler.make_system().set_constant("*use-iso-data-path*", true);
+  file_util::set_iso_data_dir(absolute(iso_data_path));
+  lg::info("set iso_data_dir to {}", absolute(iso_data_path).string());
 
   int flags = 0;
   for (const auto& flag : version_info.flags) {
@@ -257,6 +262,7 @@ int main(int argc, char** argv) {
 
   fs::path input_file_path;
   fs::path project_path_override;
+  fs::path extraction_path;
   bool flag_runall = false;
   bool flag_extract = false;
   bool flag_fail_on_validation = false;
@@ -265,16 +271,22 @@ int main(int argc, char** argv) {
   bool flag_play = false;
   bool flag_folder = false;
   std::string game_name = "jak1";
+  std::string decomp_config_override = "{}";
 
   lg::initialize();
 
-  CLI::App app{"OpenGOAL Level Extraction Tool"};
+  CLI::App app{"OpenGOAL Extractor (ISO Tools + Decompiler + Compiler)"};
   app.add_option("game-files-path", input_file_path,
                  "The path to the folder with the ISO extracted or the ISO itself")
       ->required();
   app.add_option("--proj-path", project_path_override,
                  "Explicitly set the location of the 'data/' folder");
+  app.add_option("--extract-path", extraction_path,
+                 "Explicitly set the location for where the ISO should be extracted");
   app.add_option("-g,--game", game_name, "Specify the game name, defaults to 'jak1'");
+  app.add_option(
+      "--decomp-config-override", decomp_config_override,
+      "JSON provided will be merged with the decompiler config, use to override options");
   app.add_flag("-a,--all", flag_runall, "Run all steps, from extraction to playing the game");
   app.add_flag("-e,--extract", flag_extract, "Extract the ISO");
   app.add_flag("-v,--validate", flag_fail_on_validation,
@@ -282,7 +294,7 @@ int main(int argc, char** argv) {
   app.add_flag("-d,--decompile", flag_decompile, "Decompile the game data");
   app.add_flag("-c,--compile", flag_compile, "Compile the game");
   app.add_flag("-p,--play", flag_play, "Play the game");
-  app.add_flag("-f,--folder", flag_folder, "Extract from folder");
+  app.add_flag("-f,--folder", flag_folder, "Take ISO input from a folder");
   define_common_cli_arguments(app);
   app.validate_positionals();
   CLI11_PARSE(app, argc, argv);
@@ -347,6 +359,10 @@ int main(int argc, char** argv) {
   if (flag_extract) {
     // we extract to a temporary location because we don't know what we're extracting yet!
     fs::path temp_iso_extract_location = file_util::get_jak_project_dir() / "iso_data" / "_temp";
+    if (!extraction_path.empty()) {
+      temp_iso_extract_location = extraction_path / "_temp";
+    }
+    lg::info("Extracting ISO to temporary dir at: {}", temp_iso_extract_location.string());
     if (input_file_path != temp_iso_extract_location) {
       // in case input is also output, don't just wipe everything (weird)
       fs::remove_all(temp_iso_extract_location);
@@ -379,14 +395,25 @@ int main(int argc, char** argv) {
         // We know the version since we just extracted it, so the user didn't need to provide this
         // explicitly
         data_subfolder = data_subfolders.at(version_info->game_name);
-        iso_data_path = file_util::get_jak_project_dir() / "iso_data" / data_subfolder;
-        if (fs::exists(iso_data_path)) {
+        if (!extraction_path.empty()) {
+          iso_data_path = extraction_path / data_subfolder;
+        } else {
+          iso_data_path = file_util::get_jak_project_dir() / "iso_data" / data_subfolder;
+        }
+        if (fs::exists(iso_data_path) && iso_data_path != temp_iso_extract_location) {
           fs::remove_all(iso_data_path);
         }
 
         // std::filesystem doesn't have a rename for dirs...
-        fs::copy(temp_iso_extract_location, iso_data_path, fs::copy_options::recursive);
-        fs::remove_all(temp_iso_extract_location);
+        // NOTE - potential disaster here, don't do either if the directories are the same location
+        // or don't copy if the temp location is _inside_ the destination directory
+        if (!file_util::is_dir_in_dir(iso_data_path, temp_iso_extract_location)) {
+          fs::copy(temp_iso_extract_location, iso_data_path, fs::copy_options::recursive);
+        }
+        if (iso_data_path != temp_iso_extract_location) {
+          // in case input is also output, don't just wipe everything (weird)
+          fs::remove_all(temp_iso_extract_location);
+        }
       }
     } else if (fs::is_directory(input_file_path)) {
       if (!flag_folder) {
@@ -418,12 +445,16 @@ int main(int argc, char** argv) {
   } else {
     // If we did not extract, we have no clue what game the user is trying to decompile / compile
     // this is why the user has to specify this!
-    iso_data_path = file_util::get_jak_project_dir() / "iso_data" / data_subfolder;
+    if (flag_folder) {
+      iso_data_path = input_file_path;
+    } else {
+      iso_data_path = file_util::get_jak_project_dir() / "iso_data" / data_subfolder;
+    }
   }
 
   if (flag_decompile) {
     try {
-      decompile(iso_data_path, data_subfolder);
+      decompile(iso_data_path, data_subfolder, decomp_config_override);
     } catch (std::exception& e) {
       lg::error("Error during decompile: {}", e.what());
       return static_cast<int>(ExtractorErrorCode::DECOMPILATION_GENERIC_ERROR);

game/system/hid/devices/keyboard.h

@@ -31,9 +31,11 @@ class KeyboardDevice : public InputDevice {
                      const CommandBindingGroups& commands,
                      std::shared_ptr<PadData> data,
                      std::optional<InputBindAssignmentMeta>& bind_assignment) override;
-  void close_device() override{
-      // there is nothing to close
+  // clang-format off
+  void close_device() override {
+    // there is nothing to close
   };
+  // clang-format on
 
  private:
   std::vector<ActiveKeyboardAction> m_active_actions = {};

game/system/hid/devices/mouse.h

@@ -31,9 +31,11 @@ class MouseDevice : public InputDevice {
                      const CommandBindingGroups& commands,
                      std::shared_ptr<PadData> data,
                      std::optional<InputBindAssignmentMeta>& bind_assignment) override;
-  void close_device() override{
-      // there is nothing to close
+  // clang-format off
+  void close_device() override {
+    // there is nothing to close
   };
+  // clang-format on
 
   void enable_relative_mode(const bool enable);
   void enable_camera_control(const bool enable);

goalc/build_level/jak1/build_level.cpp

@@ -104,18 +104,9 @@ bool run_build_level(const std::string& input_file,
   // TODO remove hardcoded config settings
   if ((level_json.contains("art_groups") && !level_json.at("art_groups").empty()) ||
       (level_json.contains("textures") && !level_json.at("textures").empty())) {
-    fs::path iso_folder = "";
     lg::info("Looking for ISO path...");
-    // TODO - add to file_util
-    for (const auto& entry :
-         fs::directory_iterator(file_util::get_jak_project_dir() / "iso_data")) {
-      // TODO - hard-coded to jak 1
-      if (entry.is_directory() &&
-          entry.path().filename().string().find("jak1") != std::string::npos) {
-        lg::info("Found ISO path: {}", entry.path().string());
-        iso_folder = entry.path();
-      }
-    }
+    const auto iso_folder = file_util::get_iso_dir_for_game(GameVersion::Jak1);
+    lg::info("Found ISO path: {}", iso_folder.string());
 
     if (iso_folder.empty() || !fs::exists(iso_folder)) {
       lg::warn("Could not locate ISO path!");
@@ -217,4 +208,4 @@ bool run_build_level(const std::string& input_file,
 
   return true;
 }
-}  // namespace jak1
+}  // namespace jak1

goalc/build_level/jak2/build_level.cpp

@@ -91,18 +91,9 @@ bool run_build_level(const std::string& input_file,
   // TODO remove hardcoded config settings
   if ((level_json.contains("art_groups") && !level_json.at("art_groups").empty()) ||
       (level_json.contains("textures") && !level_json.at("textures").empty())) {
-    fs::path iso_folder = "";
     lg::info("Looking for ISO path...");
-    // TODO - add to file_util
-    for (const auto& entry :
-         fs::directory_iterator(file_util::get_jak_project_dir() / "iso_data")) {
-      // TODO - hard-coded to jak 2
-      if (entry.is_directory() &&
-          entry.path().filename().string().find("jak2") != std::string::npos) {
-        lg::info("Found ISO path: {}", entry.path().string());
-        iso_folder = entry.path();
-      }
-    }
+    const auto iso_folder = file_util::get_iso_dir_for_game(GameVersion::Jak2);
+    lg::info("Found ISO path: {}", iso_folder.string());
 
     if (iso_folder.empty() || !fs::exists(iso_folder)) {
       lg::warn("Could not locate ISO path!");
@@ -203,4 +194,4 @@ bool run_build_level(const std::string& input_file,
 
   return true;
 }
-}  // namespace jak2
+}  // namespace jak2