[jak2] Set up extractor (#3042)

This sets up the extractor for jak 2. I was expecting that I'd have to
make some more significant changes to the decompiler/compiler path
stuff, but this was not the case!

The only real change is that you can now provide multiple ISO hashes for
an entry in `ISOMetadata`. This is needed for the two different NTSC
versions, which have the same configs, serials, and ELF hashes, but
slightly different contents.

I also didn't add the korean version because I don't have the info for
it.

---------

Co-authored-by: ManDude <7569514+ManDude@users.noreply.github.com>

common/util/read_iso_file.cpp

@@ -34,10 +34,18 @@ void IsoFile::Entry::print(std::string* result, const std::string& prefix) const
 namespace {
 constexpr int SECTOR_SIZE = 0x800;
 
+int fseek_64(FILE* fp, u64 offset, int origin) {
+#ifdef _WIN32
+  return _fseeki64(fp, offset, origin);
+#else
+  return fseek(fp, offset, origin);
+#endif
+}
+
 template <typename T>
 T read_file(FILE* fp, u32 sector, u32 offset_in_sector) {
   T result;
-  if (fseek(fp, sector * SECTOR_SIZE + offset_in_sector, SEEK_SET)) {
+  if (fseek_64(fp, u64(sector) * SECTOR_SIZE + offset_in_sector, SEEK_SET)) {
     ASSERT_MSG(false, "Failed to fseek iso");
   }
   if (fread(&result, sizeof(T), 1, fp) != 1) {
@@ -95,7 +103,7 @@ void unpack_entry(FILE* fp,
       lg::info("Extracting {}...", entry.name);
     }
     std::vector<u8> buffer(entry.size);
-    if (fseek(fp, entry.offset_in_file, SEEK_SET)) {
+    if (fseek_64(fp, entry.offset_in_file, SEEK_SET)) {
       ASSERT_MSG(false, "Failed to fseek iso when unpacking");
     }
     if (fread(buffer.data(), buffer.size(), 1, fp) != 1) {

decompiler/CMakeLists.txt

@@ -111,7 +111,7 @@ target_link_libraries(decompiler
 
 
 add_executable(extractor
-        extractor/main.cpp extractor/extractor_util.cpp)
+        extractor/main.cpp)
 
 target_link_libraries(extractor
         decomp

decompiler/extractor/extractor_util.cpp

@@ -15,9 +15,6 @@
 #include "third-party/json.hpp"
 #include "third-party/zstd/lib/common/xxhash.h"
 
-const std::unordered_map<std::string, GameIsoFlags> game_iso_flag_names = {
-    {"jak1-black-label", FLAG_JAK1_BLACK_LABEL}};
-
 const std::unordered_map<int, std::string> game_iso_territory_map = {
     {GAME_TERRITORY_SCEA, "NTSC-U"},
     {GAME_TERRITORY_SCEE, "PAL"},
@@ -30,47 +27,100 @@ std::string get_territory_name(int territory) {
   return game_iso_territory_map.at(territory);
 }
 
-// used for - decompiler_out/<jak1> and iso_data/<jak1>
-const std::unordered_map<std::string, std::string> data_subfolders = {{"jak1", "jak1"}};
-
 const ISOMetadata jak1_ntsc_black_label_info = {"Jak & Daxter™: The Precursor Legacy (Black Label)",
                                                 GAME_TERRITORY_SCEA,
                                                 337,
-                                                11363853835861842434U,
+                                                {11363853835861842434U},
                                                 "ntsc_v1",
                                                 "jak1",
                                                 {"jak1-black-label"}};
 
 // { SERIAL : { ELF_HASH : ISOMetadataDatabase } }
-const std::unordered_map<std::string, std::unordered_map<uint64_t, ISOMetadata>> iso_database = {
+const std::unordered_map<std::string, std::unordered_map<uint64_t, ISOMetadata>>&
-    {"SCUS-97124",
+extractor_iso_database() {
-     {{7280758013604870207U, jak1_ntsc_black_label_info},
+  static const std::unordered_map<std::string, std::unordered_map<uint64_t, ISOMetadata>> database =
-      {744661860962747854,
+      {
-       {"Jak & Daxter™: The Precursor Legacy",
+          {"SCUS-97124",
-        GAME_TERRITORY_SCEA,
+           {{7280758013604870207U, jak1_ntsc_black_label_info},
-        338,
+            {744661860962747854,
-        8538304367812415885U,
+             {"Jak & Daxter™: The Precursor Legacy",
-        "ntsc_v2",
+              GAME_TERRITORY_SCEA,
-        "jak1",
+              338,
-        {}}}}},
+              {8538304367812415885U},
-    {"SCES-50361",
+              "ntsc_v2",
-     {{12150718117852276522U,
+              "jak1",
-       {"Jak & Daxter™: The Precursor Legacy",
+              {}}}}},
-        GAME_TERRITORY_SCEE,
+          {"SCES-50361",
-        338,
+           {{12150718117852276522U,
-        16850370297611763875U,
+             {"Jak & Daxter™: The Precursor Legacy",
-        "pal",
+              GAME_TERRITORY_SCEE,
-        "jak1",
+              338,
-        {}}}}},
+              {16850370297611763875U},
-    {"SCPS-15021",
+              "pal",
-     {{16909372048085114219U,
+              "jak1",
-       {"ジャックＸダクスター　～　旧世界の遺産",
+              {}}}}},
-        GAME_TERRITORY_SCEI,
+          {"SCPS-15021",
-        338,
+           {{16909372048085114219U,
-        1262350561338887717,
+             {"ジャックＸダクスター　～　旧世界の遺産",
-        "jp",
+              GAME_TERRITORY_SCEI,
-        "jak1",
+              338,
-        {}}}}}};
+              {1262350561338887717U},
+              "jp",
+              "jak1",
+              {}}}}},
+          {"SCPS-56003",
+           {{7280758013604870207U,
+             {"Jak & Daxter: 구세계의 유산",
+              GAME_TERRITORY_SCEA,
+              338,
+              {13924540661438229398U},
+              "ntsc_v1",
+              "jak1",
+              {}}}}},
+          // Jak 2, NTSC-U v1 and v2.
+          // we put both of them together because they have the same serial and ELF.
+          {"SCUS-97265",             // serial from ELF name
+           {{18445016742498932084U,  // hash of ELF
+             {"Jak II",              // canonical name
+              GAME_TERRITORY_SCEA,
+              593,                                           // number of files
+              {4835330407820245819U, 5223305410190549348U},  // iso hash
+              "ntsc_v1",                                     // decompiler config
+              "jak2",
+              {}}}}},
+          // Jak 2 PAL
+          {"SCES-51608",             // serial from ELF name
+           {{18188891052467821088U,  // hash of ELF
+             {"Jak II: Renegade",    // canonical name
+              GAME_TERRITORY_SCEE,
+              593,                     // number of files
+              {8410801891219727031U},  // iso hash
+              "pal",                   // decompiler config
+              "jak2",
+              {}}}}},
+          // Jak 2 NTSC-J
+          {"SCPS-15057",                // serial from ELF name
+           {{7409991384254810731U,      // hash of ELF
+             {"ジャックＸダクスター2",  // canonical name
+              GAME_TERRITORY_SCEI,
+              593,                     // number of files
+              {1686904681401593185U},  // iso hash
+              "jp",                    // decompiler config
+              "jak2",
+              {}}}}},
+          // Jak 2 NTSC-K
+          {"SCKA-20010",            // serial from ELF name
+           {{8398029689314218575U,  // hash of ELF
+             {"Jak II",             // canonical name
+              GAME_TERRITORY_SCEI,
+              593,                     // number of files
+              {4637199624374114440U},  // iso hash
+              "ko",                    // decompiler config
+              "jak2",
+              {}}}}},
+      };
+  return database;
+}
 
 void to_json(nlohmann::json& j, const BuildInfo& info) {
   j = nlohmann::json{{"serial", info.serial}, {"elf_hash", info.elf_hash}};
@@ -100,8 +150,8 @@ std::optional<ISOMetadata> get_version_info_from_build_info(const BuildInfo& bui
   if (build_info.serial.empty() || build_info.elf_hash == 0) {
     return {};
   }
-  auto dbEntry = iso_database.find(build_info.serial);
+  auto dbEntry = extractor_iso_database().find(build_info.serial);
-  if (dbEntry == iso_database.end()) {
+  if (dbEntry == extractor_iso_database().end()) {
     return {};
   }
 
@@ -168,10 +218,8 @@ void log_potential_new_db_entry(ExtractorErrorCode error_code,
     lg::info(
         "If this is a new release or version that should be supported, consider adding the "
         "following serial entry to the database:");
-    lg::info(
+    lg::info("serial {}, elf hash {}, files {}, hash {}", serial, elf_hash, files_extracted,
-        "\t'{{\"{}\", {{{{{}U, {{\"GAME_TITLE\", \"NTSC-U/PAL/NTSC-J\", {}, {}U, "
+             contents_hash);
-        "\"DECOMP_CONFIG_FILENAME_NO_EXTENSION\", \"jak1|jak2|jak3|jakx\", {}}}}}}}}}'",
-        serial, elf_hash, files_extracted, contents_hash);
   } else if (error_code == ExtractorErrorCode::VALIDATION_ELF_MISSING_FROM_DB) {
     lg::info(
         "If this is a new release or version that should be supported, consider adding the "

decompiler/extractor/extractor_util.h

@@ -1,19 +1,13 @@
 #pragma once
 
 #include <optional>
-#include <regex>
+#include <set>
 #include <unordered_map>
 
-#include "common/log/log.h"
-#include "common/util/Assert.h"
 #include "common/util/FileUtil.h"
-#include "common/util/json_util.h"
 #include "common/util/read_iso_file.h"
 
-#include "game/kernel/common/kboot.h"
-
 #include "third-party/json.hpp"
-#include "third-party/zstd/lib/common/xxhash.h"
 
 enum class ExtractorErrorCode {
   SUCCESS = 0,
@@ -33,30 +27,21 @@ enum class ExtractorErrorCode {
 
 enum GameIsoFlags { FLAG_JAK1_BLACK_LABEL = (1 << 0) };
 
-extern const std::unordered_map<std::string, GameIsoFlags> game_iso_flag_names;
-
-extern const std::unordered_map<int, std::string> game_iso_territory_map;
-
-// used for - decompiler_out/<jak1> and iso_data/<jak1>
-extern const std::unordered_map<std::string, std::string> data_subfolders;
-
 std::string get_territory_name(int territory);
 
 struct ISOMetadata {
   std::string canonical_name;
   int region;  // territory code
   int num_files;
-  uint64_t contents_hash;
+  std::set<uint64_t> contents_hash;
   std::string decomp_config_version;
   std::string game_name;
   std::vector<std::string> flags;
 };
 
-extern const ISOMetadata jak1_ntsc_black_label_info;
-
 // { SERIAL : { ELF_HASH : ISOMetadataDatabase } }
-extern const std::unordered_map<std::string, std::unordered_map<uint64_t, ISOMetadata>>
+const std::unordered_map<std::string, std::unordered_map<uint64_t, ISOMetadata>>&
-    iso_database;
+extractor_iso_database();
 
 // This is all we need to re-fetch info from the database
 // - if this changes such that we have a collision in the future,

decompiler/extractor/main.cpp

@@ -1,24 +1,22 @@
-#include <map>
-#include <regex>
-#include <unordered_map>
-
-#include "extractor_util.h"
-
 #include "common/log/log.h"
+#include "common/util/Assert.h"
 #include "common/util/FileUtil.h"
-#include "common/util/json_util.h"
 #include "common/util/read_iso_file.h"
 #include "common/util/term_util.h"
 #include "common/util/unicode_util.h"
 
-#include "decompiler/Disasm/OpcodeInfo.h"
 #include "decompiler/ObjectFile/ObjectFileDB.h"
 #include "decompiler/config.h"
+#include "decompiler/extractor/extractor_util.h"
 #include "decompiler/level_extractor/extract_level.h"
 #include "goalc/compiler/Compiler.h"
 
 #include "third-party/CLI11.hpp"
 
+// used for - decompiler_out/<jak1> and iso_data/<jak1>
+const std::unordered_map<std::string, std::string> data_subfolders = {{"jak1", "jak1"},
+                                                                      {"jak2", "jak2"}};
+
 IsoFile extract_files(fs::path input_file_path, fs::path extracted_iso_path) {
   lg::info(
       "Note: Provided game data path '{}' points to a file, not a directory. Assuming it's an ISO "
@@ -52,6 +50,7 @@ std::tuple<std::optional<ISOMetadata>, ExtractorErrorCode> validate(
   }
 
   // Find the game in our tracking database
+  const auto& iso_database = extractor_iso_database();
   auto dbEntry = iso_database.find(serial.value());
   if (dbEntry == iso_database.end()) {
     lg::error("Serial '{}' not found in the validation database", serial.value());
@@ -87,9 +86,13 @@ std::tuple<std::optional<ISOMetadata>, ExtractorErrorCode> validate(
     return {std::nullopt, ExtractorErrorCode::VALIDATION_INCORRECT_EXTRACTION_COUNT};
   }
   // Check the ISO Hash
-  if (version_info.contents_hash != expected_hash) {
+  if (version_info.contents_hash.count(expected_hash) == 0) {
-    lg::error("Overall ISO content's hash does not match. Expected '{}', Actual '{}'",
+    std::string all_expected;
-              version_info.contents_hash, expected_hash);
+    for (const auto& hash : version_info.contents_hash) {
+      all_expected += fmt::format("{}, ", hash);
+    }
+    lg::error("Overall ISO content's hash does not match. Expected '{}', Actual '{}'", all_expected,
+              expected_hash);
     return {std::nullopt, ExtractorErrorCode::VALIDATION_FILE_CONTENTS_UNEXPECTED};
   }
 
@@ -193,6 +196,9 @@ void decompile(const fs::path& iso_data_path, const std::string& data_subfolder)
   }
 }
 
+const std::unordered_map<std::string, GameIsoFlags> game_iso_flag_names = {
+    {"jak1-black-label", FLAG_JAK1_BLACK_LABEL}};
+
 ExtractorErrorCode compile(const fs::path& iso_data_path, const std::string& data_subfolder) {
   // Determine which config to use from the database
   const auto version_info = get_version_info_or_default(iso_data_path);
@@ -226,8 +232,10 @@ ExtractorErrorCode compile(const fs::path& iso_data_path, const std::string& dat
   return ExtractorErrorCode::SUCCESS;
 }
 
-void launch_game() {
+void launch_game(const std::string& game_version) {
-  system(fmt::format("\"{}\"", (file_util::get_jak_project_dir() / "../gk").string()).c_str());
+  system(fmt::format("\"{}\" -g {}", (file_util::get_jak_project_dir() / "../gk").string(),
+                     game_version)
+             .c_str());
 }
 
 int main(int argc, char** argv) {
@@ -252,7 +260,7 @@ int main(int argc, char** argv) {
       ->required();
   app.add_option("--proj-path", project_path_override,
                  "Explicitly set the location of the 'data/' folder");
-  app.add_flag("-g,--game", game_name, "Specify the game name, defaults to 'jak1'");
+  app.add_option("-g,--game", game_name, "Specify the game name, defaults to 'jak1'");
   app.add_flag("-a,--all", flag_runall, "Run all steps, from extraction to playing the game");
   app.add_flag("-e,--extract", flag_extract, "Extract the ISO");
   app.add_flag("-v,--validate", flag_fail_on_validation,
@@ -413,7 +421,7 @@ int main(int argc, char** argv) {
   }
 
   if (flag_play) {
-    launch_game();
+    launch_game(game_name);
   }
 
   return 0;

lsp/protocol/completion.cpp

@@ -9,6 +9,6 @@ void LSPSpec::from_json(const json& j, CompletionParams& obj) {
   j.at("position").get_to(obj.m_position);
 }
 
-void LSPSpec::to_json(json& j, const CompletionList& obj) {}
+void LSPSpec::to_json(json& /*j*/, const CompletionList& /*obj*/) {}
 
-void LSPSpec::from_json(const json& j, CompletionList& obj) {}
+void LSPSpec::from_json(const json& /*j*/, CompletionList& /*obj*/) {}