mirror of
https://github.com/open-goal/jak-project.git
synced 2024-10-20 00:57:44 -04:00
[jak2] Set up extractor (#3042)
This sets up the extractor for jak 2. I was expecting that I'd have to make some more significant changes to the decompiler/compiler path stuff, but this was not the case! The only real change is that you can now provide multiple ISO hashes for an entry in `ISOMetadata`. This is needed for the two different NTSC versions, which have the same configs, serials, and ELF hashes, but slightly different contents. I also didn't add the korean version because I don't have the info for it. --------- Co-authored-by: ManDude <7569514+ManDude@users.noreply.github.com>
This commit is contained in:
parent
3c27b3942b
commit
af6f489657
|
@ -34,10 +34,18 @@ void IsoFile::Entry::print(std::string* result, const std::string& prefix) const
|
|||
namespace {
|
||||
constexpr int SECTOR_SIZE = 0x800;
|
||||
|
||||
int fseek_64(FILE* fp, u64 offset, int origin) {
|
||||
#ifdef _WIN32
|
||||
return _fseeki64(fp, offset, origin);
|
||||
#else
|
||||
return fseek(fp, offset, origin);
|
||||
#endif
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
T read_file(FILE* fp, u32 sector, u32 offset_in_sector) {
|
||||
T result;
|
||||
if (fseek(fp, sector * SECTOR_SIZE + offset_in_sector, SEEK_SET)) {
|
||||
if (fseek_64(fp, u64(sector) * SECTOR_SIZE + offset_in_sector, SEEK_SET)) {
|
||||
ASSERT_MSG(false, "Failed to fseek iso");
|
||||
}
|
||||
if (fread(&result, sizeof(T), 1, fp) != 1) {
|
||||
|
@ -95,7 +103,7 @@ void unpack_entry(FILE* fp,
|
|||
lg::info("Extracting {}...", entry.name);
|
||||
}
|
||||
std::vector<u8> buffer(entry.size);
|
||||
if (fseek(fp, entry.offset_in_file, SEEK_SET)) {
|
||||
if (fseek_64(fp, entry.offset_in_file, SEEK_SET)) {
|
||||
ASSERT_MSG(false, "Failed to fseek iso when unpacking");
|
||||
}
|
||||
if (fread(buffer.data(), buffer.size(), 1, fp) != 1) {
|
||||
|
|
|
@ -111,7 +111,7 @@ target_link_libraries(decompiler
|
|||
|
||||
|
||||
add_executable(extractor
|
||||
extractor/main.cpp extractor/extractor_util.cpp)
|
||||
extractor/main.cpp)
|
||||
|
||||
target_link_libraries(extractor
|
||||
decomp
|
||||
|
|
|
@ -15,9 +15,6 @@
|
|||
#include "third-party/json.hpp"
|
||||
#include "third-party/zstd/lib/common/xxhash.h"
|
||||
|
||||
const std::unordered_map<std::string, GameIsoFlags> game_iso_flag_names = {
|
||||
{"jak1-black-label", FLAG_JAK1_BLACK_LABEL}};
|
||||
|
||||
const std::unordered_map<int, std::string> game_iso_territory_map = {
|
||||
{GAME_TERRITORY_SCEA, "NTSC-U"},
|
||||
{GAME_TERRITORY_SCEE, "PAL"},
|
||||
|
@ -30,47 +27,100 @@ std::string get_territory_name(int territory) {
|
|||
return game_iso_territory_map.at(territory);
|
||||
}
|
||||
|
||||
// used for - decompiler_out/<jak1> and iso_data/<jak1>
|
||||
const std::unordered_map<std::string, std::string> data_subfolders = {{"jak1", "jak1"}};
|
||||
|
||||
const ISOMetadata jak1_ntsc_black_label_info = {"Jak & Daxter™: The Precursor Legacy (Black Label)",
|
||||
GAME_TERRITORY_SCEA,
|
||||
337,
|
||||
11363853835861842434U,
|
||||
{11363853835861842434U},
|
||||
"ntsc_v1",
|
||||
"jak1",
|
||||
{"jak1-black-label"}};
|
||||
|
||||
// { SERIAL : { ELF_HASH : ISOMetadataDatabase } }
|
||||
const std::unordered_map<std::string, std::unordered_map<uint64_t, ISOMetadata>> iso_database = {
|
||||
{"SCUS-97124",
|
||||
{{7280758013604870207U, jak1_ntsc_black_label_info},
|
||||
{744661860962747854,
|
||||
{"Jak & Daxter™: The Precursor Legacy",
|
||||
GAME_TERRITORY_SCEA,
|
||||
338,
|
||||
8538304367812415885U,
|
||||
"ntsc_v2",
|
||||
"jak1",
|
||||
{}}}}},
|
||||
{"SCES-50361",
|
||||
{{12150718117852276522U,
|
||||
{"Jak & Daxter™: The Precursor Legacy",
|
||||
GAME_TERRITORY_SCEE,
|
||||
338,
|
||||
16850370297611763875U,
|
||||
"pal",
|
||||
"jak1",
|
||||
{}}}}},
|
||||
{"SCPS-15021",
|
||||
{{16909372048085114219U,
|
||||
{"ジャックXダクスター ~ 旧世界の遺産",
|
||||
GAME_TERRITORY_SCEI,
|
||||
338,
|
||||
1262350561338887717,
|
||||
"jp",
|
||||
"jak1",
|
||||
{}}}}}};
|
||||
const std::unordered_map<std::string, std::unordered_map<uint64_t, ISOMetadata>>&
|
||||
extractor_iso_database() {
|
||||
static const std::unordered_map<std::string, std::unordered_map<uint64_t, ISOMetadata>> database =
|
||||
{
|
||||
{"SCUS-97124",
|
||||
{{7280758013604870207U, jak1_ntsc_black_label_info},
|
||||
{744661860962747854,
|
||||
{"Jak & Daxter™: The Precursor Legacy",
|
||||
GAME_TERRITORY_SCEA,
|
||||
338,
|
||||
{8538304367812415885U},
|
||||
"ntsc_v2",
|
||||
"jak1",
|
||||
{}}}}},
|
||||
{"SCES-50361",
|
||||
{{12150718117852276522U,
|
||||
{"Jak & Daxter™: The Precursor Legacy",
|
||||
GAME_TERRITORY_SCEE,
|
||||
338,
|
||||
{16850370297611763875U},
|
||||
"pal",
|
||||
"jak1",
|
||||
{}}}}},
|
||||
{"SCPS-15021",
|
||||
{{16909372048085114219U,
|
||||
{"ジャックXダクスター ~ 旧世界の遺産",
|
||||
GAME_TERRITORY_SCEI,
|
||||
338,
|
||||
{1262350561338887717U},
|
||||
"jp",
|
||||
"jak1",
|
||||
{}}}}},
|
||||
{"SCPS-56003",
|
||||
{{7280758013604870207U,
|
||||
{"Jak & Daxter: 구세계의 유산",
|
||||
GAME_TERRITORY_SCEA,
|
||||
338,
|
||||
{13924540661438229398U},
|
||||
"ntsc_v1",
|
||||
"jak1",
|
||||
{}}}}},
|
||||
// Jak 2, NTSC-U v1 and v2.
|
||||
// we put both of them together because they have the same serial and ELF.
|
||||
{"SCUS-97265", // serial from ELF name
|
||||
{{18445016742498932084U, // hash of ELF
|
||||
{"Jak II", // canonical name
|
||||
GAME_TERRITORY_SCEA,
|
||||
593, // number of files
|
||||
{4835330407820245819U, 5223305410190549348U}, // iso hash
|
||||
"ntsc_v1", // decompiler config
|
||||
"jak2",
|
||||
{}}}}},
|
||||
// Jak 2 PAL
|
||||
{"SCES-51608", // serial from ELF name
|
||||
{{18188891052467821088U, // hash of ELF
|
||||
{"Jak II: Renegade", // canonical name
|
||||
GAME_TERRITORY_SCEE,
|
||||
593, // number of files
|
||||
{8410801891219727031U}, // iso hash
|
||||
"pal", // decompiler config
|
||||
"jak2",
|
||||
{}}}}},
|
||||
// Jak 2 NTSC-J
|
||||
{"SCPS-15057", // serial from ELF name
|
||||
{{7409991384254810731U, // hash of ELF
|
||||
{"ジャックXダクスター2", // canonical name
|
||||
GAME_TERRITORY_SCEI,
|
||||
593, // number of files
|
||||
{1686904681401593185U}, // iso hash
|
||||
"jp", // decompiler config
|
||||
"jak2",
|
||||
{}}}}},
|
||||
// Jak 2 NTSC-K
|
||||
{"SCKA-20010", // serial from ELF name
|
||||
{{8398029689314218575U, // hash of ELF
|
||||
{"Jak II", // canonical name
|
||||
GAME_TERRITORY_SCEI,
|
||||
593, // number of files
|
||||
{4637199624374114440U}, // iso hash
|
||||
"ko", // decompiler config
|
||||
"jak2",
|
||||
{}}}}},
|
||||
};
|
||||
return database;
|
||||
}
|
||||
|
||||
void to_json(nlohmann::json& j, const BuildInfo& info) {
|
||||
j = nlohmann::json{{"serial", info.serial}, {"elf_hash", info.elf_hash}};
|
||||
|
@ -100,8 +150,8 @@ std::optional<ISOMetadata> get_version_info_from_build_info(const BuildInfo& bui
|
|||
if (build_info.serial.empty() || build_info.elf_hash == 0) {
|
||||
return {};
|
||||
}
|
||||
auto dbEntry = iso_database.find(build_info.serial);
|
||||
if (dbEntry == iso_database.end()) {
|
||||
auto dbEntry = extractor_iso_database().find(build_info.serial);
|
||||
if (dbEntry == extractor_iso_database().end()) {
|
||||
return {};
|
||||
}
|
||||
|
||||
|
@ -168,10 +218,8 @@ void log_potential_new_db_entry(ExtractorErrorCode error_code,
|
|||
lg::info(
|
||||
"If this is a new release or version that should be supported, consider adding the "
|
||||
"following serial entry to the database:");
|
||||
lg::info(
|
||||
"\t'{{\"{}\", {{{{{}U, {{\"GAME_TITLE\", \"NTSC-U/PAL/NTSC-J\", {}, {}U, "
|
||||
"\"DECOMP_CONFIG_FILENAME_NO_EXTENSION\", \"jak1|jak2|jak3|jakx\", {}}}}}}}}}'",
|
||||
serial, elf_hash, files_extracted, contents_hash);
|
||||
lg::info("serial {}, elf hash {}, files {}, hash {}", serial, elf_hash, files_extracted,
|
||||
contents_hash);
|
||||
} else if (error_code == ExtractorErrorCode::VALIDATION_ELF_MISSING_FROM_DB) {
|
||||
lg::info(
|
||||
"If this is a new release or version that should be supported, consider adding the "
|
||||
|
|
|
@ -1,19 +1,13 @@
|
|||
#pragma once
|
||||
|
||||
#include <optional>
|
||||
#include <regex>
|
||||
#include <set>
|
||||
#include <unordered_map>
|
||||
|
||||
#include "common/log/log.h"
|
||||
#include "common/util/Assert.h"
|
||||
#include "common/util/FileUtil.h"
|
||||
#include "common/util/json_util.h"
|
||||
#include "common/util/read_iso_file.h"
|
||||
|
||||
#include "game/kernel/common/kboot.h"
|
||||
|
||||
#include "third-party/json.hpp"
|
||||
#include "third-party/zstd/lib/common/xxhash.h"
|
||||
|
||||
enum class ExtractorErrorCode {
|
||||
SUCCESS = 0,
|
||||
|
@ -33,30 +27,21 @@ enum class ExtractorErrorCode {
|
|||
|
||||
enum GameIsoFlags { FLAG_JAK1_BLACK_LABEL = (1 << 0) };
|
||||
|
||||
extern const std::unordered_map<std::string, GameIsoFlags> game_iso_flag_names;
|
||||
|
||||
extern const std::unordered_map<int, std::string> game_iso_territory_map;
|
||||
|
||||
// used for - decompiler_out/<jak1> and iso_data/<jak1>
|
||||
extern const std::unordered_map<std::string, std::string> data_subfolders;
|
||||
|
||||
std::string get_territory_name(int territory);
|
||||
|
||||
struct ISOMetadata {
|
||||
std::string canonical_name;
|
||||
int region; // territory code
|
||||
int num_files;
|
||||
uint64_t contents_hash;
|
||||
std::set<uint64_t> contents_hash;
|
||||
std::string decomp_config_version;
|
||||
std::string game_name;
|
||||
std::vector<std::string> flags;
|
||||
};
|
||||
|
||||
extern const ISOMetadata jak1_ntsc_black_label_info;
|
||||
|
||||
// { SERIAL : { ELF_HASH : ISOMetadataDatabase } }
|
||||
extern const std::unordered_map<std::string, std::unordered_map<uint64_t, ISOMetadata>>
|
||||
iso_database;
|
||||
const std::unordered_map<std::string, std::unordered_map<uint64_t, ISOMetadata>>&
|
||||
extractor_iso_database();
|
||||
|
||||
// This is all we need to re-fetch info from the database
|
||||
// - if this changes such that we have a collision in the future,
|
||||
|
|
|
@ -1,24 +1,22 @@
|
|||
#include <map>
|
||||
#include <regex>
|
||||
#include <unordered_map>
|
||||
|
||||
#include "extractor_util.h"
|
||||
|
||||
#include "common/log/log.h"
|
||||
#include "common/util/Assert.h"
|
||||
#include "common/util/FileUtil.h"
|
||||
#include "common/util/json_util.h"
|
||||
#include "common/util/read_iso_file.h"
|
||||
#include "common/util/term_util.h"
|
||||
#include "common/util/unicode_util.h"
|
||||
|
||||
#include "decompiler/Disasm/OpcodeInfo.h"
|
||||
#include "decompiler/ObjectFile/ObjectFileDB.h"
|
||||
#include "decompiler/config.h"
|
||||
#include "decompiler/extractor/extractor_util.h"
|
||||
#include "decompiler/level_extractor/extract_level.h"
|
||||
#include "goalc/compiler/Compiler.h"
|
||||
|
||||
#include "third-party/CLI11.hpp"
|
||||
|
||||
// used for - decompiler_out/<jak1> and iso_data/<jak1>
|
||||
const std::unordered_map<std::string, std::string> data_subfolders = {{"jak1", "jak1"},
|
||||
{"jak2", "jak2"}};
|
||||
|
||||
IsoFile extract_files(fs::path input_file_path, fs::path extracted_iso_path) {
|
||||
lg::info(
|
||||
"Note: Provided game data path '{}' points to a file, not a directory. Assuming it's an ISO "
|
||||
|
@ -52,6 +50,7 @@ std::tuple<std::optional<ISOMetadata>, ExtractorErrorCode> validate(
|
|||
}
|
||||
|
||||
// Find the game in our tracking database
|
||||
const auto& iso_database = extractor_iso_database();
|
||||
auto dbEntry = iso_database.find(serial.value());
|
||||
if (dbEntry == iso_database.end()) {
|
||||
lg::error("Serial '{}' not found in the validation database", serial.value());
|
||||
|
@ -87,9 +86,13 @@ std::tuple<std::optional<ISOMetadata>, ExtractorErrorCode> validate(
|
|||
return {std::nullopt, ExtractorErrorCode::VALIDATION_INCORRECT_EXTRACTION_COUNT};
|
||||
}
|
||||
// Check the ISO Hash
|
||||
if (version_info.contents_hash != expected_hash) {
|
||||
lg::error("Overall ISO content's hash does not match. Expected '{}', Actual '{}'",
|
||||
version_info.contents_hash, expected_hash);
|
||||
if (version_info.contents_hash.count(expected_hash) == 0) {
|
||||
std::string all_expected;
|
||||
for (const auto& hash : version_info.contents_hash) {
|
||||
all_expected += fmt::format("{}, ", hash);
|
||||
}
|
||||
lg::error("Overall ISO content's hash does not match. Expected '{}', Actual '{}'", all_expected,
|
||||
expected_hash);
|
||||
return {std::nullopt, ExtractorErrorCode::VALIDATION_FILE_CONTENTS_UNEXPECTED};
|
||||
}
|
||||
|
||||
|
@ -193,6 +196,9 @@ void decompile(const fs::path& iso_data_path, const std::string& data_subfolder)
|
|||
}
|
||||
}
|
||||
|
||||
const std::unordered_map<std::string, GameIsoFlags> game_iso_flag_names = {
|
||||
{"jak1-black-label", FLAG_JAK1_BLACK_LABEL}};
|
||||
|
||||
ExtractorErrorCode compile(const fs::path& iso_data_path, const std::string& data_subfolder) {
|
||||
// Determine which config to use from the database
|
||||
const auto version_info = get_version_info_or_default(iso_data_path);
|
||||
|
@ -226,8 +232,10 @@ ExtractorErrorCode compile(const fs::path& iso_data_path, const std::string& dat
|
|||
return ExtractorErrorCode::SUCCESS;
|
||||
}
|
||||
|
||||
void launch_game() {
|
||||
system(fmt::format("\"{}\"", (file_util::get_jak_project_dir() / "../gk").string()).c_str());
|
||||
void launch_game(const std::string& game_version) {
|
||||
system(fmt::format("\"{}\" -g {}", (file_util::get_jak_project_dir() / "../gk").string(),
|
||||
game_version)
|
||||
.c_str());
|
||||
}
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
|
@ -252,7 +260,7 @@ int main(int argc, char** argv) {
|
|||
->required();
|
||||
app.add_option("--proj-path", project_path_override,
|
||||
"Explicitly set the location of the 'data/' folder");
|
||||
app.add_flag("-g,--game", game_name, "Specify the game name, defaults to 'jak1'");
|
||||
app.add_option("-g,--game", game_name, "Specify the game name, defaults to 'jak1'");
|
||||
app.add_flag("-a,--all", flag_runall, "Run all steps, from extraction to playing the game");
|
||||
app.add_flag("-e,--extract", flag_extract, "Extract the ISO");
|
||||
app.add_flag("-v,--validate", flag_fail_on_validation,
|
||||
|
@ -413,7 +421,7 @@ int main(int argc, char** argv) {
|
|||
}
|
||||
|
||||
if (flag_play) {
|
||||
launch_game();
|
||||
launch_game(game_name);
|
||||
}
|
||||
|
||||
return 0;
|
||||
|
|
|
@ -9,6 +9,6 @@ void LSPSpec::from_json(const json& j, CompletionParams& obj) {
|
|||
j.at("position").get_to(obj.m_position);
|
||||
}
|
||||
|
||||
void LSPSpec::to_json(json& j, const CompletionList& obj) {}
|
||||
void LSPSpec::to_json(json& /*j*/, const CompletionList& /*obj*/) {}
|
||||
|
||||
void LSPSpec::from_json(const json& j, CompletionList& obj) {}
|
||||
void LSPSpec::from_json(const json& /*j*/, CompletionList& /*obj*/) {}
|
||||
|
|
Loading…
Reference in a new issue