2021-12-04 12:33:18 -05:00
|
|
|
#include <set>
|
2022-04-15 20:40:10 -04:00
|
|
|
#include <thread>
|
2021-12-04 12:33:18 -05:00
|
|
|
|
|
|
|
#include "extract_level.h"
|
|
|
|
#include "decompiler/level_extractor/BspHeader.h"
|
|
|
|
#include "decompiler/level_extractor/extract_tfrag.h"
|
2021-12-26 12:33:51 -05:00
|
|
|
#include "decompiler/level_extractor/extract_tie.h"
|
2022-03-28 18:14:25 -04:00
|
|
|
#include "decompiler/level_extractor/extract_shrub.h"
|
2022-04-25 21:53:23 -04:00
|
|
|
#include "decompiler/level_extractor/extract_collide_frags.h"
|
2022-05-11 22:53:53 -04:00
|
|
|
#include "decompiler/level_extractor/extract_merc.h"
|
2022-01-07 11:52:24 -05:00
|
|
|
#include "common/util/compress.h"
|
2021-12-04 12:33:18 -05:00
|
|
|
#include "common/util/FileUtil.h"
|
2022-04-15 20:40:10 -04:00
|
|
|
#include "common/util/SimpleThreadGroup.h"
|
2021-12-04 12:33:18 -05:00
|
|
|
|
|
|
|
namespace decompiler {
|
|
|
|
|
|
|
|
/*!
|
|
|
|
* Look through files in a DGO and find the bsp-header file (the level)
|
|
|
|
*/
|
2021-12-04 16:06:01 -05:00
|
|
|
std::optional<ObjectFileRecord> get_bsp_file(const std::vector<ObjectFileRecord>& records) {
|
|
|
|
std::optional<ObjectFileRecord> result;
|
2021-12-04 12:33:18 -05:00
|
|
|
bool found = false;
|
|
|
|
for (auto& file : records) {
|
|
|
|
if (file.name.length() > 4 && file.name.substr(file.name.length() - 4) == "-vis") {
|
2022-02-08 19:02:47 -05:00
|
|
|
ASSERT(!found);
|
2021-12-04 12:33:18 -05:00
|
|
|
found = true;
|
|
|
|
result = file;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*!
|
|
|
|
* Make sure a file is a valid bsp-header.
|
|
|
|
*/
|
|
|
|
bool is_valid_bsp(const decompiler::LinkedObjectFile& file) {
|
|
|
|
if (file.segments != 1) {
|
|
|
|
fmt::print("Got {} segments, but expected 1\n", file.segments);
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
auto& first_word = file.words_by_seg.at(0).at(0);
|
|
|
|
if (first_word.kind() != decompiler::LinkedWord::TYPE_PTR) {
|
|
|
|
fmt::print("Expected the first word to be a type pointer, but it wasn't.\n");
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (first_word.symbol_name() != "bsp-header") {
|
|
|
|
fmt::print("Expected to get a bsp-header, but got {} instead.\n", first_word.symbol_name());
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
2022-03-02 20:01:37 -05:00
|
|
|
void add_all_textures_from_level(tfrag3::Level& lev,
|
|
|
|
const std::string& level_name,
|
2022-04-15 20:40:10 -04:00
|
|
|
const TextureDB& tex_db) {
|
2022-03-02 20:01:37 -05:00
|
|
|
ASSERT(lev.textures.empty());
|
2022-04-15 20:40:10 -04:00
|
|
|
const auto& level_it = tex_db.texture_ids_per_level.find(level_name);
|
|
|
|
if (level_it != tex_db.texture_ids_per_level.end()) {
|
|
|
|
for (auto id : level_it->second) {
|
|
|
|
const auto& tex = tex_db.textures.at(id);
|
|
|
|
lev.textures.emplace_back();
|
|
|
|
auto& new_tex = lev.textures.back();
|
|
|
|
new_tex.combo_id = id;
|
|
|
|
new_tex.w = tex.w;
|
|
|
|
new_tex.h = tex.h;
|
|
|
|
new_tex.debug_tpage_name = tex_db.tpage_names.at(tex.page);
|
|
|
|
new_tex.debug_name = new_tex.debug_tpage_name + tex.name;
|
|
|
|
new_tex.data = tex.rgba_bytes;
|
|
|
|
new_tex.combo_id = id;
|
|
|
|
new_tex.load_to_pool = true;
|
|
|
|
}
|
2022-03-02 20:01:37 -05:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-04-15 20:40:10 -04:00
|
|
|
void confirm_textures_identical(const TextureDB& tex_db) {
|
2022-03-02 20:01:37 -05:00
|
|
|
std::unordered_map<std::string, std::vector<u32>> tex_dupl;
|
|
|
|
for (auto& tex : tex_db.textures) {
|
2022-04-15 20:40:10 -04:00
|
|
|
auto name = tex_db.tpage_names.at(tex.second.page) + tex.second.name;
|
2022-03-02 20:01:37 -05:00
|
|
|
auto it = tex_dupl.find(name);
|
|
|
|
if (it == tex_dupl.end()) {
|
|
|
|
tex_dupl.insert({name, tex.second.rgba_bytes});
|
|
|
|
} else {
|
|
|
|
bool ok = it->second == tex.second.rgba_bytes;
|
|
|
|
if (!ok) {
|
2022-04-12 18:48:27 -04:00
|
|
|
ASSERT_MSG(false, fmt::format("BAD duplicate: {} {} vs {}", name,
|
|
|
|
tex.second.rgba_bytes.size(), it->second.size()));
|
2022-03-02 20:01:37 -05:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-05-11 22:53:53 -04:00
|
|
|
void extract_art_groups_from_level(const ObjectFileDB& db,
|
|
|
|
const TextureDB& tex_db,
|
|
|
|
const std::vector<level_tools::TextureRemap>& tex_remap,
|
|
|
|
const std::string& dgo_name,
|
|
|
|
tfrag3::Level& level_data,
|
|
|
|
bool dump_level) {
|
|
|
|
const auto& files = db.obj_files_by_dgo.at(dgo_name);
|
|
|
|
for (const auto& file : files) {
|
|
|
|
if (file.name.length() > 3 && !file.name.compare(file.name.length() - 3, 3, "-ag")) {
|
|
|
|
const auto& ag_file = db.lookup_record(file);
|
|
|
|
extract_merc(ag_file, tex_db, db.dts, tex_remap, level_data, dump_level);
|
|
|
|
}
|
2022-03-11 22:27:11 -05:00
|
|
|
}
|
2022-03-02 20:01:37 -05:00
|
|
|
}
|
|
|
|
|
2022-05-11 22:53:53 -04:00
|
|
|
std::vector<level_tools::TextureRemap> extract_bsp_from_level(const ObjectFileDB& db,
|
|
|
|
const TextureDB& tex_db,
|
|
|
|
const std::string& dgo_name,
|
|
|
|
const DecompileHacks& hacks,
|
|
|
|
bool dump_level,
|
|
|
|
bool extract_collision,
|
|
|
|
tfrag3::Level& level_data) {
|
2021-12-04 12:33:18 -05:00
|
|
|
auto bsp_rec = get_bsp_file(db.obj_files_by_dgo.at(dgo_name));
|
2021-12-04 16:06:01 -05:00
|
|
|
if (!bsp_rec) {
|
|
|
|
lg::warn("Skipping extract for {} because the BSP file was not found", dgo_name);
|
2022-05-11 22:53:53 -04:00
|
|
|
return {};
|
2021-12-04 16:06:01 -05:00
|
|
|
}
|
|
|
|
std::string level_name = bsp_rec->name.substr(0, bsp_rec->name.length() - 4);
|
2021-12-04 12:33:18 -05:00
|
|
|
|
|
|
|
fmt::print("Processing level {} ({})\n", dgo_name, level_name);
|
2022-04-15 20:40:10 -04:00
|
|
|
const auto& bsp_file = db.lookup_record(*bsp_rec);
|
2021-12-04 12:33:18 -05:00
|
|
|
bool ok = is_valid_bsp(bsp_file.linked_data);
|
2022-02-08 19:02:47 -05:00
|
|
|
ASSERT(ok);
|
2021-12-04 12:33:18 -05:00
|
|
|
|
|
|
|
level_tools::DrawStats draw_stats;
|
|
|
|
// draw_stats.debug_print_dma_data = true;
|
|
|
|
level_tools::BspHeader bsp_header;
|
|
|
|
bsp_header.read_from_file(bsp_file.linked_data, db.dts, &draw_stats);
|
2022-02-08 19:02:47 -05:00
|
|
|
ASSERT((int)bsp_header.drawable_tree_array.trees.size() == bsp_header.drawable_tree_array.length);
|
2021-12-04 12:33:18 -05:00
|
|
|
|
2022-03-28 18:14:25 -04:00
|
|
|
/*
|
|
|
|
level_tools::PrintSettings settings;
|
2022-04-25 21:53:23 -04:00
|
|
|
settings.expand_collide = true;
|
2022-03-28 18:14:25 -04:00
|
|
|
fmt::print("{}\n", bsp_header.print(settings));
|
2022-04-25 21:53:23 -04:00
|
|
|
*/
|
2022-03-28 18:14:25 -04:00
|
|
|
|
2021-12-04 12:33:18 -05:00
|
|
|
const std::set<std::string> tfrag_trees = {
|
|
|
|
"drawable-tree-tfrag", "drawable-tree-trans-tfrag", "drawable-tree-dirt-tfrag",
|
|
|
|
"drawable-tree-ice-tfrag", "drawable-tree-lowres-tfrag", "drawable-tree-lowres-trans-tfrag"};
|
|
|
|
int i = 0;
|
2022-03-02 20:01:37 -05:00
|
|
|
|
2022-04-25 21:53:23 -04:00
|
|
|
std::vector<const level_tools::DrawableTreeInstanceTie*> all_ties;
|
|
|
|
for (auto& draw_tree : bsp_header.drawable_tree_array.trees) {
|
|
|
|
auto as_tie_tree = dynamic_cast<level_tools::DrawableTreeInstanceTie*>(draw_tree.get());
|
|
|
|
if (as_tie_tree) {
|
|
|
|
all_ties.push_back(as_tie_tree);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
bool got_collide = false;
|
2021-12-04 12:33:18 -05:00
|
|
|
for (auto& draw_tree : bsp_header.drawable_tree_array.trees) {
|
|
|
|
if (tfrag_trees.count(draw_tree->my_type())) {
|
|
|
|
auto as_tfrag_tree = dynamic_cast<level_tools::DrawableTreeTfrag*>(draw_tree.get());
|
2022-02-08 19:02:47 -05:00
|
|
|
ASSERT(as_tfrag_tree);
|
2021-12-04 12:33:18 -05:00
|
|
|
std::vector<std::pair<int, int>> expected_missing_textures;
|
|
|
|
auto it = hacks.missing_textures_by_level.find(level_name);
|
|
|
|
if (it != hacks.missing_textures_by_level.end()) {
|
|
|
|
expected_missing_textures = it->second;
|
|
|
|
}
|
|
|
|
extract_tfrag(as_tfrag_tree, fmt::format("{}-{}", dgo_name, i++),
|
2022-05-11 22:53:53 -04:00
|
|
|
bsp_header.texture_remap_table, tex_db, expected_missing_textures, level_data,
|
2021-12-30 18:48:37 -05:00
|
|
|
dump_level);
|
2021-12-26 12:33:51 -05:00
|
|
|
} else if (draw_tree->my_type() == "drawable-tree-instance-tie") {
|
|
|
|
auto as_tie_tree = dynamic_cast<level_tools::DrawableTreeInstanceTie*>(draw_tree.get());
|
2022-02-08 19:02:47 -05:00
|
|
|
ASSERT(as_tie_tree);
|
2021-12-26 12:33:51 -05:00
|
|
|
extract_tie(as_tie_tree, fmt::format("{}-{}-tie", dgo_name, i++),
|
2022-05-11 22:53:53 -04:00
|
|
|
bsp_header.texture_remap_table, tex_db, level_data, dump_level);
|
2022-03-28 18:14:25 -04:00
|
|
|
} else if (draw_tree->my_type() == "drawable-tree-instance-shrub") {
|
|
|
|
auto as_shrub_tree =
|
|
|
|
dynamic_cast<level_tools::shrub_types::DrawableTreeInstanceShrub*>(draw_tree.get());
|
|
|
|
ASSERT(as_shrub_tree);
|
|
|
|
extract_shrub(as_shrub_tree, fmt::format("{}-{}-shrub", dgo_name, i++),
|
2022-05-11 22:53:53 -04:00
|
|
|
bsp_header.texture_remap_table, tex_db, {}, level_data, dump_level);
|
2022-04-25 21:53:23 -04:00
|
|
|
} else if (draw_tree->my_type() == "drawable-tree-collide-fragment" && extract_collision) {
|
|
|
|
auto as_collide_frags =
|
|
|
|
dynamic_cast<level_tools::DrawableTreeCollideFragment*>(draw_tree.get());
|
|
|
|
ASSERT(as_collide_frags);
|
|
|
|
ASSERT(!got_collide);
|
|
|
|
got_collide = true;
|
|
|
|
extract_collide_frags(as_collide_frags, all_ties, fmt::format("{}-{}-collide", dgo_name, i++),
|
2022-05-11 22:53:53 -04:00
|
|
|
level_data, dump_level);
|
2021-12-04 12:33:18 -05:00
|
|
|
} else {
|
2022-02-16 22:13:18 -05:00
|
|
|
// fmt::print(" unsupported tree {}\n", draw_tree->my_type());
|
2021-12-04 12:33:18 -05:00
|
|
|
}
|
|
|
|
}
|
2022-05-11 22:53:53 -04:00
|
|
|
level_data.level_name = level_name;
|
|
|
|
|
|
|
|
return bsp_header.texture_remap_table;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*!
|
|
|
|
* Extract stuff found in GAME.CGO.
|
|
|
|
* Even though GAME.CGO isn't technically a level, the decompiler/loader treat it like one,
|
|
|
|
* but the bsp stuff is just empty. It will contain only textures/art groups.
|
|
|
|
*/
|
|
|
|
void extract_common(const ObjectFileDB& db,
|
|
|
|
const TextureDB& tex_db,
|
|
|
|
const std::string& dgo_name,
|
|
|
|
bool dump_levels) {
|
|
|
|
if (db.obj_files_by_dgo.count(dgo_name) == 0) {
|
|
|
|
lg::warn("Skipping common extract for {} because the DGO was not part of the input", dgo_name);
|
|
|
|
return;
|
|
|
|
}
|
2021-12-04 12:33:18 -05:00
|
|
|
|
2022-05-11 22:53:53 -04:00
|
|
|
if (tex_db.textures.size() == 0) {
|
|
|
|
lg::warn("Skipping common extract because there were no textures in the input");
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
confirm_textures_identical(tex_db);
|
2022-04-25 21:53:23 -04:00
|
|
|
|
2022-05-11 22:53:53 -04:00
|
|
|
tfrag3::Level tfrag_level;
|
|
|
|
add_all_textures_from_level(tfrag_level, dgo_name, tex_db);
|
|
|
|
extract_art_groups_from_level(db, tex_db, {}, dgo_name, tfrag_level, dump_levels);
|
2021-12-04 12:33:18 -05:00
|
|
|
Serializer ser;
|
|
|
|
tfrag_level.serialize(ser);
|
2022-01-07 11:52:24 -05:00
|
|
|
auto compressed =
|
|
|
|
compression::compress_zstd(ser.get_save_result().first, ser.get_save_result().second);
|
2022-05-28 19:28:19 -04:00
|
|
|
|
|
|
|
fmt::print("stats for {}\n", dgo_name);
|
2022-02-16 22:13:18 -05:00
|
|
|
print_memory_usage(tfrag_level, ser.get_save_result().second);
|
2022-01-07 11:52:24 -05:00
|
|
|
fmt::print("compressed: {} -> {} ({:.2f}%)\n", ser.get_save_result().second, compressed.size(),
|
|
|
|
100.f * compressed.size() / ser.get_save_result().second);
|
2021-12-04 12:33:18 -05:00
|
|
|
file_util::write_binary_file(file_util::get_file_path({fmt::format(
|
|
|
|
"assets/{}.fr3", dgo_name.substr(0, dgo_name.length() - 4))}),
|
2022-01-07 11:52:24 -05:00
|
|
|
compressed.data(), compressed.size());
|
2021-12-04 12:33:18 -05:00
|
|
|
}
|
2022-04-15 20:40:10 -04:00
|
|
|
|
2022-05-11 22:53:53 -04:00
|
|
|
void extract_from_level(const ObjectFileDB& db,
|
|
|
|
const TextureDB& tex_db,
|
|
|
|
const std::string& dgo_name,
|
|
|
|
const DecompileHacks& hacks,
|
|
|
|
bool dump_level,
|
|
|
|
bool extract_collision) {
|
|
|
|
if (db.obj_files_by_dgo.count(dgo_name) == 0) {
|
|
|
|
lg::warn("Skipping extract for {} because the DGO was not part of the input", dgo_name);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
tfrag3::Level level_data;
|
|
|
|
add_all_textures_from_level(level_data, dgo_name, tex_db);
|
|
|
|
|
|
|
|
// the bsp header file data
|
|
|
|
auto tex_remap = extract_bsp_from_level(db, tex_db, dgo_name, hacks, dump_level,
|
|
|
|
extract_collision, level_data);
|
|
|
|
extract_art_groups_from_level(db, tex_db, tex_remap, dgo_name, level_data, dump_level);
|
|
|
|
|
|
|
|
Serializer ser;
|
|
|
|
level_data.serialize(ser);
|
|
|
|
auto compressed =
|
|
|
|
compression::compress_zstd(ser.get_save_result().first, ser.get_save_result().second);
|
2022-05-28 19:28:19 -04:00
|
|
|
fmt::print("stats for {}\n", dgo_name);
|
2022-05-11 22:53:53 -04:00
|
|
|
print_memory_usage(level_data, ser.get_save_result().second);
|
|
|
|
fmt::print("compressed: {} -> {} ({:.2f}%)\n", ser.get_save_result().second, compressed.size(),
|
|
|
|
100.f * compressed.size() / ser.get_save_result().second);
|
|
|
|
file_util::write_binary_file(file_util::get_file_path({fmt::format(
|
|
|
|
"assets/{}.fr3", dgo_name.substr(0, dgo_name.length() - 4))}),
|
|
|
|
compressed.data(), compressed.size());
|
|
|
|
}
|
|
|
|
|
2022-04-15 20:40:10 -04:00
|
|
|
void extract_all_levels(const ObjectFileDB& db,
|
|
|
|
const TextureDB& tex_db,
|
|
|
|
const std::vector<std::string>& dgo_names,
|
|
|
|
const std::string& common_name,
|
|
|
|
const DecompileHacks& hacks,
|
2022-04-25 21:53:23 -04:00
|
|
|
bool debug_dump_level,
|
|
|
|
bool extract_collision) {
|
2022-05-11 22:53:53 -04:00
|
|
|
extract_common(db, tex_db, common_name, debug_dump_level);
|
2022-04-15 20:40:10 -04:00
|
|
|
SimpleThreadGroup threads;
|
|
|
|
threads.run(
|
2022-04-25 21:53:23 -04:00
|
|
|
[&](int idx) {
|
|
|
|
extract_from_level(db, tex_db, dgo_names[idx], hacks, debug_dump_level, extract_collision);
|
|
|
|
},
|
2022-04-15 20:40:10 -04:00
|
|
|
dgo_names.size());
|
|
|
|
threads.join();
|
|
|
|
}
|
|
|
|
|
2021-12-04 12:33:18 -05:00
|
|
|
} // namespace decompiler
|