Archive/jak-project
1
0
Fork 0
mirror of https://github.com/open-goal/jak-project.git synced 2024-10-19 14:47:49 -04:00
Code Issues Actions 3 Packages Projects Releases Wiki Activity

Add a gltf level exporter (#1719)

Browse source 
* Add a gltf level exporter

* more fixes

* disable by default

* compile fixes for windows

* improve collide packer

* bug fix

* clang format

* fix texture bug
This commit is contained in:
water111 2022-08-05 12:25:35 -04:00 committed by GitHub
parent dc6589cef5
commit da4ec008c3
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
39 changed files with 1690 additions and 231 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
CMakeLists.txt
View file

@ -46,6 +46,8 @@ if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
# Note: this is only _reserved_ memory, not necessarily _committed_ memory
# TODO - test with add_link_options instead
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${LDFLAGS} -Xlinker /STACK:16000000")
else()
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -ggdb -g -Wextra")
endif()
# additional c++ and linker flags for release mode for our projects
@ -78,6 +80,7 @@ elseif(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
-Wcast-qual \
-Wdisabled-optimization \
-Wformat \
-Wextra \
-Wmissing-include-dirs \
-Woverloaded-virtual \
-Wredundant-decls \

1
common/custom_data/TFrag3Data.cpp
View file

@ -277,7 +277,6 @@ void MercModel::serialize(Serializer& ser) {
for (auto& effect : effects) {
effect.serialize(ser);
}
ser.from_ptr(&scale_xyz);
ser.from_ptr(&max_draws);
ser.from_ptr(&max_bones);
}

3
common/custom_data/Tfrag3Data.h
View file

@ -53,7 +53,7 @@ enum MemoryUsageCategory {
NUM_CATEGORIES
};
constexpr int TFRAG3_VERSION = 20;
constexpr int TFRAG3_VERSION = 21;
// These vertices should be uploaded to the GPU at load time and don't change
struct PreloadedVertex {
@ -376,7 +376,6 @@ struct MercEffect {
struct MercModel {
std::string name;
std::vector<MercEffect> effects;
float scale_xyz;
u32 max_draws;
u32 max_bones;
void serialize(Serializer& ser);

4
common/util/FileUtil.cpp
View file

@ -512,7 +512,7 @@ std::vector<u8> decompress_dgo(const std::vector<u8>& data_in) {
return decompressed_data;
}
FILE* open_file(const fs::path& path, std::string mode) {
FILE* open_file(const fs::path& path, const std::string& mode) {
#ifdef _WIN32
return _wfopen(path.wstring().c_str(), std::wstring(mode.begin(), mode.end()).c_str());
#else
@ -520,7 +520,7 @@ FILE* open_file(const fs::path& path, std::string mode) {
#endif
}
std::vector<fs::path> find_files_recursively(const fs::path base_dir, const std::regex& pattern) {
std::vector<fs::path> find_files_recursively(const fs::path& base_dir, const std::regex& pattern) {
std::vector<fs::path> files = {};
for (auto& p : fs::recursive_directory_iterator(base_dir)) {
if (p.is_regular_file()) {

4
common/util/FileUtil.h
View file

@ -55,6 +55,6 @@ void ISONameFromAnimationName(char* dst, const char* src);
void assert_file_exists(const char* path, const char* error_message);
bool dgo_header_is_compressed(const std::vector<u8>& data);
std::vector<u8> decompress_dgo(const std::vector<u8>& data_in);
FILE* open_file(const fs::path& path, std::string mode);
std::vector<fs::path> find_files_recursively(const fs::path base_dir, const std::regex& pattern);
FILE* open_file(const fs::path& path, const std::string& mode);
std::vector<fs::path> find_files_recursively(const fs::path& base_dir, const std::regex& pattern);
} // namespace file_util

2
common/util/FontUtils.cpp
View file

@ -164,7 +164,7 @@ std::string GameTextFontBank::convert_utf8_to_game(std::string str) const {
std::string GameTextFontBank::convert_utf8_to_game_with_escape(const std::string& str) const {
std::string newstr;
for (int i = 0; i < str.size(); ++i) {
for (size_t i = 0; i < str.size(); ++i) {
auto c = str.at(i);
if (c == '"') {
newstr.push_back('"');

8
custom_levels/test-zone/test-zone.jsonc
View file

@ -6,14 +6,20 @@
"iso_name": "TESTZONE",
// The nickname, should be exactly 3 characters
"nickname": "TSZ", // 3 char name, all uppercase
// Background mesh file.
// Must have vertex colors. Use the blender cycles renderer, bake, diffuse, uncheck color,
// and bake to vertex colors. For now, only the first vertex color group is used, so make sure you
// only have 1.
"gltf_file": "custom_levels/test-zone/test-zone2.glb",
// automatically set wall vs. ground based on angle. Useful if you don't want to assign this yourself
"automatic_wall_detection": true,
"automatic_wall_angle": 45.0,
// if your mesh has triangles with incorrect orientation, set this to make all collision mesh triangles double sided
// this makes collision 2x slower and bigger, so only use if really needed
"double_sided_collide": false,
"actors" : [
{
"trans": [-21.6238, 20.0496, 17.1191], // translation

9
decompiler/CMakeLists.txt
View file

@ -60,7 +60,9 @@ add_library(
level_extractor/extract_tfrag.cpp
level_extractor/extract_tie.cpp
level_extractor/extract_shrub.cpp
level_extractor/fr3_to_gltf.cpp
level_extractor/MercData.cpp
level_extractor/tfrag_tie_fixup.cpp
ObjectFile/LinkedObjectFile.cpp
ObjectFile/LinkedObjectFileCreation.cpp
@ -86,6 +88,7 @@ target_link_libraries(decomp
fmt
stb_image
xdelta3
tiny_gltf
)
add_executable(decompiler
@ -96,7 +99,8 @@ target_link_libraries(decompiler
common
lzokay
fmt
stb_image)
stb_image
tiny_gltf)
add_executable(extractor
@ -108,4 +112,5 @@ target_link_libraries(extractor
lzokay
fmt
compiler
stb_image)
stb_image
tiny_gltf)

10
decompiler/analysis/analyze_inspect_method.cpp
View file

@ -238,12 +238,7 @@ FieldPrint get_field_print(const std::string& str) {
return field_print;
}
int get_start_idx_process(Function& function,
LinkedObjectFile& file,
TypeInspectorResult* result,
const std::string& parent_type,
const std::string& type_name,
Env& env) {
int get_start_idx_process(Function& function, const std::string& parent_type, Env& env) {
if (function.basic_blocks.size() != 5) {
fmt::print("[iim] inspect {} had {} basic blocks, expected 5\n", function.name(),
function.basic_blocks.size());
@ -944,8 +939,7 @@ std::string inspect_inspect_method(Function& inspect_method,
inspect_method.ir2.env);
if (idx < 0) {
idx = get_start_idx_process(inspect_method, file, &result, result.parent_type_name, type_name,
inspect_method.ir2.env);
idx = get_start_idx_process(inspect_method, result.parent_type_name, inspect_method.ir2.env);
}
StructureType* old_game_type = nullptr;
if (previous_game_ts.fully_defined_type_exists(type_name)) {

2
decompiler/config.cpp
View file

@ -69,7 +69,7 @@ Config read_config_file(const fs::path& path_to_config_file, const std::string&
config.print_cfgs = cfg.at("print_cfgs").get<bool>();
config.generate_symbol_definition_map = cfg.at("generate_symbol_definition_map").get<bool>();
config.is_pal = cfg.at("is_pal").get<bool>();
config.rip_levels = cfg.at("levels_convert_to_obj").get<bool>();
config.rip_levels = cfg.at("rip_levels").get<bool>();
config.extract_collision = cfg.at("extract_collision").get<bool>();
config.generate_all_types = cfg.at("generate_all_types").get<bool>();
if (cfg.contains("old_all_types_file")) {

4
decompiler/config/jak1_jp.jsonc
View file

@ -96,8 +96,8 @@
// turn this on to extract level background graphics data
"levels_extract": true,
// turn this on if you want extracted levels to be saved out as .obj files
"levels_convert_to_obj": false,
// turn this on if you want extracted levels to be saved out as .glb files
"rip_levels": false,
// should we extract collision meshes?
// these can be displayed in game, but makes the .fr3 files slightly larger
"extract_collision": true,

2
decompiler/config/jak1_ntsc_black_label.jsonc
View file

@ -97,7 +97,7 @@
// turn this on to extract level background graphics data
"levels_extract": true,
// turn this on if you want extracted levels to be saved out as .obj files
"levels_convert_to_obj": false,
"rip_levels": false,
// should we extract collision meshes?
// these can be displayed in game, but makes the .fr3 files slightly larger
"extract_collision": true,

2
decompiler/config/jak1_pal.jsonc
View file

@ -97,7 +97,7 @@
// turn this on to extract level background graphics data
"levels_extract": true,
// turn this on if you want extracted levels to be saved out as .obj files
"levels_convert_to_obj": false,
"rip_levels": false,
// should we extract collision meshes?
// these can be displayed in game, but makes the .fr3 files slightly larger
"extract_collision": true,

2
decompiler/config/jak1_us2.jsonc
View file

@ -97,7 +97,7 @@
// turn this on to extract level background graphics data
"levels_extract": true,
// turn this on if you want extracted levels to be saved out as .obj files
"levels_convert_to_obj": false,
"rip_levels": false,
// should we extract collision meshes?
// these can be displayed in game, but makes the .fr3 files slightly larger
"extract_collision": true,

2
decompiler/config/jak2_ntsc_v1.jsonc
View file

@ -101,7 +101,7 @@
// turn this on to extract level background graphics data
"levels_extract": false,
// turn this on if you want extracted levels to be saved out as .obj files
"levels_convert_to_obj": false,
"rip_levels": false,
// should we extract collision meshes?
// these can be displayed in game, but makes the .fr3 files slightly larger
"extract_collision": true,

38
decompiler/level_extractor/extract_level.cpp
View file

@ -13,6 +13,7 @@
#include "decompiler/level_extractor/extract_shrub.h"
#include "decompiler/level_extractor/extract_tfrag.h"
#include "decompiler/level_extractor/extract_tie.h"
#include "decompiler/level_extractor/fr3_to_gltf.h"
namespace decompiler {
@ -98,13 +99,12 @@ void extract_art_groups_from_level(const ObjectFileDB& db,
const TextureDB& tex_db,
const std::vector<level_tools::TextureRemap>& tex_remap,
const std::string& dgo_name,
tfrag3::Level& level_data,
bool dump_level) {
tfrag3::Level& level_data) {
const auto& files = db.obj_files_by_dgo.at(dgo_name);
for (const auto& file : files) {
if (file.name.length() > 3 && !file.name.compare(file.name.length() - 3, 3, "-ag")) {
const auto& ag_file = db.lookup_record(file);
extract_merc(ag_file, tex_db, db.dts, tex_remap, level_data, dump_level);
extract_merc(ag_file, tex_db, db.dts, tex_remap, level_data, false);
}
}
}
@ -113,7 +113,6 @@ std::vector<level_tools::TextureRemap> extract_bsp_from_level(const ObjectFileDB
const TextureDB& tex_db,
const std::string& dgo_name,
const DecompileHacks& hacks,
bool dump_level,
bool extract_collision,
tfrag3::Level& level_data) {
auto bsp_rec = get_bsp_file(db.obj_files_by_dgo.at(dgo_name));
@ -165,18 +164,18 @@ std::vector<level_tools::TextureRemap> extract_bsp_from_level(const ObjectFileDB
}
extract_tfrag(as_tfrag_tree, fmt::format("{}-{}", dgo_name, i++),
bsp_header.texture_remap_table, tex_db, expected_missing_textures, level_data,
dump_level);
false);
} else if (draw_tree->my_type() == "drawable-tree-instance-tie") {
auto as_tie_tree = dynamic_cast<level_tools::DrawableTreeInstanceTie*>(draw_tree.get());
ASSERT(as_tie_tree);
extract_tie(as_tie_tree, fmt::format("{}-{}-tie", dgo_name, i++),
bsp_header.texture_remap_table, tex_db, level_data, dump_level);
bsp_header.texture_remap_table, tex_db, level_data, false);
} else if (draw_tree->my_type() == "drawable-tree-instance-shrub") {
auto as_shrub_tree =
dynamic_cast<level_tools::shrub_types::DrawableTreeInstanceShrub*>(draw_tree.get());
ASSERT(as_shrub_tree);
extract_shrub(as_shrub_tree, fmt::format("{}-{}-shrub", dgo_name, i++),
bsp_header.texture_remap_table, tex_db, {}, level_data, dump_level);
bsp_header.texture_remap_table, tex_db, {}, level_data, false);
} else if (draw_tree->my_type() == "drawable-tree-collide-fragment" && extract_collision) {
auto as_collide_frags =
dynamic_cast<level_tools::DrawableTreeCollideFragment*>(draw_tree.get());
@ -184,7 +183,7 @@ std::vector<level_tools::TextureRemap> extract_bsp_from_level(const ObjectFileDB
ASSERT(!got_collide);
got_collide = true;
extract_collide_frags(as_collide_frags, all_ties, fmt::format("{}-{}-collide", dgo_name, i++),
level_data, dump_level);
level_data, false);
} else {
// fmt::print(" unsupported tree {}\n", draw_tree->my_type());
}
@ -218,7 +217,8 @@ void extract_common(const ObjectFileDB& db,
tfrag3::Level tfrag_level;
add_all_textures_from_level(tfrag_level, dgo_name, tex_db);
extract_art_groups_from_level(db, tex_db, {}, dgo_name, tfrag_level, dump_levels);
extract_art_groups_from_level(db, tex_db, {}, dgo_name, tfrag_level);
Serializer ser;
tfrag_level.serialize(ser);
auto compressed =
@ -231,6 +231,11 @@ void extract_common(const ObjectFileDB& db,
file_util::write_binary_file(
output_folder / fmt::format("{}.fr3", dgo_name.substr(0, dgo_name.length() - 4)),
compressed.data(), compressed.size());
if (dump_levels) {
save_level_foreground_as_gltf(tfrag_level,
file_util::get_jak_project_dir() / "debug_out" / "common.glb");
}
}
void extract_from_level(const ObjectFileDB& db,
@ -248,9 +253,9 @@ void extract_from_level(const ObjectFileDB& db,
add_all_textures_from_level(level_data, dgo_name, tex_db);
// the bsp header file data
auto tex_remap = extract_bsp_from_level(db, tex_db, dgo_name, hacks, dump_level,
extract_collision, level_data);
extract_art_groups_from_level(db, tex_db, tex_remap, dgo_name, level_data, dump_level);
auto tex_remap =
extract_bsp_from_level(db, tex_db, dgo_name, hacks, extract_collision, level_data);
extract_art_groups_from_level(db, tex_db, tex_remap, dgo_name, level_data);
Serializer ser;
level_data.serialize(ser);
@ -263,6 +268,15 @@ void extract_from_level(const ObjectFileDB& db,
file_util::write_binary_file(
output_folder / fmt::format("{}.fr3", dgo_name.substr(0, dgo_name.length() - 4)),
compressed.data(), compressed.size());
if (dump_level) {
save_level_background_as_gltf(level_data,
file_util::get_jak_project_dir() / "debug_out" /
fmt::format("{}_background.glb", level_data.level_name));
save_level_foreground_as_gltf(level_data,
file_util::get_jak_project_dir() / "debug_out" /
fmt::format("{}_foreground.glb", level_data.level_name));
}
}
void extract_all_levels(const ObjectFileDB& db,

11
decompiler/level_extractor/extract_merc.cpp
View file

@ -827,11 +827,11 @@ u8 convert_mat(int in) {
}
}
tfrag3::MercVertex convert_vertex(const MercUnpackedVtx& vtx) {
tfrag3::MercVertex convert_vertex(const MercUnpackedVtx& vtx, float xyz_scale) {
tfrag3::MercVertex out;
out.pos[0] = vtx.pos[0];
out.pos[1] = vtx.pos[1];
out.pos[2] = vtx.pos[2];
out.pos[0] = vtx.pos[0] * xyz_scale;
out.pos[1] = vtx.pos[1] * xyz_scale;
out.pos[2] = vtx.pos[2] * xyz_scale;
out.pad0 = 0;
out.normal[0] = vtx.nrm[0];
out.normal[1] = vtx.nrm[1];
@ -896,7 +896,6 @@ void extract_merc(const ObjectFileData& ag_data,
auto& ctrl = ctrls[ci];
pc_ctrl.name = ctrl.name;
pc_ctrl.scale_xyz = ctrl.header.xyz_scale;
pc_ctrl.max_draws = 0;
pc_ctrl.max_bones = 0;
@ -906,7 +905,7 @@ void extract_merc(const ObjectFileData& ag_data,
auto& effect = all_effects[ci][ei];
u32 first_vertex = out.merc_data.vertices.size();
for (auto& vtx : effect.vertices) {
auto cvtx = convert_vertex(vtx);
auto cvtx = convert_vertex(vtx, ctrl.header.xyz_scale);
out.merc_data.vertices.push_back(cvtx);
for (int i = 0; i < 3; i++) {
pc_ctrl.max_bones = std::max(pc_ctrl.max_bones, (u32)cvtx.mats[i]);

2
decompiler/level_extractor/extract_merc.h
View file

@ -14,4 +14,4 @@ void extract_merc(const ObjectFileData& ag_data,
const std::vector<level_tools::TextureRemap>& map,
tfrag3::Level& out,
bool dump_level);
}
} // namespace decompiler

777
decompiler/level_extractor/fr3_to_gltf.cpp Normal file
View file

@ -0,0 +1,777 @@
#include "fr3_to_gltf.h"
#include "common/custom_data/Tfrag3Data.h"
#include "common/math/Vector.h"
#include "decompiler/level_extractor/tfrag_tie_fixup.h"
#include "third-party/tiny_gltf/tiny_gltf.h"
namespace {
/*!
* Convert fr3 format indices (strip format, with UINT32_MAX as restart) to unstripped tris.
* Assumes that this is the tfrag/tie format of stripping. Will flip tris as needed so the faces
* in this fragment all point a consistent way. However, the entire frag may be flipped.
*/
void unstrip_tfrag_tie(const std::vector<u32>& stripped_indices,
const std::vector<math::Vector3f>& positions,
std::vector<u32>& unstripped,
std::vector<u32>& old_to_new_start) {
fixup_and_unstrip_tfrag_tie(stripped_indices, positions, unstripped, old_to_new_start);
}
/*!
* Convert shrub strips. This doesn't assume anything about the strips.
*/
void unstrip_shrub_draws(const std::vector<u32>& stripped_indices,
std::vector<u32>& unstripped,
std::vector<u32>& draw_to_start,
std::vector<u32>& draw_to_count,
const std::vector<tfrag3::ShrubDraw>& draws) {
for (auto& draw : draws) {
draw_to_start.push_back(unstripped.size());
for (size_t i = 2; i < draw.num_indices; i++) {
int idx = i + draw.first_index_index;
u32 a = stripped_indices[idx];
u32 b = stripped_indices[idx - 1];
u32 c = stripped_indices[idx - 2];
if (a == UINT32_MAX || b == UINT32_MAX || c == UINT32_MAX) {
continue;
}
unstripped.push_back(a);
unstripped.push_back(b);
unstripped.push_back(c);
}
draw_to_count.push_back(unstripped.size() - draw_to_start.back());
}
}
/*!
* Convert merc strips. Doesn't assume anything about strips. Output is [model][effect][draw] format
*/
void unstrip_merc_draws(const std::vector<u32>& stripped_indices,
const std::vector<tfrag3::MercModel>& models,
std::vector<u32>& unstripped,
std::vector<std::vector<std::vector<u32>>>& draw_to_start,
std::vector<std::vector<std::vector<u32>>>& draw_to_count) {
for (auto& model : models) {
auto& model_dts = draw_to_start.emplace_back();
auto& model_dtc = draw_to_count.emplace_back();
for (auto& effect : model.effects) {
auto& effect_dts = model_dts.emplace_back();
auto& effect_dtc = model_dtc.emplace_back();
for (auto& draw : effect.draws) {
effect_dts.push_back(unstripped.size());
for (size_t i = 2; i < draw.index_count; i++) {
int idx = i + draw.first_index;
u32 a = stripped_indices[idx];
u32 b = stripped_indices[idx - 1];
u32 c = stripped_indices[idx - 2];
if (a == UINT32_MAX || b == UINT32_MAX || c == UINT32_MAX) {
continue;
}
unstripped.push_back(a);
unstripped.push_back(b);
unstripped.push_back(c);
}
effect_dtc.push_back(unstripped.size() - effect_dts.back());
}
}
}
}
/*!
* Get just the xyz positions from a preloaded vertex vector.
*/
std::vector<math::Vector3f> extract_positions(const std::vector<tfrag3::PreloadedVertex>& vtx) {
std::vector<math::Vector3f> result;
for (auto& v : vtx) {
auto& o = result.emplace_back();
o[0] = v.x;
o[1] = v.y;
o[2] = v.z;
}
return result;
}
/*!
* Set up a buffer for the positions of the given vertices.
* Return the index of the accessor.
*/
template <typename T>
int make_position_buffer_accessor(const std::vector<T>& vertices, tinygltf::Model& model) {
// first create a buffer:
int buffer_idx = (int)model.buffers.size();
auto& buffer = model.buffers.emplace_back();
buffer.data.resize(sizeof(float) * 3 * vertices.size());
// and fill it
u8* buffer_ptr = buffer.data.data();
for (const auto& vtx : vertices) {
if constexpr (std::is_same<T, tfrag3::MercVertex>::value) {
float xyz[3] = {vtx.pos[0] / 4096.f, vtx.pos[1] / 4096.f, vtx.pos[2] / 4096.f};
memcpy(buffer_ptr, xyz, 3 * sizeof(float));
buffer_ptr += 3 * sizeof(float);
} else {
float xyz[3] = {vtx.x / 4096.f, vtx.y / 4096.f, vtx.z / 4096.f};
memcpy(buffer_ptr, xyz, 3 * sizeof(float));
buffer_ptr += 3 * sizeof(float);
}
}
// create a view of this buffer
int buffer_view_idx = (int)model.bufferViews.size();
auto& buffer_view = model.bufferViews.emplace_back();
buffer_view.buffer = buffer_idx;
buffer_view.byteOffset = 0;
buffer_view.byteLength = buffer.data.size();
buffer_view.byteStride = 0; // tightly packed
buffer_view.target = TINYGLTF_TARGET_ARRAY_BUFFER;
int accessor_idx = (int)model.accessors.size();
auto& accessor = model.accessors.emplace_back();
accessor.bufferView = buffer_view_idx;
accessor.byteOffset = 0;
accessor.componentType = TINYGLTF_COMPONENT_TYPE_FLOAT;
accessor.count = vertices.size();
accessor.type = TINYGLTF_TYPE_VEC3;
return accessor_idx;
}
/*!
* Set up a buffer for the texture coordinates of the given vertices, multiplying by scale.
* Return the index of the accessor.
*/
template <typename T>
int make_tex_buffer_accessor(const std::vector<T>& vertices, tinygltf::Model& model, float scale) {
// first create a buffer:
int buffer_idx = (int)model.buffers.size();
auto& buffer = model.buffers.emplace_back();
buffer.data.resize(sizeof(float) * 2 * vertices.size());
// and fill it
u8* buffer_ptr = buffer.data.data();
for (const auto& vtx : vertices) {
if constexpr (std::is_same<T, tfrag3::MercVertex>::value) {
float st[2] = {vtx.st[0] * scale, vtx.st[1] * scale};
memcpy(buffer_ptr, st, 2 * sizeof(float));
buffer_ptr += 2 * sizeof(float);
} else {
float st[2] = {vtx.s * scale, vtx.t * scale};
memcpy(buffer_ptr, st, 2 * sizeof(float));
buffer_ptr += 2 * sizeof(float);
}
}
// create a view of this buffer
int buffer_view_idx = (int)model.bufferViews.size();
auto& buffer_view = model.bufferViews.emplace_back();
buffer_view.buffer = buffer_idx;
buffer_view.byteOffset = 0;
buffer_view.byteLength = buffer.data.size();
buffer_view.byteStride = 0; // tightly packed
buffer_view.target = TINYGLTF_TARGET_ARRAY_BUFFER;
int accessor_idx = (int)model.accessors.size();
auto& accessor = model.accessors.emplace_back();
accessor.bufferView = buffer_view_idx;
accessor.byteOffset = 0;
accessor.componentType = TINYGLTF_COMPONENT_TYPE_FLOAT;
accessor.count = vertices.size();
accessor.type = TINYGLTF_TYPE_VEC2;
return accessor_idx;
}
/*!
* Set up a buffer of vertex colors for the given time of day index, for tfrag.
* Uses the time of day texture to look up colors.
*/
int make_color_buffer_accessor(const std::vector<tfrag3::PreloadedVertex>& vertices,
tinygltf::Model& model,
const tfrag3::TfragTree& tfrag_tree,
int time_of_day) {
// first create a buffer:
int buffer_idx = (int)model.buffers.size();
auto& buffer = model.buffers.emplace_back();
buffer.data.resize(sizeof(float) * 4 * vertices.size());
std::vector<float> floats;
for (size_t i = 0; i < vertices.size(); i++) {
auto& color = tfrag_tree.colors.at(vertices[i].color_index);
for (int j = 0; j < 3; j++) {
floats.push_back(((float)color.rgba[time_of_day][j]) / 255.f);
}
floats.push_back(1.f);
}
memcpy(buffer.data.data(), floats.data(), sizeof(float) * floats.size());
// create a view of this buffer
int buffer_view_idx = (int)model.bufferViews.size();
auto& buffer_view = model.bufferViews.emplace_back();
buffer_view.buffer = buffer_idx;
buffer_view.byteOffset = 0;
buffer_view.byteLength = buffer.data.size();
buffer_view.byteStride = 0; // tightly packed
buffer_view.target = TINYGLTF_TARGET_ARRAY_BUFFER;
int accessor_idx = (int)model.accessors.size();
auto& accessor = model.accessors.emplace_back();
accessor.bufferView = buffer_view_idx;
accessor.byteOffset = 0;
accessor.componentType = TINYGLTF_COMPONENT_TYPE_FLOAT;
accessor.count = vertices.size();
accessor.type = TINYGLTF_TYPE_VEC4;
return accessor_idx;
}
/*!
* Set up a buffer of vertex colors for the given time of day index, for tie.
* Uses the time of day texture to look up colors.
*/
int make_color_buffer_accessor(const std::vector<tfrag3::PreloadedVertex>& vertices,
tinygltf::Model& model,
const tfrag3::TieTree& tie_tree,
int time_of_day) {
// first create a buffer:
int buffer_idx = (int)model.buffers.size();
auto& buffer = model.buffers.emplace_back();
buffer.data.resize(sizeof(float) * 4 * vertices.size());
std::vector<float> floats;
for (size_t i = 0; i < vertices.size(); i++) {
auto& color = tie_tree.colors.at(vertices[i].color_index);
for (int j = 0; j < 3; j++) {
floats.push_back(((float)color.rgba[time_of_day][j]) / 255.f);
}
floats.push_back(1.f);
}
memcpy(buffer.data.data(), floats.data(), sizeof(float) * floats.size());
// create a view of this buffer
int buffer_view_idx = (int)model.bufferViews.size();
auto& buffer_view = model.bufferViews.emplace_back();
buffer_view.buffer = buffer_idx;
buffer_view.byteOffset = 0;
buffer_view.byteLength = buffer.data.size();
buffer_view.byteStride = 0; // tightly packed
buffer_view.target = TINYGLTF_TARGET_ARRAY_BUFFER;
int accessor_idx = (int)model.accessors.size();
auto& accessor = model.accessors.emplace_back();
accessor.bufferView = buffer_view_idx;
accessor.byteOffset = 0;
accessor.componentType = TINYGLTF_COMPONENT_TYPE_FLOAT;
accessor.count = vertices.size();
accessor.type = TINYGLTF_TYPE_VEC4;
return accessor_idx;
}
int make_color_buffer_accessor(const std::vector<tfrag3::MercVertex>& vertices,
tinygltf::Model& model) {
// first create a buffer:
int buffer_idx = (int)model.buffers.size();
auto& buffer = model.buffers.emplace_back();
buffer.data.resize(sizeof(float) * 4 * vertices.size());
std::vector<float> floats;
for (size_t i = 0; i < vertices.size(); i++) {
for (int j = 0; j < 3; j++) {
floats.push_back(((float)vertices[i].rgba[j]) / 255.f);
}
floats.push_back(1.f);
}
memcpy(buffer.data.data(), floats.data(), sizeof(float) * floats.size());
// create a view of this buffer
int buffer_view_idx = (int)model.bufferViews.size();
auto& buffer_view = model.bufferViews.emplace_back();
buffer_view.buffer = buffer_idx;
buffer_view.byteOffset = 0;
buffer_view.byteLength = buffer.data.size();
buffer_view.byteStride = 0; // tightly packed
buffer_view.target = TINYGLTF_TARGET_ARRAY_BUFFER;
int accessor_idx = (int)model.accessors.size();
auto& accessor = model.accessors.emplace_back();
accessor.bufferView = buffer_view_idx;
accessor.byteOffset = 0;
accessor.componentType = TINYGLTF_COMPONENT_TYPE_FLOAT;
accessor.count = vertices.size();
accessor.type = TINYGLTF_TYPE_VEC4;
return accessor_idx;
}
/*!
* Set up a buffer of vertex colors for the given time of day index, for shrub.
* Uses the time of day texture to look up colors.
*/
int make_color_buffer_accessor(const std::vector<tfrag3::ShrubGpuVertex>& vertices,
tinygltf::Model& model,
const tfrag3::ShrubTree& shrub_tree,
int time_of_day) {
// first create a buffer:
int buffer_idx = (int)model.buffers.size();
auto& buffer = model.buffers.emplace_back();
buffer.data.resize(sizeof(float) * 4 * vertices.size());
std::vector<float> floats;
for (size_t i = 0; i < vertices.size(); i++) {
auto& color = shrub_tree.time_of_day_colors.at(vertices[i].color_index);
for (int j = 0; j < 3; j++) {
floats.push_back(((float)color.rgba[time_of_day][j]) / 255.f);
}
floats.push_back(1.f);
}
memcpy(buffer.data.data(), floats.data(), sizeof(float) * floats.size());
// create a view of this buffer
int buffer_view_idx = (int)model.bufferViews.size();
auto& buffer_view = model.bufferViews.emplace_back();
buffer_view.buffer = buffer_idx;
buffer_view.byteOffset = 0;
buffer_view.byteLength = buffer.data.size();
buffer_view.byteStride = 0; // tightly packed
buffer_view.target = TINYGLTF_TARGET_ARRAY_BUFFER;
int accessor_idx = (int)model.accessors.size();
auto& accessor = model.accessors.emplace_back();
accessor.bufferView = buffer_view_idx;
accessor.byteOffset = 0;
accessor.componentType = TINYGLTF_COMPONENT_TYPE_FLOAT;
accessor.count = vertices.size();
accessor.type = TINYGLTF_TYPE_VEC4;
return accessor_idx;
}
/*!
* Create a tinygltf buffer and buffer view for indices, and convert to gltf format.
* The map can be used to go from slots in the old index buffer to new.
*/
int make_tfrag_tie_index_buffer_view(const std::vector<u32>& indices,
const std::vector<math::Vector3f>& positions,
tinygltf::Model& model,
std::vector<u32>& map_out) {
std::vector<u32> unstripped;
unstrip_tfrag_tie(indices, positions, unstripped, map_out);
// first create a buffer:
int buffer_idx = (int)model.buffers.size();
auto& buffer = model.buffers.emplace_back();
buffer.data.resize(sizeof(u32) * unstripped.size());
// and fill it
memcpy(buffer.data.data(), unstripped.data(), buffer.data.size());
// create a view of this buffer
int buffer_view_idx = (int)model.bufferViews.size();
auto& buffer_view = model.bufferViews.emplace_back();
buffer_view.buffer = buffer_idx;
buffer_view.byteOffset = 0;
buffer_view.byteLength = buffer.data.size();
buffer_view.byteStride = 0; // tightly packed
buffer_view.target = TINYGLTF_TARGET_ELEMENT_ARRAY_BUFFER;
return buffer_view_idx;
}
/*!
* Create a tinygltf buffer and buffer view for indices, and convert to gltf format.
* The map can be used to go from slots in the old index buffer to new.
*/
int make_shrub_index_buffer_view(const std::vector<u32>& indices,
const std::vector<tfrag3::ShrubDraw>& draws,
tinygltf::Model& model,
std::vector<u32>& draw_to_start,
std::vector<u32>& draw_to_count) {
std::vector<u32> unstripped;
unstrip_shrub_draws(indices, unstripped, draw_to_start, draw_to_count, draws);
// first create a buffer:
int buffer_idx = (int)model.buffers.size();
auto& buffer = model.buffers.emplace_back();
buffer.data.resize(sizeof(u32) * unstripped.size());
// and fill it
memcpy(buffer.data.data(), unstripped.data(), buffer.data.size());
// create a view of this buffer
int buffer_view_idx = (int)model.bufferViews.size();
auto& buffer_view = model.bufferViews.emplace_back();
buffer_view.buffer = buffer_idx;
buffer_view.byteOffset = 0;
buffer_view.byteLength = buffer.data.size();
buffer_view.byteStride = 0; // tightly packed
buffer_view.target = TINYGLTF_TARGET_ELEMENT_ARRAY_BUFFER;
return buffer_view_idx;
}
int make_merc_index_buffer_view(const std::vector<u32>& indices,
const std::vector<tfrag3::MercModel>& models,
tinygltf::Model& model,
std::vector<std::vector<std::vector<u32>>>& draw_to_start,
std::vector<std::vector<std::vector<u32>>>& draw_to_count) {
std::vector<u32> unstripped;
unstrip_merc_draws(indices, models, unstripped, draw_to_start, draw_to_count);
// first create a buffer:
int buffer_idx = (int)model.buffers.size();
auto& buffer = model.buffers.emplace_back();
buffer.data.resize(sizeof(u32) * unstripped.size());
// and fill it
memcpy(buffer.data.data(), unstripped.data(), buffer.data.size());
// create a view of this buffer
int buffer_view_idx = (int)model.bufferViews.size();
auto& buffer_view = model.bufferViews.emplace_back();
buffer_view.buffer = buffer_idx;
buffer_view.byteOffset = 0;
buffer_view.byteLength = buffer.data.size();
buffer_view.byteStride = 0; // tightly packed
buffer_view.target = TINYGLTF_TARGET_ELEMENT_ARRAY_BUFFER;
return buffer_view_idx;
}
int make_index_buffer_accessor(tinygltf::Model& model,
const tfrag3::StripDraw& draw,
const std::vector<u32>& idx_map,
int buffer_view_idx) {
int accessor_idx = (int)model.accessors.size();
auto& accessor = model.accessors.emplace_back();
accessor.bufferView = buffer_view_idx;
accessor.byteOffset = sizeof(u32) * idx_map.at(draw.unpacked.idx_of_first_idx_in_full_buffer);
accessor.componentType = TINYGLTF_COMPONENT_TYPE_UNSIGNED_INT;
accessor.count = draw.num_triangles * 3;
accessor.type = TINYGLTF_TYPE_SCALAR;
return accessor_idx;
}
int make_index_buffer_accessor(tinygltf::Model& model, u32 start, u32 count, int buffer_view_idx) {
int accessor_idx = (int)model.accessors.size();
auto& accessor = model.accessors.emplace_back();
accessor.bufferView = buffer_view_idx;
accessor.byteOffset = sizeof(u32) * start;
accessor.componentType = TINYGLTF_COMPONENT_TYPE_UNSIGNED_INT;
accessor.count = count;
accessor.type = TINYGLTF_TYPE_SCALAR;
return accessor_idx;
}
int add_image_for_tex(const tfrag3::Level& level,
tinygltf::Model& model,
int tex_idx,
std::unordered_map<int, int>& tex_image_map) {
const auto& existing = tex_image_map.find(tex_idx);
if (existing != tex_image_map.end()) {
return existing->second;
}
auto& tex = level.textures.at(tex_idx);
int image_idx = (int)model.images.size();
auto& image = model.images.emplace_back();
image.pixel_type = TINYGLTF_TEXTURE_TYPE_UNSIGNED_BYTE;
image.width = tex.w;
image.height = tex.h;
image.image.resize(tex.data.size() * 4);
image.bits = 8;
image.component = 4;
image.mimeType = "image/png";
image.name = tex.debug_name;
memcpy(image.image.data(), tex.data.data(), tex.data.size() * 4);
tex_image_map[tex_idx] = image_idx;
return image_idx;
}
int add_material_for_tex(const tfrag3::Level& level,
tinygltf::Model& model,
int tex_idx,
std::unordered_map<int, int>& tex_image_map,
const DrawMode& draw_mode) {
int mat_idx = (int)model.materials.size();
auto& mat = model.materials.emplace_back();
auto& tex = level.textures.at(tex_idx);
mat.doubleSided = true;
// the 2.0 here compensates for the ps2's weird blending where 0.5 behaves like 1.0
mat.pbrMetallicRoughness.baseColorFactor = {2.0, 2.0, 2.0, 2.0};
mat.pbrMetallicRoughness.baseColorTexture.texCoord = 0; // TEXCOORD_0, I think
mat.pbrMetallicRoughness.baseColorTexture.index = model.textures.size();
mat.alphaMode = draw_mode.get_ab_enable() ? "BLEND" : "MASK";
// the foreground and background renderers both use this cutoff
mat.alphaCutoff = (float)0x26 / 255.f;
auto& gltf_texture = model.textures.emplace_back();
gltf_texture.name = tex.debug_name;
gltf_texture.sampler = model.samplers.size();
auto& sampler = model.samplers.emplace_back();
sampler.minFilter = draw_mode.get_filt_enable() ? TINYGLTF_TEXTURE_FILTER_LINEAR
: TINYGLTF_TEXTURE_FILTER_NEAREST;
sampler.magFilter = draw_mode.get_filt_enable() ? TINYGLTF_TEXTURE_FILTER_LINEAR
: TINYGLTF_TEXTURE_FILTER_NEAREST;
sampler.wrapS = draw_mode.get_clamp_s_enable() ? TINYGLTF_TEXTURE_WRAP_CLAMP_TO_EDGE
: TINYGLTF_TEXTURE_WRAP_REPEAT;
sampler.wrapT = draw_mode.get_clamp_t_enable() ? TINYGLTF_TEXTURE_WRAP_CLAMP_TO_EDGE
: TINYGLTF_TEXTURE_WRAP_REPEAT;
sampler.name = tex.debug_name;
gltf_texture.source = add_image_for_tex(level, model, tex_idx, tex_image_map);
return mat_idx;
}
constexpr int kMaxColor = 1;
/*!
* Add the given tfrag data to a node under tfrag_root.
*/
void add_tfrag(const tfrag3::Level& level,
const tfrag3::TfragTree& tfrag_in,
tinygltf::Model& model,
std::unordered_map<int, int>& tex_image_map) {
// copy and unpack in place
tfrag3::TfragTree tfrag = tfrag_in;
tfrag.unpack();
// we'll make a Node, Mesh, Primitive, then add the data to the primitive.
int node_idx = (int)model.nodes.size();
auto& node = model.nodes.emplace_back();
model.scenes.at(0).nodes.push_back(node_idx);
int mesh_idx = (int)model.meshes.size();
auto& mesh = model.meshes.emplace_back();
node.mesh = mesh_idx;
int position_buffer_accessor = make_position_buffer_accessor(tfrag.unpacked.vertices, model);
int texture_buffer_accessor = make_tex_buffer_accessor(tfrag.unpacked.vertices, model, 1.f);
std::vector<u32> index_map;
int index_buffer_view = make_tfrag_tie_index_buffer_view(
tfrag.unpacked.indices, extract_positions(tfrag.unpacked.vertices), model, index_map);
int colors[kMaxColor];
for (int i = 0; i < kMaxColor; i++) {
colors[i] = make_color_buffer_accessor(tfrag.unpacked.vertices, model, tfrag, i);
}
for (auto& draw : tfrag.draws) {
auto& prim = mesh.primitives.emplace_back();
prim.material = add_material_for_tex(level, model, draw.tree_tex_id, tex_image_map, draw.mode);
prim.indices = make_index_buffer_accessor(model, draw, index_map, index_buffer_view);
prim.attributes["POSITION"] = position_buffer_accessor;
prim.attributes["TEXCOORD_0"] = texture_buffer_accessor;
for (int i = 0; i < kMaxColor; i++) {
prim.attributes[fmt::format("COLOR_{}", i)] = colors[i];
}
prim.mode = TINYGLTF_MODE_TRIANGLES;
}
}
void add_tie(const tfrag3::Level& level,
const tfrag3::TieTree& tie_in,
tinygltf::Model& model,
std::unordered_map<int, int>& tex_image_map) {
// copy and unpack in place
tfrag3::TieTree tie = tie_in;
tie.unpack();
// we'll make a Node, Mesh, Primitive, then add the data to the primitive.
int node_idx = (int)model.nodes.size();
auto& node = model.nodes.emplace_back();
model.scenes.at(0).nodes.push_back(node_idx);
int mesh_idx = (int)model.meshes.size();
auto& mesh = model.meshes.emplace_back();
node.mesh = mesh_idx;
int position_buffer_accessor = make_position_buffer_accessor(tie.unpacked.vertices, model);
int texture_buffer_accessor = make_tex_buffer_accessor(tie.unpacked.vertices, model, 1.f);
std::vector<u32> index_map;
int index_buffer_view = make_tfrag_tie_index_buffer_view(
tie.unpacked.indices, extract_positions(tie.unpacked.vertices), model, index_map);
int colors[kMaxColor];
for (int i = 0; i < kMaxColor; i++) {
colors[i] = make_color_buffer_accessor(tie.unpacked.vertices, model, tie, i);
}
for (auto& draw : tie.static_draws) {
auto& prim = mesh.primitives.emplace_back();
prim.material = add_material_for_tex(level, model, draw.tree_tex_id, tex_image_map, draw.mode);
prim.indices = make_index_buffer_accessor(model, draw, index_map, index_buffer_view);
prim.attributes["POSITION"] = position_buffer_accessor;
prim.attributes["TEXCOORD_0"] = texture_buffer_accessor;
for (int i = 0; i < kMaxColor; i++) {
prim.attributes[fmt::format("COLOR_{}", i)] = colors[i];
}
prim.mode = TINYGLTF_MODE_TRIANGLES;
}
}
void add_shrub(const tfrag3::Level& level,
const tfrag3::ShrubTree& shrub_in,
tinygltf::Model& model,
std::unordered_map<int, int>& tex_image_map) {
// copy and unpack in place
tfrag3::ShrubTree shrub = shrub_in;
shrub.unpack();
// we'll make a Node, Mesh, Primitive, then add the data to the primitive.
int node_idx = (int)model.nodes.size();
auto& node = model.nodes.emplace_back();
model.scenes.at(0).nodes.push_back(node_idx);
int mesh_idx = (int)model.meshes.size();
auto& mesh = model.meshes.emplace_back();
node.mesh = mesh_idx;
int position_buffer_accessor = make_position_buffer_accessor(shrub.unpacked.vertices, model);
int texture_buffer_accessor =
make_tex_buffer_accessor(shrub.unpacked.vertices, model, 1.f / 4096.f);
std::vector<u32> draw_to_start, draw_to_count;
int index_buffer_view = make_shrub_index_buffer_view(shrub.indices, shrub.static_draws, model,
draw_to_start, draw_to_count);
int colors[kMaxColor];
for (int i = 0; i < kMaxColor; i++) {
colors[i] = make_color_buffer_accessor(shrub.unpacked.vertices, model, shrub, i);
}
// for (auto& draw : shrub.static_draws) {
for (size_t draw_idx = 0; draw_idx < shrub.static_draws.size(); draw_idx++) {
auto& draw = shrub.static_draws[draw_idx];
auto& prim = mesh.primitives.emplace_back();
prim.material = add_material_for_tex(level, model, draw.tree_tex_id, tex_image_map, draw.mode);
prim.indices = make_index_buffer_accessor(model, draw_to_start.at(draw_idx),
draw_to_count.at(draw_idx), index_buffer_view);
prim.attributes["POSITION"] = position_buffer_accessor;
prim.attributes["TEXCOORD_0"] = texture_buffer_accessor;
for (int i = 0; i < kMaxColor; i++) {
prim.attributes[fmt::format("COLOR_{}", i)] = colors[i];
}
prim.mode = TINYGLTF_MODE_TRIANGLES;
}
}
void add_merc(const tfrag3::Level& level,
tinygltf::Model& model,
std::unordered_map<int, int>& tex_image_map) {
const auto& mverts = level.merc_data.vertices;
// create position and uv buffers
int position_buffer_accessor = make_position_buffer_accessor(mverts, model);
int texture_buffer_accessor = make_tex_buffer_accessor(mverts, model, 1.f);
std::vector<std::vector<std::vector<u32>>> draw_to_start, draw_to_count;
int index_buffer_view = make_merc_index_buffer_view(
level.merc_data.indices, level.merc_data.models, model, draw_to_start, draw_to_count);
int colors = make_color_buffer_accessor(mverts, model);
for (size_t model_idx = 0; model_idx < level.merc_data.models.size(); model_idx++) {
const auto& mmodel = level.merc_data.models[model_idx];
int node_idx = (int)model.nodes.size();
auto& node = model.nodes.emplace_back();
model.scenes.at(0).nodes.push_back(node_idx);
node.name = mmodel.name;
int mesh_idx = (int)model.meshes.size();
auto& mesh = model.meshes.emplace_back();
mesh.name = node.name;
node.mesh = mesh_idx;
for (size_t effect_idx = 0; effect_idx < mmodel.effects.size(); effect_idx++) {
const auto& effect = mmodel.effects[effect_idx];
for (size_t draw_idx = 0; draw_idx < effect.draws.size(); draw_idx++) {
const auto& draw = effect.draws[draw_idx];
auto& prim = mesh.primitives.emplace_back();
prim.material =
add_material_for_tex(level, model, draw.tree_tex_id, tex_image_map, draw.mode);
prim.indices = make_index_buffer_accessor(
model, draw_to_start[model_idx][effect_idx][draw_idx],
draw_to_count[model_idx][effect_idx][draw_idx], index_buffer_view);
prim.attributes["POSITION"] = position_buffer_accessor;
prim.attributes["TEXCOORD_0"] = texture_buffer_accessor;
prim.attributes["COLOR_0"] = colors;
prim.mode = TINYGLTF_MODE_TRIANGLES;
}
}
}
}
} // namespace
/*!
* Export the background geometry (tie, tfrag, shrub) to a GLTF binary format (.glb) file.
*/
void save_level_background_as_gltf(const tfrag3::Level& level, const fs::path& glb_file) {
// the top level container for everything is the model.
tinygltf::Model model;
// a "scene" is a traditional scene graph, made up of Nodes.
// sadly, attempting to nest stuff makes the blender importer unhappy, so we just dump
// everything into the top level.
auto& scene = model.scenes.emplace_back();
// hack, add a default material.
tinygltf::Material mat;
mat.pbrMetallicRoughness.baseColorFactor = {1.0f, 0.9f, 0.9f, 1.0f};
mat.doubleSided = true;
model.materials.push_back(mat);
std::unordered_map<int, int> tex_image_map;
// add all hi-lod tfrag trees
for (const auto& tfrag : level.tfrag_trees.at(0)) {
add_tfrag(level, tfrag, model, tex_image_map);
}
for (const auto& tie : level.tie_trees.at(0)) {
add_tie(level, tie, model, tex_image_map);
}
for (const auto& shrub : level.shrub_trees) {
add_shrub(level, shrub, model, tex_image_map);
}
model.asset.generator = "opengoal";
tinygltf::TinyGLTF gltf;
gltf.WriteGltfSceneToFile(&model, glb_file.string(),
true, // embedImages
true, // embedBuffers
true, // pretty print
true); // write binary
}
void save_level_foreground_as_gltf(const tfrag3::Level& level, const fs::path& glb_file) {
// the top level container for everything is the model.
tinygltf::Model model;
// a "scene" is a traditional scene graph, made up of Nodes.
// sadly, attempting to nest stuff makes the blender importer unhappy, so we just dump
// everything into the top level.
auto& scene = model.scenes.emplace_back();
// hack, add a default material.
tinygltf::Material mat;
mat.pbrMetallicRoughness.baseColorFactor = {1.0f, 0.9f, 0.9f, 1.0f};
mat.doubleSided = true;
model.materials.push_back(mat);
std::unordered_map<int, int> tex_image_map;
add_merc(level, model, tex_image_map);
model.asset.generator = "opengoal";
tinygltf::TinyGLTF gltf;
gltf.WriteGltfSceneToFile(&model, glb_file.string(),
true, // embedImages
true, // embedBuffers
true, // pretty print
true); // write binary
}

10
decompiler/level_extractor/fr3_to_gltf.h Normal file
View file

@ -0,0 +1,10 @@
#pragma once
#include "common/custom_data/Tfrag3Data.h"
#include "common/util/FileUtil.h"
/*!
* Export the background geometry (tie, tfrag, shrub) to a GLTF binary format (.glb) file.
*/
void save_level_background_as_gltf(const tfrag3::Level& level, const fs::path& glb_file);
void save_level_foreground_as_gltf(const tfrag3::Level& level, const fs::path& glb_file);

456
decompiler/level_extractor/tfrag_tie_fixup.cpp Normal file
View file

@ -0,0 +1,456 @@
#include "tfrag_tie_fixup.h"
#include <algorithm>
#include <set>
#include <unordered_map>
#include "common/math/Vector.h"
#include "common/util/Assert.h"
// Approach:
// 1: un-strip vertices and make individual strips consistent. Group triangles by strip.
// 2: build a graph of neighboring groups
// 3: find connected components
// 4: for each connected component, order strips in bfs order
// 5: iterate through the strips. If flipping the strip will result in fewer inconsistencies with
// lowered-ordered neighbors, then flip it.
// 6: build final index buffer.
// vertex indices for a single triangle
struct Tri {
u32 idx[3];
};
// a list of triangles, assumed to have consistent orientation.
struct TriGroup {
std::vector<Tri> tris;
};
/*!
* Step 1: convert strips to groups of un-stripped triangles, determine the old->new mapping.
* Note: for the old->new mapping, we don't have to give the right answer for triangles inside of a
* strip, or for degenerate strips. This mapping is just convenient for users of this data.
*/
void unstrip(const std::vector<u32>& stripped_indices,
std::vector<TriGroup>& groups,
std::vector<u32>& old_to_new_start) {
// first triangle is the first triangle
old_to_new_start.push_back(0);
// doesn't matter, in the middle of a strip
old_to_new_start.push_back(0);
// the tfrag output flips every other triangle, we'll need to unflip that.
bool toggle = false;
// total number of indices created in the output.
size_t num_unstripped_idx = 0;
// the current strip
TriGroup building_group;
// loop over all groups of 3 indices...
for (size_t i = 2; i < stripped_indices.size(); i++) {
u32 a = stripped_indices[i];
u32 b = stripped_indices[i - 1];
u32 c = stripped_indices[i - 2];
old_to_new_start.push_back(num_unstripped_idx);
if (a == UINT32_MAX || b == UINT32_MAX || c == UINT32_MAX) {
toggle = false;
if (!building_group.tris.empty()) {
groups.push_back(building_group);
building_group.tris.clear();
}
continue;
} else {
num_unstripped_idx += 3;
auto& tri = building_group.tris.emplace_back();
tri.idx[0] = a;
tri.idx[1] = toggle ? b : c;
tri.idx[2] = toggle ? c : b;
toggle = !toggle;
}
}
old_to_new_start.push_back(num_unstripped_idx);
}
// A Node in the graph of groups.
// The self and and neighbors fields refers to group indices.
struct Node {
int self_idx = -1;
std::set<int> neighbors;
};
u64 group_pair_to_neighbor_key(u64 a, u64 b) {
if (a < b) {
std::swap(a, b);
}
ASSERT(a < UINT32_MAX && b < UINT32_MAX);
return a | (b << 32);
}
// For each pair of groups that are neighbors:
struct GroupPairInfo {
// which groups are neighbors
int idx[2] = {-1, -1};
// per-triangle-sharing-an-edge
std::vector<bool> matched_edge_twists; // true if twisted
};
/*!
* Arbitrary ordering of vector3's.
* We can use this to make sure that Edge(a, b) and Edge(b, a) are considered the same edge
* by always storing an edge as [min(a,b), max(a,b)].
*/
bool greater_than(const math::Vector3f& a, const math::Vector3f& b) {
for (int dim = 0; dim < 3; dim++) {
if (a[dim] > b[dim]) {
return true;
} else if (a[dim] < b[dim]) {
return false;
}
}
// ASSERT(false);
return false;
}
math::Vector3f round_vector(const math::Vector3f& in) {
math::Vector3f rounded;
for (int i = 0; i < 3; i++) {
s64 x = in[i];
rounded[i] = x;
}
return rounded;
}
/*!
* Edge that can be hashed, and Edge(a, b) == Edge(b, a), using the trick described above.
*/
class HashableEdge {
public:
HashableEdge(const math::Vector3f& a, const math::Vector3f& b) {
if (greater_than(a, b)) {
m_greater_pt = a;
m_lesser_pt = b;
} else {
m_greater_pt = b;
m_lesser_pt = a;
}
}
bool operator==(const HashableEdge& other) const {
return m_lesser_pt == other.m_lesser_pt && m_greater_pt == other.m_greater_pt;
}
struct hash {
std::size_t operator()(const HashableEdge& in) const {
std::size_t result = 0;
for (int i = 0; i < 3; i++) {
result ^= std::hash<float>()(in.m_lesser_pt[i]) ^ std::hash<float>()(in.m_greater_pt[i]);
}
return result;
}
};
private:
math::Vector3f m_lesser_pt, m_greater_pt;
};
math::Vector3f triangle_normal(const Tri& tri, const std::vector<math::Vector3f>& positions) {
const auto& a = positions.at(tri.idx[0]);
const auto& b = positions.at(tri.idx[1]);
const auto& c = positions.at(tri.idx[2]);
return (b - a).cross(c - a);
}
float triangle_normal_dot(const Tri& a,
const Tri& b,
const std::vector<math::Vector3f>& positions) {
return triangle_normal(a, positions).dot(triangle_normal(b, positions));
}
struct PerEdgeInfo {
int source_group = -1;
int tri_idx = -1; // in group
int edge_idx = -1;
bool change_order = false;
};
/*!
* Step 2: build a graph of connected groups. Groups are considered connected if they share an edge.
* Also collect information about pairs of connected groups.
* @param nodes: the node for each group.
* @param neighbor_info: info for each pair of connected group, keyed on group_pair_to_neighbor_key
* @param groups: input groups
* @param positions: vertex position input
*/
void build_graph(std::vector<Node>& nodes,
std::unordered_map<u64, GroupPairInfo>& neighbor_info,
const std::vector<TriGroup>& groups,
const std::vector<math::Vector3f>& positions) {
nodes.reserve(groups.size());
// to avoid slow O(n^2) edge checks, we'll add all edges to a hash table, recording which
// groups they appear in.
std::unordered_map<HashableEdge, std::vector<PerEdgeInfo>, HashableEdge::hash> edge_info_map;
// first pass: set up nodes and build hash table
for (int group_idx = 0; group_idx < (int)groups.size(); group_idx++) {
const auto& group = groups[group_idx];
// add the node
auto& node = nodes.emplace_back();
node.self_idx = group_idx;
for (int tri_idx = 0; tri_idx < (int)group.tris.size(); tri_idx++) {
const auto& tri = group.tris[tri_idx];
for (int edge_idx = 0; edge_idx < 3; edge_idx++) {
u32 edge_indices[2] = {
tri.idx[(edge_idx + 0) % 3],
tri.idx[(edge_idx + 1) % 3],
};
HashableEdge edge(round_vector(positions.at(edge_indices[0])),
round_vector(positions.at(edge_indices[1])));
auto& info_list = edge_info_map[edge];
bool found = false;
for (auto& x : info_list) {
if (x.source_group == group_idx) {
found = true;
break;
}
}
if (found) {
continue;
}
auto& info = edge_info_map[edge].emplace_back();
info.source_group = group_idx;
info.tri_idx = tri_idx;
info.edge_idx = edge_idx;
info.change_order =
greater_than(positions.at(edge_indices[0]), positions.at(edge_indices[1]));
}
}
}
// second pass: loop over shared edges
int shared_edge_count = 0;
for (const auto& [edge, infos] : edge_info_map) {
// skip any edge that only shows up once.
if (infos.size() < 2) {
continue;
}
shared_edge_count++;
for (int i = 0; i < (int)infos.size(); i++) {
for (int j = 0; j < i; j++) {
const auto& info_a = infos.at(i);
const auto& info_b = infos.at(j);
int group_a = info_a.source_group;
int group_b = info_b.source_group;
if (info_a.source_group == info_b.source_group) {
fmt::print("duplicate edge in group!\n"); // ??
continue;
}
// link neighbors
nodes.at(group_a).neighbors.insert(group_b);
nodes.at(group_b).neighbors.insert(group_a);
// make neighbor info (or append to existing)
auto neighbor_key = group_pair_to_neighbor_key(group_a, group_b);
auto& info = neighbor_info[neighbor_key];
int a_idx = group_a > group_b ? 0 : 1;
int b_idx = 1 - a_idx;
info.idx[a_idx] = group_a;
info.idx[b_idx] = group_b;
info.matched_edge_twists.push_back(
info_a.change_order == info_b.change_order // should be opposite dirs.
// triangle_normal_dot(groups.at(group_a).tris.at(info_a.tri_idx),
// groups.at(group_b).tris.at(info_b.tri_idx), positions)
);
}
}
}
}
struct ConnectedComponent {
std::vector<int> groups;
};
/*!
* Step 3: find connected components
*/
std::vector<ConnectedComponent> find_connected_components(const std::vector<Node>& nodes) {
std::vector<ConnectedComponent> result;
std::set<int> added;
// loop over each node
for (int node_idx = 0; node_idx < (int)nodes.size(); node_idx++) {
// skip nodes already part of a connected component.
if (added.find(node_idx) != added.end()) {
continue;
}
// new node, create a new component and find everything connected.
auto& component = result.emplace_back();
// added this first node
component.groups.push_back(node_idx);
added.insert(node_idx);
// initialize the search with neighbors of the first node
std::vector<int> to_visit;
for (auto x : nodes.at(node_idx).neighbors) {
to_visit.push_back(x);
}
// find all connected!
while (!to_visit.empty()) {
int next_idx = to_visit.back();
to_visit.pop_back();
if (added.find(next_idx) != added.end()) {
// already seen it, skip!
continue;
}
// new node, add it
added.insert(next_idx);
component.groups.push_back(next_idx);
// also look at neighbors
for (auto x : nodes.at(next_idx).neighbors) {
if (added.find(x) == added.end()) {
to_visit.push_back(x);
}
}
}
}
return result;
}
/*!
* Step 4: order components in bfs order
*/
ConnectedComponent bfs_order_connected_component(const ConnectedComponent& in,
const std::vector<Node>& graph) {
ConnectedComponent out;
// add the first one
std::set<int> added;
std::vector<int> frontier = {in.groups.at(0)};
// go!
while (!frontier.empty()) {
std::stable_sort(frontier.begin(), frontier.end(), [&graph](int a, int b) {
return graph.at(a).neighbors.size() < graph.at(b).neighbors.size();
});
std::vector<int> next_frontier;
for (auto x : frontier) {
if (added.find(x) != added.end()) {
continue;
}
added.insert(x);
out.groups.push_back(x);
auto& node = graph.at(x);
for (auto& neighbor : node.neighbors) {
if (added.find(neighbor) == added.end()) {
next_frontier.push_back(neighbor);
}
}
}
frontier = std::move(next_frontier);
}
return out;
}
std::vector<bool> compute_flips(const std::vector<Node>& graph,
const std::vector<ConnectedComponent>& components,
const std::unordered_map<u64, GroupPairInfo>& neighbor_info) {
std::vector<bool> flips;
flips.resize(graph.size(), false);
for (const auto& component : components) {
std::set<int> decided;
for (int node_idx : component.groups) {
const auto& group = graph.at(node_idx);
float flip_sum = 0;
for (int neighbor_idx : group.neighbors) {
if (decided.find(neighbor_idx) == decided.end()) {
continue; // skip, don't know this ones orientation
}
const auto& info = neighbor_info.at(group_pair_to_neighbor_key(node_idx, neighbor_idx));
for (auto originally_twisted : info.matched_edge_twists) {
bool twisted = originally_twisted;
if (flips[neighbor_idx]) {
twisted = !twisted;
}
if (twisted) {
flip_sum--;
} else {
flip_sum++;
}
}
}
if (flip_sum < 0) {
flips[node_idx] = true;
}
decided.insert(node_idx);
}
}
return flips;
}
void apply_flips(const std::vector<bool>& flips, std::vector<TriGroup>& groups) {
ASSERT(flips.size() == groups.size());
for (size_t i = 0; i < groups.size(); i++) {
if (flips[i]) {
for (auto& tri : groups[i].tris) {
std::swap(tri.idx[0], tri.idx[1]);
}
}
}
}
void make_final_indices(const std::vector<TriGroup>& groups, std::vector<u32>& idx) {
for (auto& g : groups) {
for (auto& t : g.tris) {
for (auto i : t.idx) {
idx.push_back(i);
}
}
}
}
void fixup_and_unstrip_tfrag_tie(const std::vector<u32>& stripped_indices,
const std::vector<math::Vector3f>& positions,
std::vector<u32>& unstripped,
std::vector<u32>& old_to_new_start) {
// Part 1
std::vector<TriGroup> groups;
unstrip(stripped_indices, groups, old_to_new_start);
// Part 2
std::vector<Node> nodes;
std::unordered_map<u64, GroupPairInfo> neighbor_info;
build_graph(nodes, neighbor_info, groups, positions);
// Part 3
auto connected_components = find_connected_components(nodes);
// Part 4
std::vector<ConnectedComponent> ordered_components;
for (auto& c : connected_components) {
ordered_components.push_back(bfs_order_connected_component(c, nodes));
}
// Part 5
auto flips = compute_flips(nodes, ordered_components, neighbor_info);
// Part 6
apply_flips(flips, groups);
// Part 7
make_final_indices(groups, unstripped);
}

25
decompiler/level_extractor/tfrag_tie_fixup.h Normal file
View file

@ -0,0 +1,25 @@
#pragma once
#include <vector>
#include "common/common_types.h"
#include "common/math/Vector.h"
/*!
* Fix-up tfrag/tie format mesh to a best-guess unstripped mesh with proper triangle orientation.
* The input is the tie/tfrag index list, using UINT32_MAX as primitive restart.
* The output is an unstripped index list (groups of three indices, one for each triangle)
* and a map from each element in the original index buffer to the new one.
* Ex: unstripped[old_to_new_start[i]] is the start of prim stripped_indices[i].
*
* This depends on specific behavior of the original tfrag/tie renderers, and how
* extract_tie/extract_tfrag work (basically keeping the order of vertices exactly the same as in VU
* memory).
*
* The stripping logic of shrub/merc/generic models appears to be different, and this likely won't
* work.
*/
void fixup_and_unstrip_tfrag_tie(const std::vector<u32>& stripped_indices,
const std::vector<math::Vector3f>& positions,
std::vector<u32>& unstripped,
std::vector<u32>& old_to_new_start);

2
game/graphics/opengl_renderer/Sprite3.cpp
View file

@ -415,7 +415,7 @@ void Sprite3::distort_dma(DmaFollower& dma, ScopedProfilerNode& /*prof*/) {
} else {
// VU address >= 512 is the actual vertex data
ASSERT(dest >= 512);
ASSERT(sprite_idx + (qwc / 3) <= m_sprite_distorter_frame_data.capacity());
ASSERT(sprite_idx + (qwc / 3) <= (int)m_sprite_distorter_frame_data.capacity());
unpack_to_no_stcycl(&m_sprite_distorter_frame_data.at(sprite_idx), distort_data,
VifCode::Kind::UNPACK_V4_32, qwc * 16, dest, false, false);

10
game/graphics/opengl_renderer/foreground/Merc2.cpp
View file

@ -384,7 +384,7 @@ void Merc2::handle_merc_chain(DmaFollower& dma,
* Queue up some bones to be included in the bone buffer.
* Returns the index of the first bone vector.
*/
u32 Merc2::alloc_bones(int count, float scale) {
u32 Merc2::alloc_bones(int count) {
u32 first_bone_vector = m_next_free_bone_vector;
ASSERT(count * 8 + first_bone_vector <= MAX_SHADER_BONE_VECTORS);
@ -397,12 +397,10 @@ u32 Merc2::alloc_bones(int count, float scale) {
auto* shader_mat = &m_shader_bone_vector_buffer[m_next_free_bone_vector];
int bv = 0;
// scale the transformation matrix (todo: can we move this to the extraction)
// and copy to the large bone buffer.
for (int j = 0; j < 3; j++) {
shader_mat[bv++] = skel_mat.tmat[j] * scale;
for (int j = 0; j < 4; j++) {
shader_mat[bv++] = skel_mat.tmat[j];
}
shader_mat[bv++] = skel_mat.tmat[3];
for (int j = 0; j < 3; j++) {
shader_mat[bv++] = skel_mat.nmat[j];
@ -487,7 +485,7 @@ void Merc2::flush_pending_model(SharedRenderState* render_state, ScopedProfilerN
return;
}
u32 first_bone = alloc_bones(bone_count, model->scale_xyz);
u32 first_bone = alloc_bones(bone_count);
// allocate lights
u32 lights = alloc_lights(m_current_lights);

2
game/graphics/opengl_renderer/foreground/Merc2.h
View file

@ -54,7 +54,7 @@ class Merc2 : public BucketRenderer {
void handle_matrix_dma(const DmaTransfer& dma);
void flush_pending_model(SharedRenderState* render_state, ScopedProfilerNode& prof);
u32 alloc_bones(int count, float scale);
u32 alloc_bones(int count);
std::optional<MercRef> m_current_model = std::nullopt;
u16 m_current_effect_enable_bits = 0;

4
game/graphics/pipelines/opengl.cpp
View file

@ -164,7 +164,7 @@ static void gl_exit() {
static std::shared_ptr<GfxDisplay> gl_make_display(int width,
int height,
const char* title,
GfxSettings& settings,
GfxSettings& /*settings*/,
GameVersion game_version,
bool is_main) {
GLFWwindow* window = glfwCreateWindow(width, height, title, NULL, NULL);
@ -317,7 +317,7 @@ void GLDisplay::on_window_size(GLFWwindow* /*window*/, int width, int height) {
}
}
void GLDisplay::on_iconify(GLFWwindow* window, int iconified) {
void GLDisplay::on_iconify(GLFWwindow* /*window*/, int iconified) {
m_minimized = iconified == GLFW_TRUE;
}

26
game/graphics/pipelines/opengl.h
View file

@ -23,21 +23,21 @@ class GLDisplay : public GfxDisplay {
GLDisplay(GLFWwindow* window, bool is_main);
virtual ~GLDisplay();
void* get_window() const { return m_window; }
void get_position(int* x, int* y);
void get_size(int* w, int* h);
void get_scale(float* x, float* y);
void get_screen_size(int vmode_idx, s32* w, s32* h);
int get_screen_rate(int vmode_idx);
int get_screen_vmode_count();
int get_monitor_count();
void set_size(int w, int h);
void update_fullscreen(GfxDisplayMode mode, int screen);
void render();
bool minimized();
void* get_window() const override { return m_window; }
void get_position(int* x, int* y) override;
void get_size(int* w, int* h) override;
void get_scale(float* x, float* y) override;
void get_screen_size(int vmode_idx, s32* w, s32* h) override;
int get_screen_rate(int vmode_idx) override;
int get_screen_vmode_count() override;
int get_monitor_count() override;
void set_size(int w, int h) override;
void update_fullscreen(GfxDisplayMode mode, int screen) override;
void render() override;
bool minimized() override;
bool fullscreen_pending() override;
void fullscreen_flush() override;
void set_lock(bool lock);
void set_lock(bool lock) override;
void on_key(GLFWwindow* window, int key, int scancode, int action, int mods);
void on_window_pos(GLFWwindow* window, int xpos, int ypos);
void on_window_size(GLFWwindow* window, int width, int height);

2
game/kernel/jak2/kmachine.cpp
View file

@ -532,7 +532,7 @@ void PutDisplayEnv(u32 /*ptr*/) {
ASSERT(false);
}
u32 sceGsSyncV(u32 mode) {
u32 sceGsSyncV(u32 /*mode*/) {
// stub, jak2 probably works differently here
ASSERT(false);
return 0;

13
game/kernel/jak2/kscheme.cpp
View file

@ -1739,22 +1739,22 @@ int InitHeapAndSymbol() {
return 0;
}
u64 load(u32 file_name_in, u32 heap_in) {
u64 load(u32 /*file_name_in*/, u32 /*heap_in*/) {
ASSERT(false);
return 0;
}
u64 loadb(u32 file_name_in, u32 heap_in, u32 param3) {
u64 loadb(u32 /*file_name_in*/, u32 /*heap_in*/, u32 /*param3*/) {
ASSERT(false);
return 0;
}
u64 loadc(const char* file_name, kheapinfo* heap, u32 flags) {
u64 loadc(const char* /*file_name*/, kheapinfo* /*heap*/, u32 /*flags*/) {
ASSERT(false);
return 0;
}
u64 loado(u32 file_name_in, u32 heap_in) {
u64 loado(u32 /*file_name_in*/, u32 /*heap_in*/) {
ASSERT(false);
return 0;
}
@ -1767,7 +1767,10 @@ u64 unload(u32 name) {
return 0;
}
s64 load_and_link(const char* filename, char* decode_name, kheapinfo* heap, u32 flags) {
s64 load_and_link(const char* /*filename*/,
char* /*decode_name*/,
kheapinfo* /*heap*/,
u32 /*flags*/) {
ASSERT(false);
return 0;
}

1
goalc/build_level/build_level.cpp
View file

@ -49,6 +49,7 @@ bool run_build_level(const std::string& input_file,
mesh_extract_in.filename =
file_util::get_file_path({level_json.at("gltf_file").get<std::string>()});
mesh_extract_in.auto_wall_enable = level_json.value("automatic_wall_detection", true);
mesh_extract_in.double_sided_collide = level_json.at("double_sided_collide").get<bool>();
mesh_extract_in.auto_wall_angle = level_json.value("automatic_wall_angle", 30.0);
mesh_extract_in.tex_pool = &tex_pool;
gltf_mesh_extract::Output mesh_extract_out;

271
goalc/build_level/collide_bvh.cpp
View file

@ -1,6 +1,7 @@
#include "collide_bvh.h"
#include <algorithm>
#include <unordered_set>
#include "common/log/log.h"
#include "common/util/Assert.h"
@ -16,18 +17,7 @@
namespace collide {
namespace {
constexpr int MAX_FACES_IN_FRAG = 100;
/*!
* The Collide node.
* Has either children collide node or children faces, but not both
* The size of child_nodes is either 0 or 8 at all times.
*/
struct CNode {
std::vector<CNode> child_nodes;
std::vector<CollideFace> faces;
math::Vector4f bsphere;
};
constexpr int MAX_UNIQUE_VERTS_IN_FRAG = 128;
struct BBox {
math::Vector3f mins, maxs;
@ -38,6 +28,40 @@ struct BBox {
}
};
/*!
* The Collide node.
* If it's a leaf, it has faces
* Otherwise it has 2, 4, or 8 children nodes.
*/
struct CNode {
std::vector<CNode> child_nodes;
std::vector<CollideFace> faces;
math::Vector4f bsphere;
};
struct VectorHash {
size_t operator()(const math::Vector3f& in) const {
return std::hash<float>()(in.x()) ^ std::hash<float>()(in.y()) ^ std::hash<float>()(in.z());
}
};
/*!
* Recursively get a set of unique vertices.
*/
void collect_vertices(const CNode& node, std::unordered_set<math::Vector3f, VectorHash>& verts) {
for (auto& child : node.child_nodes) {
collect_vertices(child, verts);
}
for (auto& face : node.faces) {
verts.insert(face.v[0]);
verts.insert(face.v[1]);
verts.insert(face.v[2]);
}
}
/*!
* Recursively get a list of vertices.
*/
void collect_vertices(const CNode& node, std::vector<math::Vector3f>& verts) {
for (auto& child : node.child_nodes) {
collect_vertices(child, verts);
@ -49,6 +73,33 @@ void collect_vertices(const CNode& node, std::vector<math::Vector3f>& verts) {
}
}
/*!
* Get the axis-aligned bounding box of these vertices
*/
BBox compute_my_bbox(const std::vector<math::Vector3f>& verts) {
ASSERT(!verts.empty());
BBox result;
result.mins = verts.front();
result.maxs = verts.front();
for (auto& v : verts) {
result.mins.min_in_place(v);
result.maxs.min_in_place(v);
}
return result;
}
/*!
* Get the axis-aligned bounding box of all vertices in this node and its children
*/
BBox compute_my_bbox(const CNode& node) {
std::vector<math::Vector3f> verts;
collect_vertices(node, verts);
return compute_my_bbox(verts);
}
/*!
* Find the vertex in verts that is most distant from pt.
*/
size_t find_most_distant(math::Vector3f pt, const std::vector<math::Vector3f>& verts) {
float max_dist_squared = 0;
size_t idx_of_best = 0;
@ -62,9 +113,10 @@ size_t find_most_distant(math::Vector3f pt, const std::vector<math::Vector3f>& v
return idx_of_best;
}
void compute_my_bsphere_ritters(CNode& node) {
std::vector<math::Vector3f> verts;
collect_vertices(node, verts);
/*!
* Compute a bounding sphere for a node and its children.
*/
void compute_my_bsphere_ritters(CNode& node, const std::vector<math::Vector3f>& verts) {
ASSERT(verts.size() > 0);
auto px = verts[0];
auto py = verts[find_most_distant(px, verts)];
@ -82,6 +134,15 @@ void compute_my_bsphere_ritters(CNode& node) {
node.bsphere.w() = std::sqrt(max_squared);
}
/*!
* Compute a bounding sphere for a node and its children.
*/
void compute_my_bsphere_ritters(CNode& node) {
std::vector<math::Vector3f> verts;
collect_vertices(node, verts);
compute_my_bsphere_ritters(node, verts);
}
/*!
* Split faces in two along a coordinate plane.
* Will clear the input faces
@ -132,76 +193,85 @@ void split_node_once(CNode& node, CNode* out0, CNode* out1) {
*out1 = temps[best_dim * 2 + 1];
}
/*!
* Split a node into 8 children and store these in the given node.
*/
void split_node_to_8_children(CNode& node) {
bool needs_split(const CNode& node) {
// quick reject.
if (node.faces.size() > 100) {
return true;
}
if (node.bsphere.w() > (125.f * 4096.f)) {
return true;
}
ASSERT(node.child_nodes.empty());
node.child_nodes.resize(8);
// level 0
std::unordered_set<math::Vector3f, VectorHash> unique_verts;
for (auto& f : node.faces) {
for (auto& v : f.v) {
unique_verts.insert(v);
}
}
return unique_verts.size() >= 128;
}
void split_recursive(CNode& to_split) {
ASSERT(to_split.child_nodes.empty());
ASSERT(!to_split.faces.empty());
CNode level0[2];
split_node_once(node, &level0[0], &level0[1]);
// level 1
CNode level1[4];
split_node_once(level0[0], &level1[0], &level1[1]);
split_node_once(level0[1], &level1[2], &level1[3]);
// level 2
split_node_once(level1[0], &node.child_nodes[0], &node.child_nodes[1]);
split_node_once(level1[1], &node.child_nodes[2], &node.child_nodes[3]);
split_node_once(level1[2], &node.child_nodes[4], &node.child_nodes[5]);
split_node_once(level1[3], &node.child_nodes[6], &node.child_nodes[7]);
}
struct SplitResult {
size_t max_leaf_count = 0;
float max_bsphere_w = 0;
};
/*!
* Split all leaf nodes. Returns the number of faces in the leaf with the most faces after
* splitting.
* This slightly unusual recursion pattern is to make sure we split everything to same depth,
* which we believe might be a requirement of the collision system.
*/
SplitResult split_all_leaves(CNode& node) {
SplitResult result;
if (node.child_nodes.empty()) {
// we're a leaf!
// split us:
split_node_to_8_children(node);
for (auto& child : node.child_nodes) {
result.max_leaf_count = std::max(result.max_leaf_count, child.faces.size());
result.max_bsphere_w = std::max(result.max_bsphere_w, child.bsphere.w());
}
return result;
} else {
// not a leaf, recurse
for (auto& child : node.child_nodes) {
auto cret = split_all_leaves(child);
result.max_bsphere_w = std::max(result.max_bsphere_w, cret.max_bsphere_w);
result.max_leaf_count = std::max(result.max_leaf_count, cret.max_leaf_count);
}
return result;
}
}
/*!
* Main BVH construction function. Splits leaves until it is no longer needed.
*/
void split_as_needed(CNode& root) {
int initial_tri_count = root.faces.size();
int num_leaves = 1;
bool need_to_split = true;
while (need_to_split) {
SplitResult worst = split_all_leaves(root);
num_leaves *= 8;
lg::info("after splitting, the worst leaf has {} tris, {} radius", worst.max_leaf_count,
worst.max_bsphere_w / 4096.f);
if (worst.max_leaf_count < MAX_FACES_IN_FRAG && worst.max_bsphere_w < (125.f * 4096.f)) {
need_to_split = false;
split_node_once(to_split, &level0[0], &level0[1]);
for (int i = 0; i < 2; i++) {
if (needs_split(level0[i])) {
CNode level1[2];
split_node_once(level0[i], &level1[0], &level1[1]);
for (int j = 0; j < 2; j++) {
if (needs_split(level1[j])) {
CNode level2[2];
split_node_once(level1[j], &level2[0], &level2[1]);
for (int k = 0; k < 2; k++) {
if (needs_split(level2[k])) {
to_split.child_nodes.push_back(std::move(level2[k]));
split_recursive(to_split.child_nodes.back());
} else {
to_split.child_nodes.push_back(std::move(level2[k]));
}
}
} else {
to_split.child_nodes.push_back(std::move(level1[j]));
}
}
} else {
to_split.child_nodes.push_back(std::move(level0[i]));
}
}
ASSERT(to_split.child_nodes.size() <= 8);
bool has_leaves = false;
bool has_not_leaves = false;
for (auto& child : to_split.child_nodes) {
if (!child.faces.empty()) {
has_leaves = true;
}
if (!child.child_nodes.empty()) {
has_not_leaves = true;
}
}
if (has_leaves && has_not_leaves) {
std::vector<CNode> temp_children = std::move(to_split.child_nodes);
to_split.child_nodes = {};
for (auto& c : temp_children) {
if (!c.faces.empty()) {
to_split.child_nodes.emplace_back();
to_split.child_nodes.emplace_back();
split_node_once(c, &to_split.child_nodes[to_split.child_nodes.size() - 1],
&to_split.child_nodes[to_split.child_nodes.size() - 2]);
} else {
to_split.child_nodes.push_back(std::move(c));
}
}
}
lg::info("average triangles per leaf: {}", initial_tri_count / num_leaves);
lg::info("leaf count: {}", num_leaves);
}
/*!
@ -215,35 +285,34 @@ void bsphere_recursive(CNode& node) {
}
}
void drawable_layout_helper(CNode& node, int depth, CollideTree& tree_out, size_t my_idx_check) {
if (node.child_nodes.empty()) {
// we're a leaf! add us to the frags
auto& frag = tree_out.frags.frags.emplace_back();
frag.bsphere = node.bsphere;
frag.faces = node.faces;
void drawable_layout_helper(const CNode& node_in,
CollideTree& tree_out,
DrawNode& parent_to_add_to) {
if (node_in.faces.empty()) {
ASSERT(!node_in.child_nodes.empty());
auto& next = parent_to_add_to.draw_node_children.emplace_back();
next.bsphere = node_in.bsphere;
for (auto& c : node_in.child_nodes) {
drawable_layout_helper(c, tree_out, next);
}
} else {
// not a leaf
if ((int)tree_out.node_arrays.size() <= depth) {
tree_out.node_arrays.resize(depth + 1);
}
ASSERT(my_idx_check == tree_out.node_arrays.at(depth).nodes.size());
auto& draw_node = tree_out.node_arrays[depth].nodes.emplace_back();
draw_node.bsphere = node.bsphere;
for (int i = 0; i < 8; i++) {
draw_node.children[i] = my_idx_check * 8 + i;
drawable_layout_helper(node.child_nodes.at(i), depth + 1, tree_out, draw_node.children[i]);
}
ASSERT(node_in.child_nodes.empty());
size_t frag_idx = tree_out.frags.frags.size();
auto& frag_out = tree_out.frags.frags.emplace_back();
frag_out.faces = node_in.faces;
frag_out.bsphere = node_in.bsphere;
parent_to_add_to.frag_children.push_back((int)frag_idx);
}
}
CollideTree build_collide_tree(CNode& root) {
CollideTree tree;
drawable_layout_helper(root, 0, tree, 0);
drawable_layout_helper(root, tree, tree.fake_root_node);
return tree;
}
void debug_stats(const CollideTree& tree) {
lg::info("Tree build: {} draw node layers", tree.node_arrays.size());
float sum_w = 0, max_w = 0;
for (auto& frag : tree.frags.frags) {
sum_w += frag.bsphere.w();
@ -261,7 +330,7 @@ CollideTree construct_collide_bvh(const std::vector<CollideFace>& tris) {
lg::info("Building collide bvh from {} triangles", tris.size());
CNode root;
root.faces = tris;
split_as_needed(root);
split_recursive(root);
lg::info("BVH tree constructed in {:.2f} ms", bvh_timer.getMs());
// part 2: compute bspheres

6
goalc/build_level/collide_bvh.h
View file

@ -12,7 +12,8 @@
namespace collide {
struct DrawNode {
s32 children[8] = {-1, -1, -1, -1, -1, -1, -1, -1};
std::vector<DrawNode> draw_node_children;
std::vector<int> frag_children;
math::Vector4f bsphere;
};
@ -30,7 +31,8 @@ struct DrawableInlineArrayCollideFrag {
};
struct CollideTree {
std::vector<DrawableInlineArrayNode> node_arrays;
// std::vector<DrawableInlineArrayNode> node_arrays;
DrawNode fake_root_node; // the children of this are the ones that go in the top level.
DrawableInlineArrayCollideFrag frags;
};

166
goalc/build_level/collide_drawable.cpp
View file

@ -1,5 +1,7 @@
#include "collide_drawable.h"
#include <unordered_set>
#include "common/util/Assert.h"
#include "goalc/data_compiler/DataObjectGenerator.h"
@ -136,7 +138,7 @@ size_t generate_collide_fragment(DataObjectGenerator& gen,
size_t generate_collide_fragment_array(DataObjectGenerator& gen,
const std::vector<CollideFragMeshData>& meshes,
const std::vector<size_t>& frag_mesh_locs,
std::vector<size_t>& parent_ref_out) {
std::vector<size_t>& loc_out) {
gen.align_to_basic();
gen.add_type_tag("drawable-inline-array-collide-fragment"); // 0
size_t result = gen.current_offset_bytes();
@ -161,22 +163,32 @@ size_t generate_collide_fragment_array(DataObjectGenerator& gen,
for (int j = 0; j < 4; j++) {
gen.add_word_float(mesh.bsphere[j]);
}
if ((i % 8) == 0) {
parent_ref_out.push_back(me);
}
loc_out.push_back(me);
}
return result;
}
size_t generate_collide_draw_node_array(DataObjectGenerator& gen,
const std::vector<collide::DrawNode>& nodes,
u32 flag,
const std::vector<size_t>& children,
std::vector<size_t>& parent_ref_out) {
int child_count(const collide::DrawNode* node) {
if (node->frag_children.empty()) {
return node->draw_node_children.size();
} else {
return node->frag_children.size();
}
}
std::unordered_map<const collide::DrawNode*, size_t> add_draw_nodes(
DataObjectGenerator& gen,
const std::vector<const collide::DrawNode*>& nodes,
const std::vector<size_t>& frag_locs,
size_t& array_out) {
std::unordered_map<const collide::DrawNode*, size_t> result;
std::unordered_map<size_t, const collide::DrawNode*> back_map;
gen.align_to_basic();
gen.add_type_tag("drawable-inline-array-node"); // 0
size_t result = gen.current_offset_bytes();
array_out = gen.current_offset_bytes();
ASSERT(nodes.size() < UINT16_MAX);
gen.add_word(nodes.size() << 16); // 4, 6
gen.add_word(0); // 8
gen.add_word(0); // 12
@ -185,30 +197,72 @@ size_t generate_collide_draw_node_array(DataObjectGenerator& gen,
gen.add_word(0); // 24
gen.add_word(0); // 28
ASSERT(nodes.size() == children.size());
for (size_t i = 0; i < nodes.size(); i++) {
auto& node = nodes[i];
for (auto& node : nodes) {
bool is_draw_node = node->frag_children.empty();
// should be 8 words here:
gen.add_type_tag("draw-node"); // 1
size_t me = gen.current_offset_bytes();
u32 packed_flags = 0;
packed_flags |= (8 << 16); // TODO hard-coded size here
packed_flags |= (flag << 24);
gen.add_word(packed_flags); // 2
gen.link_word_to_byte(gen.add_word(0), children[i]); // 3
gen.add_word(0); // 4
if ((i % 8) == 0) {
parent_ref_out.push_back(me);
packed_flags |=
((is_draw_node ? node->draw_node_children.size() : node->frag_children.size()) << 16);
packed_flags |= ((is_draw_node ? 1 : 0) << 24);
gen.add_word(packed_flags); // 2
if (is_draw_node) {
// gen.link_word_to_byte(gen.add_word(0), result.at(&node->draw_node_children.at(0))); // 3
back_map[gen.add_word(0)] = &node->draw_node_children.at(0);
} else {
gen.link_word_to_byte(gen.add_word(0), frag_locs.at(node->frag_children.at(0)));
}
gen.add_word_float(node.bsphere.x()); // 5
gen.add_word_float(node.bsphere.y()); // 6
gen.add_word_float(node.bsphere.z()); // 7
gen.add_word_float(node.bsphere.w()); // 8
result[node] = me;
gen.add_word(0); // 4
gen.add_word_float(node->bsphere.x()); // 5
gen.add_word_float(node->bsphere.y()); // 6
gen.add_word_float(node->bsphere.z()); // 7
gen.add_word_float(node->bsphere.w()); // 8
}
for (const auto& [loc, node] : back_map) {
gen.link_word_to_byte(loc, result.at(node));
}
return result;
}
std::vector<const collide::DrawNode*> bfs_nodes(const collide::DrawNode& fake_root) {
std::vector<const collide::DrawNode*> out;
std::unordered_set<const collide::DrawNode*> added;
std::vector<const collide::DrawNode*> frontier;
for (auto& dnc : fake_root.draw_node_children) {
frontier.push_back(&dnc);
}
while (!frontier.empty()) {
std::vector<const collide::DrawNode*> next_frontier;
for (auto x : frontier) {
if (added.find(x) != added.end()) {
continue;
}
added.insert(x);
out.push_back(x);
for (auto& child : x->draw_node_children) {
if (added.find(&child) == added.end()) {
next_frontier.push_back(&child);
}
}
}
frontier = std::move(next_frontier);
}
return out;
}
size_t DrawableTreeCollideFragment::add_to_object_file(DataObjectGenerator& gen) const {
// generate pat array
size_t pat_array_loc = generate_pat_array(gen, packed_frags.pats);
@ -226,38 +280,64 @@ size_t DrawableTreeCollideFragment::add_to_object_file(DataObjectGenerator& gen)
packed_data_locs[i], pat_array_loc));
}
std::vector<size_t> array_locs;
array_locs.resize(bvh.node_arrays.size() + 1); // plus one for the frags.
int array_slot = bvh.node_arrays.size();
std::vector<size_t> children_refs;
array_locs[array_slot--] = generate_collide_fragment_array(gen, packed_frags.packed_frag_data,
collide_frag_meshes, children_refs);
u32 flag = 0;
while (array_slot >= 0) {
ASSERT(children_refs.size() == bvh.node_arrays.at(array_slot).nodes.size());
std::vector<size_t> next_children;
generate_collide_fragment_array(gen, packed_frags.packed_frag_data, collide_frag_meshes,
children_refs);
array_locs[array_slot] = generate_collide_draw_node_array(
gen, bvh.node_arrays.at(array_slot).nodes, flag, children_refs, next_children);
auto order = bfs_nodes(bvh.fake_root_node);
size_t others_array;
auto located_nodes = add_draw_nodes(gen, order, children_refs, others_array);
children_refs = std::move(next_children);
array_slot--;
flag = 1;
size_t root_dian = -1;
{
gen.align_to_basic();
gen.add_type_tag("drawable-inline-array-node"); // 0
root_dian = gen.current_offset_bytes();
gen.add_word(bvh.fake_root_node.draw_node_children.size() << 16); // 4, 6
gen.add_word(0); // 8
gen.add_word(0); // 12
gen.add_word(0); // 16
gen.add_word(0); // 20
gen.add_word(0); // 24
gen.add_word(0); // 28
for (auto& node : bvh.fake_root_node.draw_node_children) {
bool is_draw_node = node.frag_children.empty();
// should be 8 words here:
gen.add_type_tag("draw-node"); // 1
u32 packed_flags = 0;
packed_flags |=
((is_draw_node ? node.draw_node_children.size() : node.frag_children.size()) << 16);
packed_flags |= ((is_draw_node ? 1 : 0) << 24);
gen.add_word(packed_flags); // 2
if (is_draw_node) {
gen.link_word_to_byte(gen.add_word(0),
located_nodes.at(&node.draw_node_children.at(0))); // 3
} else {
gen.link_word_to_byte(gen.add_word(0), children_refs.at(node.frag_children.at(0)));
}
gen.add_word(0); // 4
gen.add_word_float(node.bsphere.x()); // 5
gen.add_word_float(node.bsphere.y()); // 6
gen.add_word_float(node.bsphere.z()); // 7
gen.add_word_float(node.bsphere.w()); // 8
}
}
{
gen.align_to_basic();
gen.add_type_tag("drawable-tree-collide-fragment");
size_t result = gen.current_offset_bytes();
gen.add_word((array_locs.size() - 1) << 16); // todo the minus one here??
gen.add_word(2 << 16); // todo the minus one here??
for (int i = 0; i < 6; i++) {
gen.add_word(0);
}
for (size_t i = 1; i < array_locs.size(); i++) { // todo the offset here?
gen.link_word_to_byte(gen.add_word(0), array_locs[i]);
}
gen.link_word_to_byte(gen.add_word(0), root_dian);
gen.link_word_to_byte(gen.add_word(0), others_array);
return result;
}

8
goalc/build_level/collide_pack.cpp
View file

@ -200,6 +200,12 @@ CollideFragMeshDataArray pack_collide_frags(const std::vector<collide::CollideFr
auto indexed = dedup_frag_mesh(frag_in, &pat_map);
// first part of packed_data is the u16 vertex data:
frag_out.vertex_count = indexed.vertices_u16.vertex.size();
if (frag_out.vertex_count > 128) {
fmt::print("frag with too many vertices: {} had {} tris\n", frag_out.vertex_count,
frag_in.faces.size());
lg::error("SHOULD CRASH\n");
}
// the
frag_out.packed_data.resize(sizeof(u16) * frag_out.vertex_count * 3);
memcpy(frag_out.packed_data.data(), indexed.vertices_u16.vertex.data(),
frag_out.packed_data.size());
@ -228,12 +234,14 @@ CollideFragMeshDataArray pack_collide_frags(const std::vector<collide::CollideFr
// gonna guess here:
frag_out.poly_count = indexed.faces.size();
frag_out.total_qwc = frag_out.packed_data.size() / 16;
ASSERT(frag_out.total_qwc <= 128);
frag_out.base_trans_xyz_s32 = indexed.vertices_u16.base;
frag_out.bsphere = frag_in.bsphere;
total_pack_bytes += frag_out.packed_data.size();
}
result.pats = pat_map.pats;
lg::info("Collide pack used {} unique pats", result.pats.size());
lg::info("Total packed data size: {} kB, took {:.2f} ms", total_pack_bytes / 1024,
pack_timer.getMs());
return result;

23
goalc/build_level/gltf_mesh_extract.cpp
View file

@ -58,7 +58,7 @@ std::vector<math::Vector2f> extract_vec2f(const u8* data, u32 count, u32 stride)
}
/*!
* Convert a GLTF color buffer to u8 colors.
* Convert a GLTF color buffer (u16 format) to u8 colors.
*/
std::vector<math::Vector<u8, 4>> extract_color_from_vec4_u16(const u8* data,
u32 count,
@ -111,6 +111,9 @@ struct ExtractedVertices {
std::vector<math::Vector3f> normals;
};
/*!
* Extract positions, colors, and normals from a mesh.
*/
ExtractedVertices gltf_vertices(const tinygltf::Model& model,
const std::map<std::string, int>& attributes,
const math::Matrix4f& w_T_local,
@ -298,6 +301,8 @@ int texture_pool_add_texture(TexturePool* pool, const tinygltf::Image& tex) {
if (existing != pool->textures_by_name.end()) {
lg::info("Reusing image: {}", tex.name);
return existing->second;
} else {
lg::info("adding new texture: {}, size {} kB", tex.name, tex.width * tex.height * 4 / 1024);
}
ASSERT(tex.bits == 8);
@ -313,9 +318,9 @@ int texture_pool_add_texture(TexturePool* pool, const tinygltf::Image& tex) {
tt.debug_tpage_name = "custom-level";
tt.load_to_pool = false;
tt.combo_id = 0; // doesn't matter, not a pool tex
tt.data.resize(tt.w * tt.h * 4);
tt.data.resize(tt.w * tt.h);
ASSERT(tex.image.size() >= tt.data.size());
memcpy(tt.data.data(), tex.image.data(), tt.data.size());
memcpy(tt.data.data(), tex.image.data(), tt.data.size() * 4);
return idx;
}
} // namespace
@ -671,7 +676,7 @@ struct PatResult {
PatSurface pat;
};
PatResult custom_props_to_pat(const tinygltf::Value& val, const std::string& debug_name) {
PatResult custom_props_to_pat(const tinygltf::Value& val, const std::string& /*debug_name*/) {
PatResult result;
if (!val.IsObject() || !val.Has("set_collision") || !val.Get("set_collision").Get<int>()) {
// unset.
@ -806,6 +811,16 @@ void extract(const Input& in,
fixed_faces.push_back(face);
}
}
if (in.double_sided_collide) {
size_t os = fixed_faces.size();
for (size_t i = 0; i < os; i++) {
auto f0 = fixed_faces.at(i);
std::swap(f0.v[0], f0.v[1]);
fixed_faces.push_back(f0);
}
}
out.faces = std::move(fixed_faces);
if (in.auto_wall_enable) {

1
goalc/build_level/gltf_mesh_extract.h
View file

@ -15,6 +15,7 @@ struct Input {
bool get_colors = true;
bool auto_wall_enable = true;
float auto_wall_angle = 30.f;
bool double_sided_collide = false;
};
struct TfragOutput {

2
test/offline/offline_test_main.cpp
View file

@ -423,7 +423,7 @@ int main(int argc, char* argv[]) {
lg::info("Finding files...");
auto files = find_files(game_name, config->dgos);
if (max_files > 0 && max_files < files.size()) {
if (max_files > 0 && max_files < (int)files.size()) {
files.erase(files.begin() + max_files, files.end());
}

2
third-party/libco/CMakeLists.txt generated vendored
View file

@ -1,4 +1,4 @@
set(CMAKE_C_STANDARD 17)
set(CMAKE_C_STANDARD 11)
set(LIBCO_SOURCES
libco.c

5
tools/memory_dump_tool/main.cpp
View file

@ -582,11 +582,6 @@ void inspect_basics(const Ram& ram,
}
}
static bool ends_with(const std::string& str, const std::string& suffix) {
return str.size() >= suffix.size() &&
0 == str.compare(str.size() - suffix.size(), suffix.size(), suffix);
}
void inspect_symbols(const Ram& ram,
const std::unordered_map<u32, std::string>& types,
const SymbolMap& symbols) {