Background improvements for custom levels (#3672)

This only applies to the background for now:

- support for alpha for vertex colors in custom levels
- switch time of day palette generation from octree to k-d tree
- support for alpha masking in custom levels
- support for transparent textures
- support for envmap in custom levels

---------

Co-authored-by: water111 <awaterford1111445@gmail.com>

common/custom_data/TFrag3Data.cpp

@@ -304,13 +304,15 @@ void TieTree::unpack() {
 
   for (auto& draw : static_draws) {
     draw.unpacked.idx_of_first_idx_in_full_buffer = unpacked.indices.size();
-    ASSERT(draw.plain_indices.empty());
+    // indices can come from either runs or already in plain indices.
     for (auto& run : draw.runs) {
       for (u32 ri = 0; ri < run.length; ri++) {
         unpacked.indices.push_back(run.vertex0 + ri);
       }
       unpacked.indices.push_back(UINT32_MAX);
     }
+    unpacked.indices.insert(unpacked.indices.end(), draw.plain_indices.begin(),
+                            draw.plain_indices.end());
   }
 }
 
@@ -406,6 +408,8 @@ void TieTree::serialize(Serializer& ser) {
   colors.serialize(ser);
   bvh.serialize(ser);
 
+  ser.from_ptr(&use_strips);
+
   ser.from_ptr(&has_per_proto_visibility_toggle);
   ser.from_string_vector(&proto_names);
 }
@@ -850,4 +854,11 @@ std::size_t PreloadedVertex::hash::operator()(const PreloadedVertex& v) const {
          std::hash<float>()(v.s) ^ std::hash<float>()(v.t) ^ std::hash<u16>()(v.color_index);
 }
 
+std::size_t PackedTieVertices::Vertex::hash::operator()(const Vertex& v) const {
+  return std::hash<float>()(v.x) ^ std::hash<float>()(v.y) ^ std::hash<float>()(v.z) ^
+         std::hash<float>()(v.r) ^ std::hash<float>()(v.g) ^ std::hash<float>()(v.b) ^
+         std::hash<float>()(v.a) ^ std::hash<float>()(v.s) ^ std::hash<float>()(v.t) ^
+         std::hash<float>()(v.nx) ^ std::hash<float>()(v.ny) ^ std::hash<float>()(v.nz);
+}
+
 }  // namespace tfrag3

common/custom_data/Tfrag3Data.h

@@ -18,7 +18,7 @@ namespace tfrag3 {
 // - if changing any large things (vertices, vis, bvh, colors, textures) update get_memory_usage
 // - if adding a new category to the memory usage, update extract_level to print it.
 
-constexpr int TFRAG3_VERSION = 41;
+constexpr int TFRAG3_VERSION = 42;
 
 enum MemoryUsageCategory {
   TEXTURE,
@@ -116,6 +116,16 @@ struct PackedTieVertices {
     float s, t;
     s8 nx, ny, nz;
     u8 r, g, b, a;
+
+    struct hash {
+      std::size_t operator()(const Vertex& x) const;
+    };
+
+    bool operator==(const Vertex& other) const {
+      return x == other.x && y == other.y && z == other.z && s == other.s && t == other.t &&
+             nx == other.nx && ny == other.ny && nz == other.nz && r == other.r && g == other.g &&
+             b == other.b && a == other.a;
+    }
   };
 
   struct MatrixGroup {
@@ -420,6 +430,8 @@ struct TieTree {
   bool has_per_proto_visibility_toggle = false;
   std::vector<std::string> proto_names;
 
+  bool use_strips = true;
+
   struct {
     std::vector<PreloadedVertex> vertices;  // mesh vertices
     std::vector<u32> indices;

common/util/gltf_util.cpp

@@ -387,7 +387,7 @@ int texture_pool_debug_checker(TexturePool* pool) {
   }
 }
 
-int texture_pool_add_texture(TexturePool* pool, const tinygltf::Image& tex) {
+int texture_pool_add_texture(TexturePool* pool, const tinygltf::Image& tex, int alpha_shift) {
   const auto& existing = pool->textures_by_name.find(tex.name);
   if (existing != pool->textures_by_name.end()) {
     lg::info("Reusing image: {}", tex.name);
@@ -412,6 +412,14 @@ int texture_pool_add_texture(TexturePool* pool, const tinygltf::Image& tex) {
   tt.data.resize(tt.w * tt.h);
   ASSERT(tex.image.size() >= tt.data.size());
   memcpy(tt.data.data(), tex.image.data(), tt.data.size() * 4);
+
+  // adjust alpha colors for PS2/PC difference
+  for (auto& color : tt.data) {
+    u32 alpha = color >> 24;
+    alpha >>= alpha_shift;
+    color &= 0xff'ff'ff;
+    color |= (alpha << 24);
+  }
   return idx;
 }
 
@@ -549,22 +557,38 @@ void dedup_vertices(const std::vector<tfrag3::PreloadedVertex>& vertices_in,
   }
 }
 
-DrawMode draw_mode_from_sampler(const tinygltf::Sampler& sampler) {
+void setup_alpha_from_material(const tinygltf::Material& material, DrawMode* mode) {
-  DrawMode mode = make_default_draw_mode();
+  if (material.alphaMode == "OPAQUE") {
+    mode->disable_ab();
+  } else if (material.alphaMode == "MASK") {
+    mode->enable_at();
+    mode->set_alpha_test(DrawMode::AlphaTest::GEQUAL);
+    mode->set_alpha_fail(GsTest::AlphaFail::KEEP);
+    mode->set_aref(material.alphaCutoff * 127);
+  } else if (material.alphaMode == "BLEND") {
+    mode->enable_ab();
+    mode->set_alpha_blend(DrawMode::AlphaBlend::SRC_DST_SRC_DST);
+    mode->set_depth_write_enable(false);
+  } else {
+    lg::die("Unknown GLTF alphaMode {}", material.alphaMode);
+  }
+}
+
+void setup_draw_mode_from_sampler(const tinygltf::Sampler& sampler, DrawMode* mode) {
   if (sampler.magFilter == TINYGLTF_TEXTURE_FILTER_NEAREST) {
     ASSERT(sampler.minFilter == TINYGLTF_TEXTURE_FILTER_NEAREST);
-    mode.set_filt_enable(false);
+    mode->set_filt_enable(false);
   } else {
     ASSERT(sampler.minFilter != TINYGLTF_TEXTURE_FILTER_NEAREST);
-    mode.set_filt_enable(true);
+    mode->set_filt_enable(true);
   }
 
   switch (sampler.wrapS) {
     case TINYGLTF_TEXTURE_WRAP_CLAMP_TO_EDGE:
-      mode.set_clamp_s_enable(true);
+      mode->set_clamp_s_enable(true);
       break;
     case TINYGLTF_TEXTURE_WRAP_REPEAT:
-      mode.set_clamp_s_enable(false);
+      mode->set_clamp_s_enable(false);
       break;
     default:
       ASSERT(false);
@@ -572,16 +596,14 @@ DrawMode draw_mode_from_sampler(const tinygltf::Sampler& sampler) {
 
   switch (sampler.wrapT) {
     case TINYGLTF_TEXTURE_WRAP_CLAMP_TO_EDGE:
-      mode.set_clamp_t_enable(true);
+      mode->set_clamp_t_enable(true);
       break;
     case TINYGLTF_TEXTURE_WRAP_REPEAT:
-      mode.set_clamp_t_enable(false);
+      mode->set_clamp_t_enable(false);
       break;
     default:
       ASSERT(false);
   }
-
-  return mode;
 }
 
 std::optional<int> find_single_skin(const tinygltf::Model& model,
@@ -622,4 +644,22 @@ std::vector<float> extract_floats(const tinygltf::Model& model, int accessor_idx
   return result;
 }
 
+std::size_t TieFullVertex::hash::operator()(const TieFullVertex& x) const {
+  return tfrag3::PackedTieVertices::Vertex::hash()(x.vertex) ^ std::hash<u16>()(x.color_index);
+}
+
+tfrag3::PackedTimeOfDay pack_time_of_day(const std::vector<math::Vector<u8, 4>>& color_palette) {
+  tfrag3::PackedTimeOfDay colors;
+  colors.color_count = (color_palette.size() + 3) & (~3);
+  colors.data.resize(colors.color_count * 8 * 4);
+  for (u32 color_index = 0; color_index < color_palette.size(); color_index++) {
+    for (u32 palette = 0; palette < 8; palette++) {
+      for (u32 channel = 0; channel < 4; channel++) {
+        colors.read(color_index, palette, channel) = color_palette[color_index][channel];
+      }
+    }
+  }
+  return colors;
+}
+
 }  // namespace gltf_util

common/util/gltf_util.h

@@ -61,9 +61,10 @@ DrawMode make_default_draw_mode();
 struct TexturePool {
   std::unordered_map<std::string, int> textures_by_name;
   std::vector<tfrag3::Texture> textures_by_idx;
+  std::map<std::pair<int, int>, int> envmap_textures_by_gltf_id;
 };
 
-int texture_pool_add_texture(TexturePool* pool, const tinygltf::Image& tex);
+int texture_pool_add_texture(TexturePool* pool, const tinygltf::Image& tex, int alpha_shift = 1);
 int texture_pool_debug_checker(TexturePool* pool);
 
 struct NodeWithTransform {
@@ -71,13 +72,47 @@ struct NodeWithTransform {
   math::Matrix4f w_T_node;
 };
 
-void dedup_vertices(const std::vector<tfrag3::PreloadedVertex>& vertices_in,
+struct TieFullVertex {
-                    std::vector<tfrag3::PreloadedVertex>& vertices_out,
+  tfrag3::PackedTieVertices::Vertex vertex;
-                    std::vector<u32>& old_to_new_out);
+  u16 color_index = 0;
+  struct hash {
+    std::size_t operator()(const TieFullVertex& x) const;
+  };
+
+  bool operator==(const TieFullVertex& other) const {
+    return vertex == other.vertex && color_index == other.color_index;
+  }
+};
+
+template <typename T>
+void dedup_vertices(const std::vector<T>& vertices_in,
+                    std::vector<T>& vertices_out,
+                    std::vector<u32>& old_to_new_out) {
+  ASSERT(vertices_out.empty());
+  ASSERT(old_to_new_out.empty());
+  old_to_new_out.resize(vertices_in.size(), -1);
+
+  std::unordered_map<T, u32, typename T::hash> vtx_to_new;
+
+  for (size_t in_idx = 0; in_idx < vertices_in.size(); in_idx++) {
+    auto& vtx = vertices_in[in_idx];
+    const auto& lookup = vtx_to_new.find(vtx);
+    if (lookup == vtx_to_new.end()) {
+      // first time seeing this one
+      size_t new_idx = vertices_out.size();
+      vertices_out.push_back(vtx);
+      old_to_new_out[in_idx] = new_idx;
+      vtx_to_new[vtx] = new_idx;
+    } else {
+      old_to_new_out[in_idx] = lookup->second;
+    }
+  }
+}
 
 std::vector<NodeWithTransform> flatten_nodes_from_all_scenes(const tinygltf::Model& model);
 
-DrawMode draw_mode_from_sampler(const tinygltf::Sampler& sampler);
+void setup_alpha_from_material(const tinygltf::Material& material, DrawMode* mode);
+void setup_draw_mode_from_sampler(const tinygltf::Sampler& sampler, DrawMode* mode);
 
 /*!
  * Find the index of the skin for this model. Returns nullopt if there is no skin, the index of the
@@ -127,4 +162,7 @@ std::vector<float> extract_floats(const tinygltf::Model& model, int accessor_idx
 math::Matrix4f matrix_from_trs(const math::Vector3f& trans,
                                const math::Vector4f& quat,
                                const math::Vector3f& scale);
+
+tfrag3::PackedTimeOfDay pack_time_of_day(const std::vector<math::Vector<u8, 4>>& color_palette);
+
 }  // namespace gltf_util

decompiler/level_extractor/merc_replacement.cpp

@@ -63,14 +63,14 @@ void extract(const std::string& name,
     draw.mode = make_default_draw_mode();
 
     if (mat_idx == -1) {
-      lg::warn("Draw had a material index of -1, using default texture.");
+      lg::warn("Draw had a material index of -1, using bogus texture.");
       draw.tree_tex_id = 0;
       continue;
     }
     const auto& mat = model.materials[mat_idx];
     int tex_idx = mat.pbrMetallicRoughness.baseColorTexture.index;
     if (tex_idx == -1) {
-      lg::warn("Material {} has no texture, using default texture.", mat.name);
+      lg::warn("Material {} has no texture, using bogus texture.", mat.name);
       draw.tree_tex_id = 0;
       continue;
     }
@@ -78,7 +78,8 @@ void extract(const std::string& name,
     const auto& tex = model.textures[tex_idx];
     ASSERT(tex.sampler >= 0);
     ASSERT(tex.source >= 0);
-    draw.mode = draw_mode_from_sampler(model.samplers.at(tex.sampler));
+    setup_alpha_from_material(mat, &draw.mode);
+    setup_draw_mode_from_sampler(model.samplers.at(tex.sampler), &draw.mode);
 
     const auto& img = model.images[tex.source];
     draw.tree_tex_id = tex_offset + texture_pool_add_texture(&out.tex_pool, img);

game/graphics/opengl_renderer/SkyRenderer.cpp

@@ -122,6 +122,7 @@ void SkyBlendHandler::draw_debug_window() {
 
   if (ImGui::TreeNode("tfrag")) {
     m_tfrag_renderer.draw_debug_window();
+    ImGui::Checkbox("enable", &m_tfrag_renderer.enabled());
     ImGui::TreePop();
   }
 }

game/graphics/opengl_renderer/background/Tie3.cpp

@@ -108,6 +108,7 @@ void Tie3::load_from_fr3_data(const LevelData* loader_data) {
       // OpenGL index buffer (fixed index buffer for multidraw system)
       lod_tree[l_tree].index_buffer = loader_data->tie_data[l_geo][l_tree].index_buffer;
       lod_tree[l_tree].category_draw_indices = tree.category_draw_indices;
+      lod_tree[l_tree].draw_mode = tree.use_strips ? GL_TRIANGLE_STRIP : GL_TRIANGLES;
 
       // set up vertex attributes
       glBindBuffer(GL_ARRAY_BUFFER, lod_tree[l_tree].vertex_buffer);
@@ -586,11 +587,11 @@ void Tie3::draw_matching_draws_for_tree(int idx,
     prof.add_draw_call();
 
     if (render_state->no_multidraw) {
-      glDrawElements(GL_TRIANGLE_STRIP, singledraw_indices.second, GL_UNSIGNED_INT,
+      glDrawElements(tree.draw_mode, singledraw_indices.second, GL_UNSIGNED_INT,
                      (void*)(singledraw_indices.first * sizeof(u32)));
     } else {
       glMultiDrawElements(
-          GL_TRIANGLE_STRIP, &tree.multidraw_count_buffer[multidraw_indices.first], GL_UNSIGNED_INT,
+          tree.draw_mode, &tree.multidraw_count_buffer[multidraw_indices.first], GL_UNSIGNED_INT,
           &tree.multidraw_index_offset_buffer[multidraw_indices.first], multidraw_indices.second);
     }
 
@@ -606,13 +607,13 @@ void Tie3::draw_matching_draws_for_tree(int idx,
                     double_draw.aref_second);
         glDepthMask(GL_FALSE);
         if (render_state->no_multidraw) {
-          glDrawElements(GL_TRIANGLE_STRIP, singledraw_indices.second, GL_UNSIGNED_INT,
+          glDrawElements(tree.draw_mode, singledraw_indices.second, GL_UNSIGNED_INT,
                          (void*)(singledraw_indices.first * sizeof(u32)));
         } else {
-          glMultiDrawElements(
+          glMultiDrawElements(tree.draw_mode, &tree.multidraw_count_buffer[multidraw_indices.first],
-              GL_TRIANGLE_STRIP, &tree.multidraw_count_buffer[multidraw_indices.first],
+                              GL_UNSIGNED_INT,
-              GL_UNSIGNED_INT, &tree.multidraw_index_offset_buffer[multidraw_indices.first],
+                              &tree.multidraw_index_offset_buffer[multidraw_indices.first],
-              multidraw_indices.second);
+                              multidraw_indices.second);
         }
         break;
       default:
@@ -674,11 +675,11 @@ void Tie3::envmap_second_pass_draw(const Tree& tree,
     prof.add_draw_call();
 
     if (render_state->no_multidraw) {
-      glDrawElements(GL_TRIANGLE_STRIP, singledraw_indices.second, GL_UNSIGNED_INT,
+      glDrawElements(tree.draw_mode, singledraw_indices.second, GL_UNSIGNED_INT,
                      (void*)(singledraw_indices.first * sizeof(u32)));
     } else {
       glMultiDrawElements(
-          GL_TRIANGLE_STRIP, &tree.multidraw_count_buffer[multidraw_indices.first], GL_UNSIGNED_INT,
+          tree.draw_mode, &tree.multidraw_count_buffer[multidraw_indices.first], GL_UNSIGNED_INT,
           &tree.multidraw_index_offset_buffer[multidraw_indices.first], multidraw_indices.second);
     }
 
@@ -941,7 +942,7 @@ void Tie3::render_tree_wind(int idx,
       prof.add_draw_call();
       prof.add_tri(grp.num);
 
-      glDrawElements(GL_TRIANGLE_STRIP, grp.num, GL_UNSIGNED_INT,
+      glDrawElements(tree.draw_mode, grp.num, GL_UNSIGNED_INT,
                      (void*)((off + tree.wind_vertex_index_offsets.at(draw_idx)) * sizeof(u32)));
       off += grp.num;
 
@@ -958,7 +959,7 @@ void Tie3::render_tree_wind(int idx,
               glGetUniformLocation(render_state->shaders[ShaderId::TFRAG3].id(), "alpha_max"),
               double_draw.aref_second);
           glDepthMask(GL_FALSE);
-          glDrawElements(GL_TRIANGLE_STRIP, draw.vertex_index_stream.size(), GL_UNSIGNED_INT,
+          glDrawElements(tree.draw_mode, draw.vertex_index_stream.size(), GL_UNSIGNED_INT,
                          (void*)0);
           break;
         default:

game/graphics/opengl_renderer/background/Tie3.h

@@ -130,6 +130,7 @@ class Tie3 : public BucketRenderer {
     std::vector<std::pair<int, int>> multidraw_offset_per_stripdraw;
     std::vector<GLsizei> multidraw_count_buffer;
     std::vector<void*> multidraw_index_offset_buffer;
+    u64 draw_mode = 0;  // strip or not, GL enum
   };
 
   void envmap_second_pass_draw(const Tree& tree,

game/graphics/opengl_renderer/loader/LoaderStages.cpp

@@ -344,6 +344,10 @@ class TieLoadStage : public LoaderStage {
           if (m_next_tree >= data.lev_data->level->tie_trees[m_next_geo].size()) {
             m_next_tree = 0;
             m_next_geo++;
+            while (data.lev_data->level->tie_trees[m_next_geo].empty() &&
+                   m_next_geo < tfrag3::TIE_GEOS) {
+              m_next_geo++;
+            }
             if (m_next_geo >= tfrag3::TIE_GEOS) {
               m_verts_done = true;
               m_next_tree = 0;
@@ -448,6 +452,10 @@ class TieLoadStage : public LoaderStage {
           if (m_next_tree >= data.lev_data->level->tie_trees[m_next_geo].size()) {
             m_next_tree = 0;
             m_next_geo++;
+            while (data.lev_data->level->tie_trees[m_next_geo].empty() &&
+                   m_next_geo < tfrag3::TIE_GEOS) {
+              m_next_geo++;
+            }
             if (m_next_geo >= tfrag3::TIE_GEOS) {
               m_indices_done = true;
               m_next_tree = 0;

goal_src/jak1/engine/gfx/tie/tie-methods.gc

@@ -299,12 +299,12 @@
             (set! (-> a1-7 generic-next-clear) (the-as uint128 0)))
           0)
         (let* ((s1-0 (-> (the-as drawable-inline-array-instance-tie v1-16) data)) ;; the inline array of instances
-               (s0-0 (&-> (scratchpad-object terrain-context) work background vis-list (/ (-> s1-0 0 id) 8))) ;; vis for first.
+               ;; (s0-0 (&-> (scratchpad-object terrain-context) work background vis-list (if (zero? (-> arg0 length)) 0 (/ (-> s1-0 0 id) 8)))) ;; vis for first.
                (s3-1 (-> *display* frames (-> *display* on-screen) frame global-buf)) ;; dma buf to write to
                )
-          (set! sv-16 (-> (the-as drawable-inline-array-node v1-16) length)) ;; number of instances
+          ;; (set! sv-16 (-> (the-as drawable-inline-array-node v1-16) length)) ;; number of instances
           ;; if we actually have things to draw
-          (when (nonzero? sv-16)
+          (when #t ;; (nonzero? sv-16)
             ;; this is some buffer for the generic renderer
             (let* ((v1-21 (logand (the-as int *gsf-buffer*) 8191))
                    (v1-23 (logand (the-as int (&- (logand (the-as int (&-> (-> s4-1 data) -512)) 8191) (the-as uint v1-21))) 8191)))
@@ -325,9 +325,9 @@
               ;; note: this is a bit wasteful because we only care about generic ties.
               ;; non-generics are drawn fully in C++, but we're computing unused stuff here.
               ;; This ends up being so fast it's probably not worth worrying about yet.
-              (when (not *use-etie*)
+              ;; (when (not *use-etie*)
-                (with-profiler "tie-instance"
+              ;;  (with-profiler "tie-instance"
-                  (draw-inline-array-instance-tie s0-0 s1-0 sv-16 s3-1)))
+              ;;    (draw-inline-array-instance-tie s0-0 s1-0 sv-16 s3-1)))
               ;; finish perf stats
               (read! (-> *perf-stats* data 9))
               (update-wait-stats (-> *perf-stats* data 9)
@@ -558,34 +558,47 @@
 ;;;;;;;;;;;;;;;;;
 
 ;; note: the first three methods appear twice in the original code.
+;; modified for PC: check length before colliding.
 
 (defmethod collide-with-box ((this drawable-tree-instance-tie) (arg0 int) (arg1 collide-list))
-  (collide-with-box (-> this data 0) (-> this length) arg1)
+  (when (nonzero? (-> this length))
+    (collide-with-box (-> this data 0) (-> this length) arg1)
+    )
   0
   (none))
 
 (defmethod collide-y-probe ((this drawable-tree-instance-tie) (arg0 int) (arg1 collide-list))
-  (collide-y-probe (-> this data 0) (-> this length) arg1)
+  (when (nonzero? (-> this length))
+    (collide-y-probe (-> this data 0) (-> this length) arg1)
+    )
   0
   (none))
 
 (defmethod collide-ray ((this drawable-tree-instance-tie) (arg0 int) (arg1 collide-list))
-  (collide-ray (-> this data 0) (-> this length) arg1)
+  (when (nonzero? (-> this length))
+    (collide-ray (-> this data 0) (-> this length) arg1)
+    )
   0
   (none))
 
 (defmethod collide-with-box ((this drawable-inline-array-instance-tie) (arg0 int) (arg1 collide-list))
-  (collide-with-box (the-as instance-tie (-> this data)) (-> this length) arg1)
+  (when (nonzero? (-> this length))
+    (collide-with-box (the-as instance-tie (-> this data)) (-> this length) arg1)
+    )
   0
   (none))
 
 (defmethod collide-y-probe ((this drawable-inline-array-instance-tie) (arg0 int) (arg1 collide-list))
-  (collide-y-probe (the-as instance-tie (-> this data)) (-> this length) arg1)
+  (when (nonzero? (-> this length))
+    (collide-y-probe (the-as instance-tie (-> this data)) (-> this length) arg1)
+    )
   0
   (none))
 
 (defmethod collide-ray ((this drawable-inline-array-instance-tie) (arg0 int) (arg1 collide-list))
-  (collide-ray (the-as instance-tie (-> this data)) (-> this length) arg1)
+  (when (nonzero? (-> this length))
+    (collide-ray (the-as instance-tie (-> this data)) (-> this length) arg1)
+    )
   0
   (none))
 

goalc/CMakeLists.txt

@@ -31,6 +31,7 @@ add_library(compiler
         build_level/jak3/LevelFile.cpp
         build_level/common/ResLump.cpp
         build_level/common/Tfrag.cpp
+        build_level/common/Tie.cpp
         build_level/jak1/ambient.cpp
         compiler/Compiler.cpp
         compiler/Env.cpp

goalc/build_actor/common/MercExtract.cpp

@@ -108,7 +108,8 @@ void extract(const std::string& name,
     const auto& tex = model.textures[tex_idx];
     ASSERT(tex.sampler >= 0);
     ASSERT(tex.source >= 0);
-    draw.mode = gltf_util::draw_mode_from_sampler(model.samplers.at(tex.sampler));
+    gltf_util::setup_draw_mode_from_sampler(model.samplers.at(tex.sampler), &draw.mode);
+    gltf_util::setup_alpha_from_material(mat, &draw.mode);
 
     const auto& img = model.images[tex.source];
     draw.tree_tex_id = tex_offset + texture_pool_add_texture(&out.tex_pool, img);

goalc/build_actor/common/animation_processing.cpp

@@ -276,6 +276,9 @@ CompressedAnim compress_animation(const UncompressedJointAnim& in) {
       compress_scale(&out.fixed, joint_data.scale_frames[0]);
     }
   }
+
+  lg::info("animation {} size {:.2f} kB", in.name,
+           (out.fixed.size_bytes() + out.frames.size() * out.frames.at(0).size_bytes()) / 1024.f);
   return out;
 }
 }  // namespace anim

goalc/build_level/collide/jak1/collide_bvh.cpp

@@ -109,7 +109,7 @@ void split_along_dim(std::vector<jak1::CollideFace>& faces,
             [=](const jak1::CollideFace& a, const jak1::CollideFace& b) {
               return a.bsphere[dim] < b.bsphere[dim];
             });
-  lg::print("splitting with size: {}\n", faces.size());
+  // lg::print("splitting with size: {}\n", faces.size());
   size_t split_idx = faces.size() / 2;
   out0->insert(out0->end(), faces.begin(), faces.begin() + split_idx);
   out1->insert(out1->end(), faces.begin() + split_idx, faces.end());

goalc/build_level/common/Tfrag.cpp

@@ -5,26 +5,33 @@
 #include "gltf_mesh_extract.h"
 
 #include "common/custom_data/pack_helpers.h"
+#include "common/util/gltf_util.h"
 
 #include "goalc/data_compiler/DataObjectGenerator.h"
 
+void add_tree(std::vector<tfrag3::TfragTree>& out_pc,
+              const gltf_mesh_extract::TfragOutput& mesh_extract_out,
+              const std::vector<tfrag3::StripDraw>& draws,
+              tfrag3::TFragmentTreeKind kind) {
+  auto& normal = out_pc.emplace_back();
+  normal.kind = kind;
+  normal.draws = draws;
+  pack_tfrag_vertices(&normal.packed_vertices, mesh_extract_out.tfrag_vertices);
+  normal.colors = gltf_util::pack_time_of_day(mesh_extract_out.color_palette);
+  normal.use_strips = false;
+}
+
 void tfrag_from_gltf(const gltf_mesh_extract::TfragOutput& mesh_extract_out,
-                     DrawableTreeTfrag& /*out*/,
+                     std::vector<tfrag3::TfragTree>& out_pc) {
-                     tfrag3::TfragTree& out_pc) {
+  if (!mesh_extract_out.normal_strip_draws.empty()) {
-  out_pc.kind = tfrag3::TFragmentTreeKind::NORMAL;  // todo more types?
+    add_tree(out_pc, mesh_extract_out, mesh_extract_out.normal_strip_draws,
-  out_pc.draws = std::move(mesh_extract_out.strip_draws);
+             tfrag3::TFragmentTreeKind::NORMAL);
-  pack_tfrag_vertices(&out_pc.packed_vertices, mesh_extract_out.vertices);
+  }
-  out_pc.colors.color_count = (mesh_extract_out.color_palette.size() + 3) & (~3);
+
-  out_pc.colors.data.resize(out_pc.colors.color_count * 8 * 4);
+  if (!mesh_extract_out.trans_strip_draws.empty()) {
-  for (u32 color_index = 0; color_index < mesh_extract_out.color_palette.size(); color_index++) {
+    add_tree(out_pc, mesh_extract_out, mesh_extract_out.trans_strip_draws,
-    for (u32 palette = 0; palette < 8; palette++) {
+             tfrag3::TFragmentTreeKind::TRANS);
-      for (u32 channel = 0; channel < 4; channel++) {
-        out_pc.colors.read(color_index, palette, channel) =
-            mesh_extract_out.color_palette[color_index][channel];
-      }
-    }
   }
-  out_pc.use_strips = false;
 }
 
 /*
@@ -88,7 +95,7 @@ size_t add_empty_dia(const std::string& name, DataObjectGenerator& gen, int tota
 
 size_t DrawableTreeTfrag::add_to_object_file(DataObjectGenerator& gen) const {
   gen.align_to_basic();
-  gen.add_type_tag("drawable-tree-tfrag");
+  gen.add_type_tag(m_type);
   size_t result = gen.current_offset_bytes();
   gen.add_word(1 << 16);
   for (int i = 0; i < 6; i++) {
@@ -96,7 +103,7 @@ size_t DrawableTreeTfrag::add_to_object_file(DataObjectGenerator& gen) const {
   }
   size_t slot = gen.add_word(0);
   ASSERT(slot * 4 - result == 28);
-  gen.link_word_to_byte(slot, add_empty_dia("drawable-inline-array-tfrag", gen, 0x64));
+  gen.link_word_to_byte(slot, add_empty_dia(m_array_type, gen, 0x64));
 
   return result;
 }

goalc/build_level/common/Tfrag.h

@@ -8,10 +8,18 @@
 
 class DataObjectGenerator;
 
-struct DrawableTreeTfrag {
+class DrawableTreeTfrag {
+  // "drawable-tree-tfrag"
+  // "drawable-inline-array-tfrag"
+ public:
+  DrawableTreeTfrag(const std::string& type, const std::string& array_type)
+      : m_type(type), m_array_type(array_type) {}
   size_t add_to_object_file(DataObjectGenerator& gen) const;
+
+ private:
+  std::string m_type;
+  std::string m_array_type;
 };
 
 void tfrag_from_gltf(const gltf_mesh_extract::TfragOutput& mesh_extract_out,
-                     DrawableTreeTfrag& out,
+                     std::vector<tfrag3::TfragTree>& out_pc);
-                     tfrag3::TfragTree& out_pc);

goalc/build_level/common/Tie.cpp

@@ -0,0 +1,129 @@
+#include "Tie.h"
+
+void tie_from_gltf(const gltf_mesh_extract::TieOutput& mesh_extract_out,
+                   std::vector<tfrag3::TieTree>& out_pc) {
+  auto& out = out_pc.emplace_back();
+  // bvh: leave default
+  // draws
+  // categories
+  out.category_draw_indices[0] = 0;
+  for (int category_idx = 0; category_idx < tfrag3::kNumTieCategories; category_idx++) {
+    switch ((tfrag3::TieCategory)category_idx) {
+      case tfrag3::TieCategory::NORMAL_ENVMAP:
+        out.static_draws.insert(out.static_draws.end(), mesh_extract_out.base_draws.begin(),
+                                mesh_extract_out.base_draws.end());
+        break;
+      case tfrag3::TieCategory::NORMAL_ENVMAP_SECOND_DRAW:
+        out.static_draws.insert(out.static_draws.end(), mesh_extract_out.envmap_draws.begin(),
+                                mesh_extract_out.envmap_draws.end());
+        break;
+      default:
+        break;
+    }
+    out.category_draw_indices[category_idx + 1] = out.static_draws.size();
+  }
+  // packed
+  out.packed_vertices.color_indices = mesh_extract_out.color_indices;
+  out.packed_vertices.vertices = mesh_extract_out.vertices;
+  auto& matrix_group = out.packed_vertices.matrix_groups.emplace_back();
+  matrix_group.matrix_idx = -1;
+  matrix_group.start_vert = 0;
+  matrix_group.end_vert = out.packed_vertices.vertices.size();
+  matrix_group.has_normals = true;
+
+  // colors
+  out.colors = gltf_util::pack_time_of_day(mesh_extract_out.color_palette);
+  // wind (none)
+  // proto vis toggle
+  out.has_per_proto_visibility_toggle = false;
+
+  out.use_strips = false;
+}
+
+/*
+
+(deftype drawable (basic)
+  ((id      int16          :offset-assert 4)
+   (bsphere vector :inline :offset-assert 16)
+   )
+
+(deftype drawable-tree (drawable-group)
+  ()
+  :flag-assert #x1200000024
+  )
+
+(deftype drawable-group (drawable)
+  ((length  int16       :offset 6)
+   (data    drawable 1  :offset-assert 32)
+   )
+  (:methods
+    (new (symbol type int) _type_)
+    )
+  :flag-assert #x1200000024
+  )
+
+(deftype drawable-tree-instance-tie (drawable-tree)
+  ((prototypes proxy-prototype-array-tie           :offset 8)
+   )
+  :method-count-assert 18
+  :size-assert         #x24
+  :flag-assert         #x1200000024
+  )
+
+(deftype proxy-prototype-array-tie (basic)
+  ((prototype-array-tie prototype-array-tie   :offset-assert 4)
+   (wind-vectors        uint32  :offset-assert 8) ; likely a pointer
+   )
+  :method-count-assert 9
+  :size-assert         #xc
+  :flag-assert         #x90000000c
+  )
+
+(deftype prototype-array-tie (array)
+  ((array-data prototype-bucket-tie :dynamic :offset 16)
+   )
+  :method-count-assert 10
+  :size-assert         #x10
+  :flag-assert         #xa00000010
+  (:methods
+    (login (_type_) none) ;; 9
+    )
+  )
+*/
+
+size_t add_prototype_array_tie(DataObjectGenerator& gen) {
+  gen.align_to_basic();
+  gen.add_type_tag("prototype-array-tie");  // 0
+  size_t ret = gen.current_offset_bytes();
+  gen.add_word(0);                           // 4 length
+  gen.add_word(0);                           // 8 allocated-length
+  gen.add_type_tag("prototype-bucket-tie");  // 12 content type (might be wrong?)
+  return ret;
+}
+
+size_t add_proxy_prototype_array_tie(DataObjectGenerator& gen) {
+  const size_t array_offset = add_prototype_array_tie(gen);
+
+  gen.align_to_basic();
+  gen.add_type_tag("proxy-prototype-array-tie");  // 0
+  const size_t result = gen.current_offset_bytes();
+  const size_t array_slot = gen.add_word(0);  // 4 prototype-array-tie
+  gen.link_word_to_byte(array_slot, array_offset);
+  gen.add_word(0);  // 8 wind-vectors
+  return result;
+}
+
+size_t add_tie_tree_to_object_file(DataObjectGenerator& gen) {
+  const size_t proxy = add_proxy_prototype_array_tie(gen);
+  gen.align_to_basic();
+  gen.add_type_tag("drawable-tree-instance-tie");  // 0
+  size_t result = gen.current_offset_bytes();
+  gen.add_word(0);                      // 4 (id = 0, length = 0)
+  const size_t slot = gen.add_word(0);  // 8
+  gen.link_word_to_byte(slot, proxy);
+  return result;
+}
+
+size_t DrawableTreeInstanceTie::add_to_object_file(DataObjectGenerator& gen) const {
+  return add_tie_tree_to_object_file(gen);
+}

goalc/build_level/common/Tie.h

@@ -0,0 +1,17 @@
+#pragma once
+
+#include "Tie.h"
+
+#include "common/custom_data/Tfrag3Data.h"
+#include "common/util/gltf_util.h"
+
+#include "goalc/build_level/common/gltf_mesh_extract.h"
+#include "goalc/data_compiler/DataObjectGenerator.h"
+
+void tie_from_gltf(const gltf_mesh_extract::TieOutput& mesh_extract_out,
+                   std::vector<tfrag3::TieTree>& out_pc);
+
+class DrawableTreeInstanceTie {
+ public:
+  size_t add_to_object_file(DataObjectGenerator& gen) const;
+};

goalc/build_level/common/color_quantization.cpp

@@ -1,10 +1,12 @@
 #include "color_quantization.h"
 
 #include <algorithm>
+#include <set>
 #include <unordered_map>
 
 #include "common/log/log.h"
 #include "common/util/Assert.h"
+#include "common/util/Timer.h"
 
 /*!
  * Just removes duplicate colors, which can work if there are only a few unique colors.
@@ -221,3 +223,123 @@ QuantizedColors quantize_colors_octree(const std::vector<math::Vector<u8, 4>>& i
 
   return out;
 }
+
+struct KdNode {
+  // if not a leaf
+  std::unique_ptr<KdNode> left, right;
+
+  // if leaf
+  std::vector<Color> colors;
+};
+
+void split_kd(KdNode* in, u32 depth, int next_split_dim) {
+  if (!depth) {
+    return;
+  }
+
+  // sort by split dimension
+  std::stable_sort(in->colors.begin(), in->colors.end(), [=](const Color& a, const Color& b) {
+    return a[next_split_dim] < b[next_split_dim];
+  });
+
+  in->left = std::make_unique<KdNode>();
+  in->right = std::make_unique<KdNode>();
+
+  size_t i = 0;
+  size_t mid = in->colors.size() / 2;
+  if (depth & 1) {
+    while (mid > 1 && in->colors[mid] == in->colors[mid - 1]) {
+      mid--;
+    }
+  } else {
+    while (mid + 2 < in->colors.size() && in->colors[mid] == in->colors[mid + 1]) {
+      mid++;
+    }
+  }
+
+  for (; i < mid; i++) {
+    in->left->colors.push_back(in->colors[i]);
+  }
+
+  for (; i < in->colors.size(); i++) {
+    in->right->colors.push_back(in->colors[i]);
+  }
+
+  split_kd(in->left.get(), depth - 1, (next_split_dim + 1) % 4);
+  split_kd(in->right.get(), depth - 1, (next_split_dim + 1) % 4);
+}
+
+template <typename Func>
+void for_each_child(KdNode* node, Func&& f) {
+  if (node->left) {
+    for_each_child(node->left.get(), f);
+    for_each_child(node->right.get(), f);
+  } else {
+    f(node);
+  }
+}
+
+u32 color_as_u32(const Color& color) {
+  u32 ret = 0;
+  memcpy(&ret, color.data(), 4);
+  return ret;
+}
+
+Color u32_as_color(u32 in) {
+  Color ret;
+  memcpy(ret.data(), &in, 4);
+  return ret;
+}
+
+std::vector<Color> deduplicated_colors(const std::vector<Color>& in) {
+  std::set<u32> unique;
+  for (auto& x : in) {
+    unique.insert(color_as_u32(x));
+  }
+  std::vector<Color> out;
+  for (auto& x : unique) {
+    out.push_back(u32_as_color(x));
+  }
+  return out;
+}
+
+QuantizedColors quantize_colors_kd_tree(const std::vector<math::Vector<u8, 4>>& in,
+                                        u32 target_depth) {
+  Timer timer;
+  // Build root node:
+  KdNode root;
+  root.colors = deduplicated_colors(in);
+  // root.colors = in;
+
+  // Split tree:
+  split_kd(&root, target_depth, 0);
+
+  // Get final colors:
+  std::unordered_map<u32, u32> color_value_to_color_idx;
+  QuantizedColors result;
+  for_each_child(&root, [&](KdNode* node) {
+    if (node->colors.empty()) {
+      return;
+    }
+
+    const u32 slot = result.final_colors.size();
+    u32 totals[4] = {0, 0, 0, 0};
+    u32 n = node->colors.size();
+    for (auto& color : node->colors) {
+      color_value_to_color_idx[color_as_u32(color)] = slot;
+      for (int i = 0; i < 4; i++) {
+        totals[i] += color[i];
+      }
+    }
+    result.final_colors.emplace_back(totals[0] / n, totals[1] / n, totals[2] / n,
+                                     totals[3] / (2 * n));
+  });
+
+  for (auto& color : in) {
+    result.vtx_to_color.push_back(color_value_to_color_idx.at(color_as_u32(color)));
+  }
+
+  lg::warn("Quantize colors: {} input colors -> {} output in {:.3f} ms\n", in.size(),
+           result.final_colors.size(), timer.getMs());
+  return result;
+}

goalc/build_level/common/color_quantization.h

@@ -19,3 +19,6 @@ QuantizedColors quantize_colors_dumb(const std::vector<math::Vector<u8, 4>>& in)
 
 QuantizedColors quantize_colors_octree(const std::vector<math::Vector<u8, 4>>& in,
                                        u32 target_count);
+
+QuantizedColors quantize_colors_kd_tree(const std::vector<math::Vector<u8, 4>>& in,
+                                        u32 target_depth);

goalc/build_level/common/gltf_mesh_extract.cpp

@@ -16,19 +16,61 @@
 using namespace gltf_util;
 namespace gltf_mesh_extract {
 
-void dedup_vertices(TfragOutput& data) {
+void dedup_tfrag_vertices(TfragOutput& data) {
   Timer timer;
-  size_t original_size = data.vertices.size();
+  size_t original_size = data.tfrag_vertices.size();
   std::vector<tfrag3::PreloadedVertex> new_verts;
   std::vector<u32> old_to_new;
 
-  gltf_util::dedup_vertices(data.vertices, new_verts, old_to_new);
+  gltf_util::dedup_vertices(data.tfrag_vertices, new_verts, old_to_new);
-  data.vertices = std::move(new_verts);
+  data.tfrag_vertices = std::move(new_verts);
 
-  for (auto& draw : data.strip_draws) {
+  // TODO: properly split vertices between trees...
-    ASSERT(draw.runs.empty());  // not supported yet
+  for (auto drawlist : {&data.normal_strip_draws, &data.trans_strip_draws}) {
-    for (auto& idx : draw.plain_indices) {
+    for (auto& draw : *drawlist) {
-      idx = old_to_new.at(idx);
+      ASSERT(draw.runs.empty());  // not supported yet
+      for (auto& idx : draw.plain_indices) {
+        idx = old_to_new.at(idx);
+      }
+    }
+  }
+
+  lg::info("Deduplication took {:.2f} ms, {} -> {} ({:.2f} %)", timer.getMs(), original_size,
+           data.tfrag_vertices.size(), 100.f * data.tfrag_vertices.size() / original_size);
+}
+
+void dedup_tie_vertices(TieOutput& data) {
+  Timer timer;
+  size_t original_size = data.vertices.size();
+
+  std::vector<TieFullVertex> old_verts;
+  old_verts.reserve(data.vertices.size());
+  for (size_t i = 0; i < data.vertices.size(); i++) {
+    auto& x = old_verts.emplace_back();
+    x.color_index = data.color_indices[i];
+    x.vertex = data.vertices[i];
+  }
+
+  std::vector<TieFullVertex> new_verts;
+  std::vector<u32> old_to_new;
+
+  gltf_util::dedup_vertices(old_verts, new_verts, old_to_new);
+  data.vertices.clear();
+  data.color_indices.clear();
+  data.vertices.reserve(new_verts.size());
+  data.color_indices.reserve(new_verts.size());
+  for (auto& x : new_verts) {
+    data.vertices.push_back(x.vertex);
+    data.color_indices.push_back(x.color_index);
+  }
+
+  // TODO: properly split vertices between trees...
+  for (auto drawlist : {&data.base_draws, &data.envmap_draws}) {
+    for (auto& draw : *drawlist) {
+      ASSERT(draw.runs.empty());  // not supported yet
+      for (auto& idx : draw.plain_indices) {
+        idx = old_to_new.at(idx);
+      }
     }
   }
 
@@ -36,13 +78,43 @@ void dedup_vertices(TfragOutput& data) {
            data.vertices.size(), 100.f * data.vertices.size() / original_size);
 }
 
+bool prim_needs_tie(const tinygltf::Model& model, const tinygltf::Primitive& prim) {
+  if (prim.material >= 0) {
+    auto mat = model.materials.at(prim.material);
+    return mat.extensions.contains("KHR_materials_specular");
+  }
+  return false;
+}
+
+struct EnvmapSettings {
+  int texture_idx = -1;
+};
+
+EnvmapSettings envmap_settings_from_gltf(const tinygltf::Material& mat) {
+  EnvmapSettings settings;
+
+  ASSERT(mat.extensions.contains("KHR_materials_specular"));
+  const auto& specular_extension = mat.extensions.at("KHR_materials_specular");
+  ASSERT(specular_extension.Has("specularColorTexture"));
+
+  auto& texture = specular_extension.Get("specularColorTexture");
+  ASSERT(texture.Has("index"));
+  settings.texture_idx = texture.Get("index").Get<int>();
+  return settings;
+}
+
 void extract(const Input& in,
              TfragOutput& out,
              const tinygltf::Model& model,
              const std::vector<NodeWithTransform>& all_nodes) {
   std::vector<math::Vector<u8, 4>> all_vtx_colors;
-  ASSERT(out.vertices.empty());
+  ASSERT(out.tfrag_vertices.empty());
-  std::map<int, tfrag3::StripDraw> draw_by_material;
+
+  struct MaterialInfo {
+    tfrag3::StripDraw draw;
+    bool needs_tie = false;
+  };
+  std::map<int, MaterialInfo> info_by_material;
   int mesh_count = 0;
   int prim_count = 0;
 
@@ -59,85 +131,311 @@ void extract(const Input& in,
             model.materials[prim.material].extras.Get("set_invisible").Get<int>()) {
           continue;
         }
+
+        if (prim_needs_tie(model, prim)) {
+          continue;
+        }
         prim_count++;
         // extract index buffer
-        std::vector<u32> prim_indices = gltf_index_buffer(model, prim.indices, out.vertices.size());
+        std::vector<u32> prim_indices =
+            gltf_index_buffer(model, prim.indices, out.tfrag_vertices.size());
         ASSERT_MSG(prim.mode == TINYGLTF_MODE_TRIANGLES, "Unsupported triangle mode");
         // extract vertices
-        auto verts =
+        auto verts = gltf_vertices(model, prim.attributes, n.w_T_node, true, false, mesh.name);
-            gltf_vertices(model, prim.attributes, n.w_T_node, in.get_colors, false, mesh.name);
+        out.tfrag_vertices.insert(out.tfrag_vertices.end(), verts.vtx.begin(), verts.vtx.end());
-        out.vertices.insert(out.vertices.end(), verts.vtx.begin(), verts.vtx.end());
+        all_vtx_colors.insert(all_vtx_colors.end(), verts.vtx_colors.begin(),
-        if (in.get_colors) {
+                              verts.vtx_colors.end());
-          all_vtx_colors.insert(all_vtx_colors.end(), verts.vtx_colors.begin(),
+        ASSERT(all_vtx_colors.size() == out.tfrag_vertices.size());
-                                verts.vtx_colors.end());
-          ASSERT(all_vtx_colors.size() == out.vertices.size());
-        }
 
-        // TODO: just putting it all in one material
+        auto& info = info_by_material[prim.material];
-        auto& draw = draw_by_material[prim.material];
+        info.draw.mode = make_default_draw_mode();                        // todo rm
-        draw.mode = make_default_draw_mode();                        // todo rm
+        info.draw.tree_tex_id = texture_pool_debug_checker(in.tex_pool);  // todo rm
-        draw.tree_tex_id = texture_pool_debug_checker(in.tex_pool);  // todo rm
+        info.draw.num_triangles += prim_indices.size() / 3;
-        draw.num_triangles += prim_indices.size() / 3;
+        if (info.draw.vis_groups.empty()) {
-        if (draw.vis_groups.empty()) {
+          auto& grp = info.draw.vis_groups.emplace_back();
-          auto& grp = draw.vis_groups.emplace_back();
           grp.num_inds += prim_indices.size();
-          grp.num_tris += draw.num_triangles;
+          grp.num_tris += info.draw.num_triangles;
           grp.vis_idx_in_pc_bvh = UINT16_MAX;
         } else {
-          auto& grp = draw.vis_groups.back();
+          auto& grp = info.draw.vis_groups.back();
           grp.num_inds += prim_indices.size();
-          grp.num_tris += draw.num_triangles;
+          grp.num_tris += info.draw.num_triangles;
           grp.vis_idx_in_pc_bvh = UINT16_MAX;
         }
 
-        draw.plain_indices.insert(draw.plain_indices.end(), prim_indices.begin(),
+        info.draw.plain_indices.insert(info.draw.plain_indices.end(), prim_indices.begin(),
-                                  prim_indices.end());
+                                       prim_indices.end());
       }
     }
   }
 
-  for (const auto& [mat_idx, d_] : draw_by_material) {
+  for (const auto& [mat_idx, d_] : info_by_material) {
-    out.strip_draws.push_back(d_);
+    // out.strip_draws.push_back(d_);
-    auto& draw = out.strip_draws.back();
+    // auto& draw = out.strip_draws.back();
+    tfrag3::StripDraw draw = d_.draw;
     draw.mode = make_default_draw_mode();
 
     if (mat_idx == -1) {
       lg::warn("Draw had a material index of -1, using default texture.");
       draw.tree_tex_id = texture_pool_debug_checker(in.tex_pool);
+      out.normal_strip_draws.push_back(draw);
       continue;
     }
+
     const auto& mat = model.materials[mat_idx];
+    setup_alpha_from_material(mat, &draw.mode);
     int tex_idx = mat.pbrMetallicRoughness.baseColorTexture.index;
     if (tex_idx == -1) {
       lg::warn("Material {} has no texture, using default texture.", mat.name);
       draw.tree_tex_id = texture_pool_debug_checker(in.tex_pool);
+      if (draw.mode.get_ab_enable()) {
+        out.trans_strip_draws.push_back(draw);
+      } else {
+        out.normal_strip_draws.push_back(draw);
+      }
       continue;
     }
 
     const auto& tex = model.textures[tex_idx];
     ASSERT(tex.sampler >= 0);
     ASSERT(tex.source >= 0);
-    draw.mode = draw_mode_from_sampler(model.samplers.at(tex.sampler));
+    setup_draw_mode_from_sampler(model.samplers.at(tex.sampler), &draw.mode);
 
     const auto& img = model.images[tex.source];
     draw.tree_tex_id = texture_pool_add_texture(in.tex_pool, img);
+
+    if (draw.mode.get_ab_enable()) {
+      out.trans_strip_draws.push_back(draw);
+    } else {
+      out.normal_strip_draws.push_back(draw);
+    }
   }
-  lg::info("total of {} unique materials", out.strip_draws.size());
+  lg::info("total of {} normal, {} transparent unique materials", out.normal_strip_draws.size(),
+           out.trans_strip_draws.size());
+
+  lg::info("Merged {} meshes and {} prims into {} vertices", mesh_count, prim_count,
+           out.tfrag_vertices.size());
+
+  Timer quantize_timer;
+  auto quantized = quantize_colors_kd_tree(all_vtx_colors, 10);
+  for (size_t i = 0; i < out.tfrag_vertices.size(); i++) {
+    out.tfrag_vertices[i].color_index = quantized.vtx_to_color[i];
+  }
+  out.color_palette = std::move(quantized.final_colors);
+  lg::info("Color palette generation took {:.2f} ms", quantize_timer.getMs());
+
+  dedup_tfrag_vertices(out);
+}
+
+s8 normal_to_s8(float in) {
+  s32 in_s32 = in * 127.f;
+  ASSERT(in_s32 <= INT8_MAX);
+  ASSERT(in_s32 >= INT8_MIN);
+  return in_s32;
+}
+
+void add_to_packed_verts(std::vector<tfrag3::PackedTieVertices::Vertex>* out,
+                         const std::vector<tfrag3::PreloadedVertex>& vtx,
+                         const std::vector<math::Vector3f>& normals) {
+  ASSERT(vtx.size() == normals.size());
+  for (size_t i = 0; i < normals.size(); i++) {
+    auto& x = out->emplace_back();
+    // currently not supported.
+    x.r = 255;
+    x.g = 255;
+    x.b = 255;
+    x.a = 255;
+
+    x.x = vtx[i].x;
+    x.y = vtx[i].y;
+    x.z = vtx[i].z;
+
+    x.s = vtx[i].s;
+    x.t = vtx[i].t;
+
+    x.nx = normal_to_s8(normals[i].x());
+    x.ny = normal_to_s8(normals[i].y());
+    x.nz = normal_to_s8(normals[i].z());
+  }
+}
+
+int texture_pool_add_envmap_control_texture(TexturePool* pool,
+                                            const tinygltf::Model& model,
+                                            int rgb_image_id,
+                                            int mr_image_id) {
+  const auto& existing = pool->envmap_textures_by_gltf_id.find({rgb_image_id, mr_image_id});
+  if (existing != pool->envmap_textures_by_gltf_id.end()) {
+    lg::info("Reusing envmap textures");
+    return existing->second;
+  }
+  const auto& rgb_tex = model.images.at(rgb_image_id);
+  const auto& mr_tex = model.images.at(mr_image_id);
+  lg::info("new envmap texture {} {}", rgb_tex.name, mr_tex.name);
+  ASSERT(rgb_tex.bits == 8);
+  ASSERT(rgb_tex.component == 4);
+  ASSERT(rgb_tex.pixel_type == TINYGLTF_TEXTURE_TYPE_UNSIGNED_BYTE);
+
+  ASSERT(mr_tex.bits == 8);
+  ASSERT(mr_tex.pixel_type == TINYGLTF_TEXTURE_TYPE_UNSIGNED_BYTE);
+  ASSERT(mr_tex.component == 4);
+
+  ASSERT(rgb_tex.width == mr_tex.width);
+  ASSERT(rgb_tex.height == mr_tex.height);
+
+  size_t idx = pool->textures_by_idx.size();
+  pool->envmap_textures_by_gltf_id[{rgb_image_id, mr_image_id}] = idx;
+  auto& tt = pool->textures_by_idx.emplace_back();
+  tt.w = rgb_tex.width;
+  tt.h = rgb_tex.height;
+  tt.debug_name = rgb_tex.name;
+  tt.debug_tpage_name = "custom-level";
+  tt.load_to_pool = false;
+  tt.combo_id = 0;  // doesn't matter, not a pool tex
+  tt.data.resize(tt.w * tt.h);
+  ASSERT(rgb_tex.image.size() >= tt.data.size());
+  memcpy(tt.data.data(), rgb_tex.image.data(), tt.data.size() * 4);
+
+  // adjust alpha from metallic channel
+  for (size_t i = 0; i < tt.data.size(); i++) {
+    u32 rgb = tt.data[i];
+    u32 metal = mr_tex.image[4 * i + 2] / 4;
+    rgb &= 0xff'ff'ff;
+    rgb |= (metal << 24);
+    tt.data[i] = rgb;
+  }
+
+  return idx;
+}
+
+void extract(const Input& in,
+             TieOutput& out,
+             const tinygltf::Model& model,
+             const std::vector<NodeWithTransform>& all_nodes) {
+  std::vector<math::Vector<u8, 4>> all_vtx_colors;
+
+  struct MaterialInfo {
+    tfrag3::StripDraw draw;
+    bool needs_tie = false;
+  };
+  std::map<int, MaterialInfo> info_by_material;
+  int mesh_count = 0;
+  int prim_count = 0;
+
+  for (const auto& n : all_nodes) {
+    const auto& node = model.nodes[n.node_idx];
+    if (node.extras.Has("set_invisible") && node.extras.Get("set_invisible").Get<int>()) {
+      continue;
+    }
+    if (node.mesh >= 0) {
+      const auto& mesh = model.meshes[node.mesh];
+      mesh_count++;
+      for (const auto& prim : mesh.primitives) {
+        if (prim.material >= 0 && model.materials[prim.material].extras.Has("set_invisible") &&
+            model.materials[prim.material].extras.Get("set_invisible").Get<int>()) {
+          continue;
+        }
+
+        if (!prim_needs_tie(model, prim)) {
+          continue;
+        }
+        prim_count++;
+        // extract index buffer
+        std::vector<u32> prim_indices = gltf_index_buffer(model, prim.indices, out.vertices.size());
+        ASSERT_MSG(prim.mode == TINYGLTF_MODE_TRIANGLES, "Unsupported triangle mode");
+        // extract vertices
+        auto verts = gltf_vertices(model, prim.attributes, n.w_T_node, true, true, mesh.name);
+        add_to_packed_verts(&out.vertices, verts.vtx, verts.normals);
+        all_vtx_colors.insert(all_vtx_colors.end(), verts.vtx_colors.begin(),
+                              verts.vtx_colors.end());
+        ASSERT(all_vtx_colors.size() == out.vertices.size());
+
+        auto& info = info_by_material[prim.material];
+        info.draw.mode = make_default_draw_mode();                        // todo rm
+        info.draw.tree_tex_id = texture_pool_debug_checker(in.tex_pool);  // todo rm
+        info.draw.num_triangles += prim_indices.size() / 3;
+        if (info.draw.vis_groups.empty()) {
+          auto& grp = info.draw.vis_groups.emplace_back();
+          grp.num_inds += prim_indices.size();
+          grp.num_tris += info.draw.num_triangles;
+          grp.vis_idx_in_pc_bvh = UINT16_MAX;
+        } else {
+          auto& grp = info.draw.vis_groups.back();
+          grp.num_inds += prim_indices.size();
+          grp.num_tris += info.draw.num_triangles;
+          grp.vis_idx_in_pc_bvh = UINT16_MAX;
+        }
+
+        info.draw.plain_indices.insert(info.draw.plain_indices.end(), prim_indices.begin(),
+                                       prim_indices.end());
+      }
+    }
+  }
+
+  for (const auto& [mat_idx, d_] : info_by_material) {
+    // out.strip_draws.push_back(d_);
+    // auto& draw = out.strip_draws.back();
+    tfrag3::StripDraw draw = d_.draw;
+    draw.mode = make_default_draw_mode();
+
+    if (mat_idx == -1) {
+      lg::warn("Draw had a material index of -1, using default texture.");
+      draw.tree_tex_id = texture_pool_debug_checker(in.tex_pool);
+      out.base_draws.push_back(draw);
+      continue;
+    }
+
+    const auto& mat = model.materials[mat_idx];
+    setup_alpha_from_material(mat, &draw.mode);
+    int base_tex_idx = mat.pbrMetallicRoughness.baseColorTexture.index;
+    if (base_tex_idx == -1) {
+      lg::warn("Material {} has no texture, using default texture.", mat.name);
+      draw.tree_tex_id = texture_pool_debug_checker(in.tex_pool);
+      out.base_draws.push_back(draw);
+      continue;
+    }
+    int roughness_tex_idx = mat.pbrMetallicRoughness.metallicRoughnessTexture.index;
+    ASSERT(roughness_tex_idx >= 0);
+    const auto& base_tex = model.textures[base_tex_idx];
+    ASSERT(base_tex.sampler >= 0);
+    ASSERT(base_tex.source >= 0);
+    setup_draw_mode_from_sampler(model.samplers.at(base_tex.sampler), &draw.mode);
+    const auto& roughness_tex = model.textures.at(roughness_tex_idx);
+    ASSERT(roughness_tex.sampler >= 0);
+    ASSERT(roughness_tex.source >= 0);
+
+    // draw.tree_tex_id = texture_pool_add_texture(in.tex_pool, model.images[base_tex.source]);
+    draw.tree_tex_id = texture_pool_add_envmap_control_texture(in.tex_pool, model, base_tex.source,
+                                                               roughness_tex.source);
+    out.base_draws.push_back(draw);
+
+    // now, setup envmap draw:
+    auto envmap_settings = envmap_settings_from_gltf(mat);
+    const auto& envmap_tex = model.textures[envmap_settings.texture_idx];
+    ASSERT(envmap_tex.sampler >= 0);
+    ASSERT(envmap_tex.source >= 0);
+    draw.mode = make_default_draw_mode();
+    setup_draw_mode_from_sampler(model.samplers.at(envmap_tex.sampler), &draw.mode);
+    draw.tree_tex_id = texture_pool_add_texture(in.tex_pool, model.images[envmap_tex.source]);
+    draw.mode.set_alpha_blend(DrawMode::AlphaBlend::SRC_0_DST_DST);
+    draw.mode.enable_ab();
+
+    out.envmap_draws.push_back(draw);
+  }
+  lg::info("total of {} normal TIE draws, {} envmap", out.base_draws.size(),
+           out.envmap_draws.size());
 
   lg::info("Merged {} meshes and {} prims into {} vertices", mesh_count, prim_count,
            out.vertices.size());
 
-  if (in.get_colors) {
+  Timer quantize_timer;
-    Timer quantize_timer;
+  auto quantized = quantize_colors_kd_tree(all_vtx_colors, 10);
-    auto quantized = quantize_colors_octree(all_vtx_colors, 1024);
+  for (size_t i = 0; i < out.vertices.size(); i++) {
-    for (size_t i = 0; i < out.vertices.size(); i++) {
+    out.color_indices.push_back(quantized.vtx_to_color[i]);
-      out.vertices[i].color_index = quantized.vtx_to_color[i];
-    }
-    out.color_palette = std::move(quantized.final_colors);
-    lg::info("Color palette generation took {:.2f} ms", quantize_timer.getMs());
   }
+  out.color_palette = std::move(quantized.final_colors);
+  lg::info("Color palette generation took {:.2f} ms", quantize_timer.getMs());
 
-  dedup_vertices(out);
+  dedup_tie_vertices(out);
 }
 
 std::optional<std::vector<jak1::CollideFace>> subdivide_face_if_needed(jak1::CollideFace face_in) {
@@ -386,6 +684,7 @@ void extract(const Input& in, Output& out) {
   auto all_nodes = flatten_nodes_from_all_scenes(model);
   extract(in, out.tfrag, model, all_nodes);
   extract(in, out.collide, model, all_nodes);
+  extract(in, out.tie, model, all_nodes);
   lg::info("GLTF total took {:.2f} ms", read_timer.getMs());
 }
 }  // namespace gltf_mesh_extract

goalc/build_level/common/gltf_mesh_extract.h

@@ -17,15 +17,15 @@ namespace gltf_mesh_extract {
 struct Input {
   std::string filename;
   gltf_util::TexturePool* tex_pool = nullptr;
-  bool get_colors = true;
   bool auto_wall_enable = true;
   float auto_wall_angle = 30.f;
   bool double_sided_collide = false;
 };
 
 struct TfragOutput {
-  std::vector<tfrag3::StripDraw> strip_draws;
+  std::vector<tfrag3::StripDraw> normal_strip_draws;
-  std::vector<tfrag3::PreloadedVertex> vertices;
+  std::vector<tfrag3::StripDraw> trans_strip_draws;
+  std::vector<tfrag3::PreloadedVertex> tfrag_vertices;
   std::vector<math::Vector<u8, 4>> color_palette;
 };
 
@@ -33,9 +33,18 @@ struct CollideOutput {
   std::vector<jak1::CollideFace> faces;
 };
 
+struct TieOutput {
+  std::vector<tfrag3::StripDraw> base_draws;
+  std::vector<tfrag3::StripDraw> envmap_draws;
+  std::vector<tfrag3::PackedTieVertices::Vertex> vertices;
+  std::vector<u16> color_indices;
+  std::vector<math::Vector<u8, 4>> color_palette;
+};
+
 struct Output {
   TfragOutput tfrag;
   CollideOutput collide;
+  TieOutput tie;
 };
 
 struct PatResult {

goalc/build_level/jak1/LevelFile.cpp

@@ -26,6 +26,7 @@ size_t DrawableTreeArray::add_to_object_file(DataObjectGenerator& gen) const {
   num_trees += tfrags.size();
   num_trees += collides.size();
   num_trees += ambients.size();
+  num_trees += ties.size();
   gen.add_word(num_trees << 16);
   gen.add_word(0);
   gen.add_word(0);
@@ -35,8 +36,6 @@ size_t DrawableTreeArray::add_to_object_file(DataObjectGenerator& gen) const {
   gen.add_word(0);
   gen.add_word(0);
 
-  // todo add trees...
-
   if (num_trees == 0) {
     gen.add_word(0);  // the one at the end.
   } else {
@@ -54,6 +53,10 @@ size_t DrawableTreeArray::add_to_object_file(DataObjectGenerator& gen) const {
       gen.link_word_to_byte(tree_word++, collide.add_to_object_file(gen));
     }
 
+    for (auto& tie : ties) {
+      gen.link_word_to_byte(tree_word++, tie.add_to_object_file(gen));
+    }
+
     for (auto& ambient : ambients) {
       gen.link_word_to_byte(tree_word++, ambient.add_to_object_file(gen, ambient_arr_slot));
     }

goalc/build_level/jak1/LevelFile.h

@@ -15,14 +15,13 @@
 #include "goalc/build_level/collide/jak1/collide_drawable.h"
 #include "goalc/build_level/collide/jak1/collide_pack.h"
 #include "goalc/build_level/common/Tfrag.h"
+#include "goalc/build_level/common/Tie.h"
 
 namespace jak1 {
 struct VisibilityString {
   std::vector<u8> bytes;
 };
 
-struct DrawableTreeInstanceTie {};
-
 struct DrawableTreeActor {};
 
 struct DrawableTreeInstanceShrub {};

goalc/build_level/jak1/build_level.cpp

@@ -80,9 +80,19 @@ bool run_build_level(const std::string& input_file,
   pc_level.level_name = file.name;
 
   // TFRAG
-  auto& tfrag_drawable_tree = file.drawable_trees.tfrags.emplace_back();
+  file.drawable_trees.tfrags.emplace_back("drawable-tree-tfrag", "drawable-inline-array-tfrag");
-  tfrag_from_gltf(mesh_extract_out.tfrag, tfrag_drawable_tree,
+  file.drawable_trees.tfrags.emplace_back("drawable-tree-trans-tfrag",
-                  pc_level.tfrag_trees[0].emplace_back());
+                                          "drawable-inline-array-trans-tfrag");
+
+  tfrag_from_gltf(mesh_extract_out.tfrag, pc_level.tfrag_trees[0]);
+
+  // TIE
+  if (!mesh_extract_out.tie.base_draws.empty()) {
+    file.drawable_trees.ties.emplace_back();
+    tie_from_gltf(mesh_extract_out.tie, pc_level.tie_trees[0]);
+  }
+
+  // TEXTURE
   pc_level.textures = std::move(tex_pool.textures_by_idx);
 
   // COLLIDE

goalc/build_level/jak2/build_level.cpp

@@ -69,9 +69,8 @@ bool run_build_level(const std::string& input_file,
   pc_level.level_name = file.name;
 
   // TFRAG
-  auto& tfrag_drawable_tree = file.drawable_trees.tfrags.emplace_back();
+  file.drawable_trees.tfrags.emplace_back("drawable-tree-tfrag", "drawable-inline-array-tfrag");
-  tfrag_from_gltf(mesh_extract_out.tfrag, tfrag_drawable_tree,
+  tfrag_from_gltf(mesh_extract_out.tfrag, pc_level.tfrag_trees[0]);
-                  pc_level.tfrag_trees[0].emplace_back());
   pc_level.textures = std::move(tex_pool.textures_by_idx);
 
   // COLLIDE

goalc/build_level/jak3/build_level.cpp

@@ -67,9 +67,8 @@ bool run_build_level(const std::string& input_file,
   pc_level.level_name = file.name;
 
   // TFRAG
-  auto& tfrag_drawable_tree = file.drawable_trees.tfrags.emplace_back();
+  file.drawable_trees.tfrags.emplace_back("drawable-tree-tfrag", "drawable-inline-array-tfrag");
-  tfrag_from_gltf(mesh_extract_out.tfrag, tfrag_drawable_tree,
+  tfrag_from_gltf(mesh_extract_out.tfrag, pc_level.tfrag_trees[0]);
-                  pc_level.tfrag_trees[0].emplace_back());
   pc_level.textures = std::move(tex_pool.textures_by_idx);
 
   // COLLIDE