Improvements for stuttering (#1768)

* tie changes

* other changes

* cleanpu

* remove silly profiling

* loader frames bug fix

* fix merge issue

game/graphics/opengl_renderer/background/Tie3.cpp

@@ -1,5 +1,7 @@
 #include "Tie3.h"
 
+#include "common/global_profiler/GlobalProfiler.h"
+
 #include "third-party/imgui/imgui.h"
 
 Tie3::Tie3(const std::string& name, BucketId my_id, int level_id)
@@ -15,6 +17,7 @@ Tie3::~Tie3() {
 }
 
 void Tie3::update_load(const LevelData* loader_data) {
+  auto ul = scoped_prof("update-load");
   const tfrag3::Level* lev_data = loader_data->level.get();
   m_wind_vectors.clear();
   // We changed level!
@@ -31,6 +34,7 @@ void Tie3::update_load(const LevelData* loader_data) {
   size_t max_inds = 0;
   for (u32 l_geo = 0; l_geo < tfrag3::TIE_GEOS; l_geo++) {
     for (u32 l_tree = 0; l_tree < lev_data->tie_trees[l_geo].size(); l_tree++) {
+      auto ul = scoped_prof("load-tree");
       size_t wind_idx_buffer_len = 0;
       size_t num_grps = 0;
       const auto& tree = lev_data->tie_trees[l_geo][l_tree];
@@ -90,11 +94,7 @@ void Tie3::update_load(const LevelData* loader_data) {
       );
 
       glGenBuffers(1, &lod_tree[l_tree].single_draw_index_buffer);
-      glGenBuffers(1, &lod_tree[l_tree].index_buffer);
-      glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, lod_tree[l_tree].index_buffer);
-      // todo: move to loader, this will probably be quite slow.
-      glBufferData(GL_ELEMENT_ARRAY_BUFFER, tree.unpacked.indices.size() * sizeof(u32),
-                   tree.unpacked.indices.data(), GL_STATIC_DRAW);
+      lod_tree[l_tree].index_buffer = loader_data->tie_data[l_geo][l_tree].index_buffer;
 
       if (wind_idx_buffer_len > 0) {
         lod_tree[l_tree].wind_matrix_cache.resize(tree.wind_instance_info.size());
@@ -278,7 +278,7 @@ void Tie3::discard_tree_cache() {
     for (auto& tree : m_trees[geo]) {
       glBindTexture(GL_TEXTURE_1D, tree.time_of_day_texture);
       glDeleteTextures(1, &tree.time_of_day_texture);
-      glDeleteBuffers(1, &tree.index_buffer);
+      // glDeleteBuffers(1, &tree.index_buffer);
       glDeleteBuffers(1, &tree.single_draw_index_buffer);
       glDeleteVertexArrays(1, &tree.vao);
     }

game/graphics/opengl_renderer/loader/Loader.cpp

@@ -1,5 +1,6 @@
 #include "Loader.h"
 
+#include "common/global_profiler/GlobalProfiler.h"
 #include "common/util/FileUtil.h"
 #include "common/util/Timer.h"
 #include "common/util/compress.h"
@@ -192,6 +193,7 @@ void Loader::load_common(TexturePool& tex_pool, const std::string& name) {
 bool Loader::upload_textures(Timer& timer, LevelData& data, TexturePool& texture_pool) {
   // try to move level from initializing to initialized:
 
+  auto evt = scoped_prof("upload-textures");
   constexpr int MAX_TEX_BYTES_PER_FRAME = 1024 * 128;
 
   int bytes_this_run = 0;
@@ -303,6 +305,7 @@ void Loader::update(TexturePool& texture_pool) {
       loader_input.tex_pool = &texture_pool;
 
       for (auto& stage : m_loader_stages) {
+        auto evt = scoped_prof(fmt::format("stage-{}", stage->name()).c_str());
         Timer stage_timer;
         done = stage->run(loader_timer, loader_input);
         if (stage_timer.getMs() > 5.f) {
@@ -314,6 +317,7 @@ void Loader::update(TexturePool& texture_pool) {
       }
 
       if (done) {
+        auto evt = scoped_prof("finish-stages");
         lk.lock();
         m_loaded_tfrag3_levels[name] = std::move(lev);
         m_initializing_tfrag3_levels.erase(it);
@@ -326,6 +330,7 @@ void Loader::update(TexturePool& texture_pool) {
   }
 
   if (!did_gpu_stuff) {
+    auto evt = scoped_prof("gpu-unload");
     // try to remove levels.
     Timer unload_timer;
     if (m_loaded_tfrag3_levels.size() >= 3) {
@@ -362,6 +367,7 @@ void Loader::update(TexturePool& texture_pool) {
               if (tie_tree.has_wind) {
                 glDeleteBuffers(1, &tie_tree.wind_indices);
               }
+              glDeleteBuffers(1, &tie_tree.index_buffer);
             }
           }
 

game/graphics/opengl_renderer/loader/LoaderStages.cpp

@@ -2,6 +2,8 @@
 
 #include "Loader.h"
 
+#include "common/global_profiler/GlobalProfiler.h"
+
 constexpr float LOAD_BUDGET = 2.5f;
 
 /*!
@@ -278,6 +280,7 @@ class TieLoadStage : public LoaderStage {
     }
 
     if (!m_opengl_created) {
+      auto evt = scoped_prof("tie-opengl-create");
       for (int geo = 0; geo < tfrag3::TIE_GEOS; geo++) {
         auto& in_trees = data.lev_data->level->tie_trees[geo];
         for (auto& in_tree : in_trees) {
@@ -287,6 +290,11 @@ class TieLoadStage : public LoaderStage {
           glBufferData(GL_ARRAY_BUFFER,
                        in_tree.unpacked.vertices.size() * sizeof(tfrag3::PreloadedVertex), nullptr,
                        GL_STATIC_DRAW);
+
+          glGenBuffers(1, &tree_out.index_buffer);
+          glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, tree_out.index_buffer);
+          glBufferData(GL_ELEMENT_ARRAY_BUFFER, in_tree.unpacked.indices.size() * sizeof(u32),
+                       nullptr, GL_STATIC_DRAW);
         }
       }
       m_opengl_created = true;
@@ -294,6 +302,7 @@ class TieLoadStage : public LoaderStage {
     }
 
     if (!m_verts_done) {
+      auto evt = scoped_prof("tie-verts");
       constexpr u32 CHUNK_SIZE = 32768;
       u32 uploaded_bytes = 0;
 
@@ -324,8 +333,12 @@ class TieLoadStage : public LoaderStage {
                      data.lev_data->tie_data[m_next_geo][m_next_tree].vertex_buffer);
         u32 upload_size =
             (end_vert_for_chunk - start_vert_for_chunk) * sizeof(tfrag3::PreloadedVertex);
-        glBufferSubData(GL_ARRAY_BUFFER, start_vert_for_chunk * sizeof(tfrag3::PreloadedVertex),
-                        upload_size, tree.unpacked.vertices.data() + start_vert_for_chunk);
+        {
+          auto bsd = scoped_prof(fmt::format("buffer-{}k", upload_size / 1024).c_str());
+          glBufferSubData(GL_ARRAY_BUFFER, start_vert_for_chunk * sizeof(tfrag3::PreloadedVertex),
+                          upload_size, tree.unpacked.vertices.data() + start_vert_for_chunk);
+        }
+
         uploaded_bytes += upload_size;
 
         if (complete_tree) {
@@ -352,6 +365,7 @@ class TieLoadStage : public LoaderStage {
     }
 
     if (!m_wind_indices_done) {
+      auto evt = scoped_prof("tie-wind");
       bool abort = false;
       for (; m_next_geo < tfrag3::TIE_GEOS; m_next_geo++) {
         auto& geo_trees = data.lev_data->level->tie_trees[m_next_geo];
@@ -383,12 +397,75 @@ class TieLoadStage : public LoaderStage {
             abort = true;
           }
         }
-        m_next_tree = 0;
       }
 
-      m_indices_done = true;
-      m_done = true;
-      return true;
+      m_wind_indices_done = true;
+      m_next_geo = 0;
+      m_next_vert = 0;
+      m_next_tree = 0;
+
+      if (timer.getMs() > LOAD_BUDGET) {
+        return false;
+      }
+    }
+
+    if (!m_indices_done) {
+      auto evt = scoped_prof("tie-ind");
+      constexpr u32 CHUNK_SIZE = 32768 * 8;
+      u32 uploaded_bytes = 0;
+
+      while (true) {
+        const auto& tree = data.lev_data->level->tie_trees[m_next_geo][m_next_tree];
+        u32 end_ind_in_tree = tree.unpacked.indices.size();
+        // the number of indices we'd need to finish the tree right now
+        size_t num_inds_left_in_tree = end_ind_in_tree - m_next_vert;
+        size_t start_ind_for_chunk;
+        size_t end_ind_for_chunk;
+
+        bool complete_tree;
+
+        if (num_inds_left_in_tree > CHUNK_SIZE) {
+          complete_tree = false;
+          // should only do partial
+          start_ind_for_chunk = m_next_vert;
+          end_ind_for_chunk = start_ind_for_chunk + CHUNK_SIZE;
+          m_next_vert += CHUNK_SIZE;
+        } else {
+          // should do all!
+          start_ind_for_chunk = m_next_vert;
+          end_ind_for_chunk = end_ind_in_tree;
+          complete_tree = true;
+        }
+
+        glBindBuffer(GL_ELEMENT_ARRAY_BUFFER,
+                     data.lev_data->tie_data[m_next_geo][m_next_tree].index_buffer);
+        u32 upload_size = (end_ind_for_chunk - start_ind_for_chunk) * sizeof(u32);
+        glBufferSubData(GL_ELEMENT_ARRAY_BUFFER, start_ind_for_chunk * sizeof(u32), upload_size,
+                        tree.unpacked.indices.data() + start_ind_for_chunk);
+        uploaded_bytes += upload_size;
+
+        if (complete_tree) {
+          // and move on to next tree
+          m_next_vert = 0;
+          m_next_tree++;
+          if (m_next_tree >= data.lev_data->level->tie_trees[m_next_geo].size()) {
+            m_next_tree = 0;
+            m_next_geo++;
+            if (m_next_geo >= tfrag3::TIE_GEOS) {
+              m_indices_done = true;
+              m_next_tree = 0;
+              m_next_geo = 0;
+              m_next_vert = 0;
+              m_done = true;
+              return true;
+            }
+          }
+        }
+
+        if (timer.getMs() > LOAD_BUDGET || (uploaded_bytes / 1024) > 2048) {
+          return false;
+        }
+      }
     }
 
     return false;
@@ -461,6 +538,23 @@ class CollideLoaderStage : public LoaderStage {
   bool m_done = false;
 };
 
+class StallLoaderStage : public LoaderStage {
+ public:
+  StallLoaderStage() : LoaderStage("stall") {}
+  bool run(Timer&, LoaderInput& data) override {
+    m_count++;
+    if (m_count > 10) {
+      return true;
+    }
+    return false;
+  }
+
+  void reset() override { m_count = 0; }
+
+ private:
+  int m_count = 0;
+};
+
 MercLoaderStage::MercLoaderStage() : LoaderStage("merc") {}
 void MercLoaderStage::reset() {
   m_done = false;
@@ -531,5 +625,6 @@ std::vector<std::unique_ptr<LoaderStage>> make_loader_stages() {
   ret.push_back(std::make_unique<ShrubLoadStage>());
   ret.push_back(std::make_unique<CollideLoaderStage>());
   ret.push_back(std::make_unique<MercLoaderStage>());
+  ret.push_back(std::make_unique<StallLoaderStage>());
   return ret;
 }

game/graphics/opengl_renderer/loader/common.h

@@ -15,6 +15,7 @@ struct LevelData {
 
   struct TieOpenGL {
     GLuint vertex_buffer;
+    GLuint index_buffer;
     bool has_wind = false;
     GLuint wind_indices;
   };

game/graphics/pipelines/opengl.cpp

@@ -213,7 +213,8 @@ static std::shared_ptr<GfxDisplay> gl_make_display(int width,
   }
 
   auto display = std::make_shared<GLDisplay>(window, is_main);
-
+  display->set_imgui_visible(Gfx::get_debug_menu_visible_on_startup());
+  display->update_cursor_visibility(window, display->is_imgui_visible());
   // lg::debug("init display #x{:x}", (uintptr_t)display);
 
   // setup imgui
@@ -482,19 +483,40 @@ void render_game_frame(int game_width,
 }
 
 void GLDisplay::get_position(int* x, int* y) {
-  glfwGetWindowPos(m_window, x, y);
+  std::lock_guard<std::mutex> lk(m_lock);
+  if (x) {
+    *x = m_display_state.window_pos_x;
+  }
+  if (y) {
+    *y = m_display_state.window_pos_y;
+  }
 }
 
 void GLDisplay::get_size(int* width, int* height) {
-  glfwGetFramebufferSize(m_window, width, height);
+  std::lock_guard<std::mutex> lk(m_lock);
+  if (width) {
+    *width = m_display_state.window_size_width;
+  }
+  if (height) {
+    *height = m_display_state.window_size_height;
+  }
 }
 
 void GLDisplay::get_scale(float* xs, float* ys) {
-  glfwGetWindowContentScale(m_window, xs, ys);
+  std::lock_guard<std::mutex> lk(m_lock);
+  if (xs) {
+    *xs = m_display_state.window_scale_x;
+  }
+  if (ys) {
+    *ys = m_display_state.window_scale_y;
+  }
 }
 
 void GLDisplay::set_size(int width, int height) {
-  glfwSetWindowSize(m_window, width, height);
+  // glfwSetWindowSize(m_window, width, height);
+  m_pending_size.width = width;
+  m_pending_size.height = height;
+  m_pending_size.pending = true;
 
   if (windowed()) {
     m_last_windowed_width = width;
@@ -566,48 +588,45 @@ void GLDisplay::update_fullscreen(GfxDisplayMode mode, int screen) {
 }
 
 int GLDisplay::get_screen_vmode_count() {
-  int count = 0;
-  glfwGetVideoModes(get_monitor(fullscreen_screen()), &count);
-  return count;
+  std::lock_guard<std::mutex> lk(m_lock);
+  return m_display_state.num_vmodes;
 }
 
 void GLDisplay::get_screen_size(int vmode_idx, s32* w_out, s32* h_out) {
-  GLFWmonitor* monitor = get_monitor(fullscreen_screen());
-  auto vmode = glfwGetVideoMode(monitor);
-  int count = 0;
-  auto vmodes = glfwGetVideoModes(monitor, &count);
-  if (vmode_idx >= 0) {
-    vmode = &vmodes[vmode_idx];
-  } else if (fullscreen_mode() == GfxDisplayMode::Fullscreen) {
-    for (int i = 0; i < count; ++i) {
-      if (!vmode || vmode->height < vmodes[i].height) {
-        vmode = &vmodes[i];
-      }
+  std::lock_guard<std::mutex> lk(m_lock);
+  if (vmode_idx >= 0 && vmode_idx < MAX_VMODES) {
+    if (w_out) {
+      *w_out = m_display_state.vmodes[vmode_idx].width;
+    }
+    if (h_out) {
+      *h_out = m_display_state.vmodes[vmode_idx].height;
+    }
+  } else if (fullscreen_mode() == Fullscreen) {
+    if (w_out) {
+      *w_out = m_display_state.largest_vmode_width;
+    }
+    if (h_out) {
+      *h_out = m_display_state.largest_vmode_height;
+    }
+  } else {
+    if (w_out) {
+      *w_out = m_display_state.current_vmode.width;
+    }
+    if (h_out) {
+      *h_out = m_display_state.current_vmode.height;
     }
-  }
-  if (w_out) {
-    *w_out = vmode->width;
-  }
-  if (h_out) {
-    *h_out = vmode->height;
   }
 }
 
 int GLDisplay::get_screen_rate(int vmode_idx) {
-  GLFWmonitor* monitor = get_monitor(fullscreen_screen());
-  auto vmode = glfwGetVideoMode(monitor);
-  int count = 0;
-  auto vmodes = glfwGetVideoModes(monitor, &count);
-  if (vmode_idx >= 0) {
-    vmode = &vmodes[vmode_idx];
+  std::lock_guard<std::mutex> lk(m_lock);
+  if (vmode_idx >= 0 && vmode_idx < MAX_VMODES) {
+    return m_display_state.vmodes[vmode_idx].refresh_rate;
   } else if (fullscreen_mode() == GfxDisplayMode::Fullscreen) {
-    for (int i = 0; i < count; ++i) {
-      if (!vmode || vmode->refreshRate < vmodes[i].refreshRate) {
-        vmode = &vmodes[i];
-      }
-    }
+    return m_display_state.largest_vmode_refresh_rate;
+  } else {
+    return m_display_state.current_vmode.refresh_rate;
   }
-  return vmode->refreshRate;
 }
 
 GLFWmonitor* GLDisplay::get_monitor(int index) {
@@ -632,8 +651,17 @@ void GLDisplay::set_lock(bool lock) {
 }
 
 bool GLDisplay::fullscreen_pending() {
-  GLFWmonitor* monitor = get_monitor(fullscreen_screen());
-  auto vmode = glfwGetVideoMode(monitor);
+  GLFWmonitor* monitor;
+  {
+    auto _ = scoped_prof("get_monitor");
+    monitor = get_monitor(fullscreen_screen());
+  }
+
+  const GLFWvidmode* vmode;
+  {
+    auto _ = scoped_prof("get-video-mode");
+    vmode = glfwGetVideoMode(monitor);
+  }
 
   return GfxDisplay::fullscreen_pending() ||
          (vmode->width != m_last_video_mode.width || vmode->height != m_last_video_mode.height ||
@@ -658,19 +686,73 @@ void update_global_profiler() {
   prof().set_enable(g_gfx_data->debug_gui.record_events);
 }
 
+void GLDisplay::VMode::set(const GLFWvidmode* vmode) {
+  width = vmode->width;
+  height = vmode->height;
+  refresh_rate = vmode->refreshRate;
+}
+
+void GLDisplay::update_glfw() {
+  auto p = scoped_prof("update_glfw");
+
+  glfwPollEvents();
+  glfwMakeContextCurrent(m_window);
+  auto& mapping_info = Gfx::get_button_mapping();
+  Pad::update_gamepads(mapping_info);
+
+  glfwGetFramebufferSize(m_window, &m_display_state_copy.window_size_width,
+                         &m_display_state_copy.window_size_height);
+
+  glfwGetWindowContentScale(m_window, &m_display_state_copy.window_scale_x,
+                            &m_display_state_copy.window_scale_y);
+
+  glfwGetWindowPos(m_window, &m_display_state_copy.window_pos_x,
+                   &m_display_state_copy.window_pos_y);
+
+  GLFWmonitor* monitor = get_monitor(fullscreen_screen());
+  auto current_vmode = glfwGetVideoMode(monitor);
+  if (current_vmode) {
+    m_display_state_copy.current_vmode.set(current_vmode);
+  }
+
+  int count = 0;
+  auto vmodes = glfwGetVideoModes(monitor, &count);
+
+  if (count > MAX_VMODES) {
+    fmt::print("got too many vmodes: {}\n", count);
+    count = MAX_VMODES;
+  }
+
+  m_display_state_copy.num_vmodes = count;
+
+  m_display_state_copy.largest_vmode_width = 1;
+  m_display_state_copy.largest_vmode_refresh_rate = 1;
+  for (int i = 0; i < count; i++) {
+    if (vmodes[i].width > m_display_state_copy.largest_vmode_width) {
+      m_display_state_copy.largest_vmode_height = vmodes[i].height;
+      m_display_state_copy.largest_vmode_width = vmodes[i].width;
+    }
+
+    if (vmodes[i].refreshRate > m_display_state_copy.largest_vmode_refresh_rate) {
+      m_display_state_copy.largest_vmode_refresh_rate = vmodes[i].refreshRate;
+    }
+    m_display_state_copy.vmodes[i].set(&vmodes[i]);
+  }
+
+  if (m_pending_size.pending) {
+    glfwSetWindowSize(m_window, m_pending_size.width, m_pending_size.height);
+    m_pending_size.pending = false;
+  }
+
+  std::lock_guard<std::mutex> lk(m_lock);
+  m_display_state = m_display_state_copy;
+}
+
 /*!
  * Main function called to render graphics frames. This is called in a loop.
  */
 void GLDisplay::render() {
-  // poll events
-  {
-    auto p = scoped_prof("poll-gamepads");
-    glfwPollEvents();
-    glfwMakeContextCurrent(m_window);
-
-    auto& mapping_info = Gfx::get_button_mapping();
-    Pad::update_gamepads(mapping_info);
-  }
+  update_glfw();
 
   // imgui start of frame
   {
@@ -717,18 +799,30 @@ void GLDisplay::render() {
     ImGui_ImplOpenGL3_RenderDrawData(ImGui::GetDrawData());
   }
 
+  // update fullscreen mode, if requested
+  {
+    auto p = scoped_prof("fullscreen-update");
+    update_last_fullscreen_mode();
+
+    if (fullscreen_pending() && !minimized()) {
+      fullscreen_flush();
+    }
+  }
+
   // actual vsync
   g_gfx_data->debug_gui.finish_frame();
-  {
-    auto p = scoped_prof("swap-buffers");
-    glfwSwapBuffers(m_window);
-  }
   if (Gfx::g_global_settings.framelimiter) {
     auto p = scoped_prof("frame-limiter");
     g_gfx_data->frame_limiter.run(
         Gfx::g_global_settings.target_fps, Gfx::g_global_settings.experimental_accurate_lag,
         Gfx::g_global_settings.sleep_in_frame_limiter, g_gfx_data->last_engine_time);
   }
+
+  {
+    auto p = scoped_prof("swap-buffers");
+    glfwSwapBuffers(m_window);
+  }
+
   // actually wait for vsync
   if (g_gfx_data->debug_gui.should_gl_finish()) {
     glFinish();
@@ -755,16 +849,6 @@ void GLDisplay::render() {
     g_gfx_data->sync_cv.notify_all();
   }
 
-  // update fullscreen mode, if requested
-  {
-    auto p = scoped_prof("fullscreen-update");
-    update_last_fullscreen_mode();
-
-    if (fullscreen_pending() && !minimized()) {
-      fullscreen_flush();
-    }
-  }
-
   // reboot whole game, if requested
   if (g_gfx_data->debug_gui.want_reboot_in_debug) {
     g_gfx_data->debug_gui.want_reboot_in_debug = false;
@@ -845,6 +929,11 @@ void gl_send_chain(const void* data, u32 offset) {
   }
 }
 
+/*!
+ * Upload texture outside of main DMA chain.
+ * We trust the game to not remove textures that are currently being used, but if the game is messed
+ * up, there is a possible race to updating this texture.
+ */
 void gl_texture_upload_now(const u8* tpage, int mode, u32 s7_ptr) {
   // block
   if (g_gfx_data) {
@@ -855,6 +944,10 @@ void gl_texture_upload_now(const u8* tpage, int mode, u32 s7_ptr) {
   }
 }
 
+/*!
+ * Handle a local->local texture copy. The texture pool can just update texture pointers.
+ * This is called from the main thread and the texture pool itself will handle locking.
+ */
 void gl_texture_relocate(u32 destination, u32 source, u32 format) {
   if (g_gfx_data) {
     g_gfx_data->texture_pool->relocate(destination, source, format);

game/graphics/pipelines/opengl.h

@@ -6,6 +6,8 @@
  */
 
 #define GLFW_INCLUDE_NONE
+#include <mutex>
+
 #include "game/graphics/display.h"
 #include "game/graphics/gfx.h"
 
@@ -52,10 +54,44 @@ class GLDisplay : public GfxDisplay {
   void update_cursor_visibility(GLFWwindow* window, bool is_visible);
 
  private:
+  void update_glfw();
+
   GLFWwindow* m_window;
   bool m_minimized = false;
   GLFWvidmode m_last_video_mode = {0, 0, 0, 0, 0, 0};
 
+  static constexpr int MAX_VMODES = 128;
+
+  struct VMode {
+    void set(const GLFWvidmode* vmode);
+    int width = 640, height = 480;
+    int refresh_rate = 60;
+  };
+
+  struct DisplayState {
+    s32 window_pos_x = 0;
+    s32 window_pos_y = 0;
+    int window_size_width = 640, window_size_height = 480;
+    float window_scale_x = 1.f, window_scale_y = 1.f;
+
+    bool pending_size_change = false;
+    s32 requested_size_width = 0;
+    s32 requested_size_height = 0;
+
+    int num_vmodes = 0;
+    VMode vmodes[MAX_VMODES];
+    int largest_vmode_width = 640, largest_vmode_height = 480;
+    int largest_vmode_refresh_rate = 60;
+    VMode current_vmode;
+  } m_display_state, m_display_state_copy;
+  std::mutex m_lock;
+
+  struct {
+    bool pending = false;
+    int width = 0;
+    int height = 0;
+  } m_pending_size;
+
   GLFWmonitor* get_monitor(int index);
 };
 

game/kernel/common/kmachine.cpp

@@ -340,6 +340,7 @@ void DecodeTime(u32 ptr) {
  */
 /*!
  * Get a 300MHz timer value.
+ * Called from EE thread
  */
 u64 read_ee_timer() {
   u64 ns = ee_clock_timer.getNs();
@@ -354,7 +355,7 @@ void c_memmove(u32 dst, u32 src, u32 size) {
 }
 
 /*!
- * Returns size of window.
+ * Returns size of window. Called from game thread
  */
 void get_window_size(u32 w_ptr, u32 h_ptr) {
   if (w_ptr) {

game/kernel/jak1/kmachine.cpp

@@ -381,10 +381,16 @@ int ShutdownMachine() {
 }
 
 // todo, these could probably be moved to common
+/*!
+ * Called from game thread to submit rendering DMA chain.
+ */
 void send_gfx_dma_chain(u32 /*bank*/, u32 chain) {
   Gfx::send_chain(g_ee_main_mem, chain);
 }
 
+/*!
+ * Called from game thread to upload a texture outside of the main DMA chain.
+ */
 void pc_texture_upload_now(u32 page, u32 mode) {
   Gfx::texture_upload_now(Ptr<u8>(page).c(), mode, s7.offset);
 }
@@ -393,11 +399,20 @@ void pc_texture_relocate(u32 dst, u32 src, u32 format) {
   Gfx::texture_relocate(dst, src, format);
 }
 
+/*!
+ * Called from the game thread at initialization.
+ * The game thread is the only one to touch the mips2c function table (through the linker and
+ * through this function), so no locking is needed.
+ */
 u64 pc_get_mips2c(u32 name) {
   const char* n = Ptr<String>(name).c()->data();
   return Mips2C::gLinkedFunctionTable.get(n);
 }
 
+/*!
+ * Called from the game thread at each frame to tell the PC rendering code which levels to start
+ * loading. The loader internally handles locking.
+ */
 void pc_set_levels(u32 l0, u32 l1) {
   std::string l0s = Ptr<String>(l0).c()->data();
   std::string l1s = Ptr<String>(l1).c()->data();

game/runtime.cpp

@@ -355,20 +355,28 @@ RuntimeExitStatus exec_runtime(int argc, char** argv) {
   // TODO relegate this to its own function
   if (enable_display) {
     Gfx::Loop([]() { return MasterExit == RuntimeExitStatus::RUNNING; });
-    Gfx::Exit();
   }
 
   // hack to make the IOP die quicker if it's loading/unloading music
   gMusicFade = 0;
 
+  // if we have no display, wait here for DECI to shutdown
   deci_thread.join();
-  // DECI has been killed, shutdown!
+
+  // fully shut down EE first before stopping the other threads
+  ee_thread.join();
 
   // to be extra sure
   tm.shutdown();
 
   // join and exit
   tm.join();
+
+  // kill renderer after all threads are stopped.
+  // this makes sure the std::shared_ptr<Display> is destroyed in the main thread.
+  if (enable_display) {
+    Gfx::Exit();
+  }
   lg::info("GOAL Runtime Shutdown (code {})", MasterExit);
   munmap(g_ee_main_mem, EE_MAIN_MEM_SIZE);
   return MasterExit;