Fix UTF-8 handling when running the game, env-vars, and setting the project path (#1632)

* fix utf-8 handling around env-vars

* fix file opening errors related to unicode

* add uncaught exception handler in `gk` to ensure something is logged

* gracefully fail if window icon cant be loaded and work with unicode

* linux fix and add changes to vendor file

.vs/launch.vs.json

@@ -168,8 +168,8 @@
       "type": "default",
       "project": "CMakeLists.txt",
       "projectTarget": "extractor.exe (bin\\extractor.exe)",
-      "name": "Tools - Extractor - Extract",
+      "name": "Tools - Extractor - Full",
-      "args": ["E:\\ISOs\\Jak\\Jak 1.iso"]
+      "args": ["\"E:\\ISOs\\Jak\\Jak 1.iso\""]
     }
   ]
 }

common/CMakeLists.txt

@@ -49,7 +49,8 @@ add_library(common
         util/os.cpp
         util/print_float.cpp
         util/FontUtils.cpp
-        util/FrameLimiter.cpp "util/unicode_util.h" "util/unicode_util.cpp")
+        util/FrameLimiter.cpp
+        util/unicode_util.cpp)
 
 target_link_libraries(common fmt lzokay replxx libzstd_static)
 

common/goos/Interpreter.cpp

@@ -10,6 +10,7 @@
 #include "ParseHelpers.h"
 
 #include "common/util/FileUtil.h"
+#include "common/util/unicode_util.h"
 
 #include "third-party/fmt/core.h"
 
@@ -1682,8 +1683,8 @@ Object Interpreter::eval_get_env(const Object& form,
                                  const std::shared_ptr<EnvironmentObject>&) {
   vararg_check(form, args, {ObjectType::STRING}, {{"default", {false, ObjectType::STRING}}});
   const std::string var_name = args.unnamed.at(0).as_string()->data;
-  const char* env_p = std::getenv(var_name.c_str());
+  auto env_p = get_env(var_name);
-  if (env_p == NULL) {
+  if (env_p.empty()) {
     if (args.has_named("default")) {
       return args.get_named("default");
     } else {

common/util/FileUtil.cpp

@@ -15,6 +15,7 @@
 
 #include "common/common_types.h"
 #include "common/util/BinaryReader.h"
+#include "common/util/unicode_util.h"
 
 // This disables the use of PCLMULQDQ which is probably ok, but let's just be safe and disable it
 // because nobody will care if png compression is 10% slower.
@@ -39,10 +40,10 @@ namespace file_util {
 fs::path get_user_home_dir() {
 #ifdef _WIN32
   // NOTE - on older systems, this may case issues if it cannot be found!
-  std::string home_dir = std::getenv("USERPROFILE");
+  std::string home_dir = get_env("USERPROFILE");
   return fs::path(home_dir);
 #else
-  std::string home_dir = std::getenv("HOME");
+  std::string home_dir = get_env("HOME");
   return fs::path(home_dir);
 #endif
 }
@@ -50,16 +51,16 @@ fs::path get_user_home_dir() {
 fs::path get_user_config_dir() {
   fs::path config_base_path;
 #ifdef _WIN32
-  auto config_base_dir = std::getenv("APPDATA");
+  auto config_base_dir = get_env("APPDATA");
-  config_base_path = fs::path(std::string(config_base_dir));
+  config_base_path = fs::path(config_base_dir);
 #elif __linux
   // Docs - https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html
   // Prefer XDG_CONFIG_HOME if available
-  auto config_base_dir = std::getenv("XDG_CONFIG_HOME");
+  auto config_base_dir = get_env("XDG_CONFIG_HOME");
-  if (!config_base_dir) {
+  if (config_base_dir.empty()) {
     config_base_path = get_user_home_dir() / ".config";
   } else {
-    config_base_path = std::string(config_base_dir);
+    config_base_path = fs::path(config_base_dir);
   }
 #endif
   return config_base_path / "OpenGOAL";
@@ -85,9 +86,10 @@ struct {
  */
 std::string get_current_executable_path() {
 #ifdef _WIN32
-  char buffer[FILENAME_MAX];
+  // NOTE - MAX_PATH is kinda wrong here as you can have a path longer than 260 in windows
-  GetModuleFileNameA(NULL, buffer, FILENAME_MAX);
+  wchar_t path[MAX_PATH];
-  std::string file_path(buffer);
+  GetModuleFileNameW(NULL, path, MAX_PATH);
+  std::string file_path = wide_string_to_utf8_string(path);
   if (file_path.rfind("\\\\?\\", 0) == 0) {
     return file_path.substr(4);
   }
@@ -280,7 +282,7 @@ std::vector<uint8_t> read_binary_file(const fs::path& path) {
 }
 
 std::string read_text_file(const fs::path& path) {
-  std::ifstream file(path.string());
+  fs::ifstream file(path);
   if (!file.good()) {
     throw std::runtime_error("couldn't open " + path.string());
   }

common/util/unicode_util.cpp

@@ -14,48 +14,66 @@
 #endif
 // clang-format on
 
-std::string utf8_from_utf16(const wchar_t* utf16_string) {
 #ifdef _WIN32
-  if (utf16_string == nullptr) {
+std::wstring utf8_string_to_wide_string(const std::string_view& str) {
-    return std::string();
+  std::wstring ret;
-  }
+  if (!utf8_string_to_wide_string(ret, str))
-  int target_length = WideCharToMultiByte(CP_UTF8, WC_ERR_INVALID_CHARS, utf16_string, -1, nullptr,
-                                          0, nullptr, nullptr);
-  if (target_length == 0) {
-    return std::string();
-  }
-  std::string utf8_string;
-  utf8_string.resize(target_length);
-  int converted_length = WideCharToMultiByte(CP_UTF8, WC_ERR_INVALID_CHARS, utf16_string, -1,
-                                             utf8_string.data(), target_length, nullptr, nullptr);
-  if (converted_length == 0) {
-    return std::string();
-  }
-  return utf8_string;
-#else
-  return "don't call this on linux";
-#endif
-}
-
-std::vector<std::string> get_widechar_cli_args() {
-#ifdef _WIN32
-  // Convert the UTF-16 command line arguments to UTF-8 for the Engine to use.
-  int argc;
-  wchar_t** argv = CommandLineToArgvW(GetCommandLineW(), &argc);
-  if (argv == nullptr) {
     return {};
+
+  return ret;
+}
+
+bool utf8_string_to_wide_string(std::wstring& dest, const std::string_view& str) {
+  int wlen =
+      MultiByteToWideChar(CP_UTF8, 0, str.data(), static_cast<int>(str.length()), nullptr, 0);
+  if (wlen < 0)
+    return false;
+
+  dest.resize(wlen);
+  if (wlen > 0 && MultiByteToWideChar(CP_UTF8, 0, str.data(), static_cast<int>(str.length()),
+                                      dest.data(), wlen) < 0)
+    return false;
+
+  return true;
+}
+
+std::string wide_string_to_utf8_string(const std::wstring_view& str) {
+  std::string ret;
+  if (!wide_string_to_utf8_string(ret, str))
+    ret.clear();
+
+  return ret;
+}
+
+bool wide_string_to_utf8_string(std::string& dest, const std::wstring_view& str) {
+  int mblen = WideCharToMultiByte(CP_UTF8, 0, str.data(), static_cast<int>(str.length()), nullptr,
+                                  0, nullptr, nullptr);
+  if (mblen < 0)
+    return false;
+
+  dest.resize(mblen);
+  if (mblen > 0 && WideCharToMultiByte(CP_UTF8, 0, str.data(), static_cast<int>(str.length()),
+                                       dest.data(), mblen, nullptr, nullptr) < 0) {
+    return false;
   }
 
-  std::vector<std::string> command_line_arguments;
+  return true;
+}
+#endif
 
-  for (int i = 0; i < argc; i++) {
+std::string get_env(const std::string& name) {
-    command_line_arguments.push_back(utf8_from_utf16(argv[i]));
+#ifdef _WIN32
+  auto name_wide = utf8_string_to_wide_string(name);
+  auto val = _wgetenv(name_wide.data());
+  if (!val) {
+    return "";
   }
-
+  return wide_string_to_utf8_string(val);
-  LocalFree(argv);
-
-  return command_line_arguments;
 #else
-  return {"don't call this on linux"};
+  auto val = std::getenv(name.data());
+  if (!val) {
+    return "";
+  }
+  return val;
 #endif
 }

common/util/unicode_util.h

@@ -3,5 +3,11 @@
 #include <string>
 #include <vector>
 
-std::string utf8_from_utf16(const wchar_t* utf16_string);
+#ifdef _WIN32
-std::vector<std::string> get_widechar_cli_args();
+std::wstring utf8_string_to_wide_string(const std::string_view& str);
+bool utf8_string_to_wide_string(std::wstring& dest, const std::string_view& str);
+std::string wide_string_to_utf8_string(const std::wstring_view& str);
+bool wide_string_to_utf8_string(std::string& dest, const std::wstring_view& str);
+#endif
+
+std::string get_env(const std::string& name);

decompiler/data/TextureDB.cpp

@@ -3,6 +3,7 @@
 #include "common/util/Assert.h"
 
 #include "third-party/fmt/core.h"
+#define STBI_WINDOWS_UTF8
 #include "third-party/stb_image/stb_image.h"
 
 namespace decompiler {

decompiler/extractor/main.cpp

@@ -234,14 +234,11 @@ int main(int argc, char** argv) {
   bool flag_folder = false;
   std::string game_name = "jak1";
 
-#ifdef _WIN32
+  fs::u8arguments u8guard(argc, argv);
-  auto args = get_widechar_cli_args();
+  if (!u8guard.valid()) {
-  std::vector<char*> string_ptrs;
+    lg::error("Bad encoding, needs UTF-8");
-  for (auto& str : args) {
+    exit(EXIT_FAILURE);
-    string_ptrs.push_back(str.data());
   }
-  argv = string_ptrs.data();
-#endif
 
   lg::initialize();
 
@@ -267,7 +264,7 @@ int main(int argc, char** argv) {
 
   // If no flag is set, we default to running everything
   if (!flag_extract && !flag_decompile && !flag_compile && !flag_play) {
-    fmt::print("Running all steps, no flags provided!\n");
+    lg::info("Running all steps, no flags provided!");
     flag_runall = true;
   }
   if (flag_runall) {
@@ -284,11 +281,22 @@ int main(int argc, char** argv) {
       lg::error("Error: project path override '{}' does not exist", project_path_override.string());
       return static_cast<int>(ExtractorErrorCode::INVALID_CLI_INPUT);
     }
-    file_util::setup_project_path(project_path_override);
+    auto ok = file_util::setup_project_path(project_path_override);
+    if (!ok) {
+      lg::error("Could not setup project path!");
+      return 1;
+    }
   } else {
-    file_util::setup_project_path({});
+    auto ok = file_util::setup_project_path({});
+    if (!ok) {
+      lg::error("Could not setup project path!");
+      return 1;
+    }
   }
 
+  auto log_path = file_util::get_jak_project_dir() / "extractor.log";
+  lg::set_file(log_path.string());
+
   fs::path iso_data_path;
 
   // - INPUT VALIDATION

decompiler/main.cpp

@@ -18,14 +18,10 @@
 #include "decompiler/level_extractor/extract_level.h"
 
 int main(int argc, char** argv) {
-#ifdef _WIN32
+  fs::u8arguments u8guard(argc, argv);
-  auto args = get_widechar_cli_args();
+  if (!u8guard.valid()) {
-  std::vector<char*> string_ptrs;
+    exit(EXIT_FAILURE);
-  for (auto& str : args) {
-    string_ptrs.push_back(str.data());
   }
-  argv = string_ptrs.data();
-#endif
 
   Timer decomp_timer;
 

game/graphics/pipelines/opengl.cpp

@@ -29,6 +29,7 @@
 #include "third-party/imgui/imgui.h"
 #include "third-party/imgui/imgui_impl_glfw.h"
 #include "third-party/imgui/imgui_impl_opengl3.h"
+#define STBI_WINDOWS_UTF8
 #include "third-party/stb_image/stb_image.h"
 
 namespace {
@@ -182,43 +183,14 @@ static std::shared_ptr<GfxDisplay> gl_make_display(int width,
       (file_util::get_jak_project_dir() / "game" / "assets" / "appicon.png").string();
 
   GLFWimage images[1];
-  images[0].pixels =
+  auto load_result = stbi_load(image_path.c_str(), &images[0].width, &images[0].height, 0, 4);
-      stbi_load(image_path.c_str(), &images[0].width, &images[0].height, 0, 4);  // rgba channels
+  if (load_result) {
-  glfwSetWindowIcon(window, 1, images);
+    images[0].pixels = load_result;  // rgba channels
-  stbi_image_free(images[0].pixels);
+    glfwSetWindowIcon(window, 1, images);
-
+    stbi_image_free(images[0].pixels);
-  // init framerate settings
-  /*
-  if (GLFWmonitor* primary_monitor = glfwGetPrimaryMonitor()) {
-    auto primary_monitor_video_mode = glfwGetVideoMode(primary_monitor);
-
-    if (primary_monitor_video_mode && primary_monitor_video_mode->refreshRate > 60) {
-      // Use the framelimiter by default and disable vsync
-      Gfx::g_global_settings.framelimiter = true;
-      Gfx::g_global_settings.vsync = false;
-      glfwSwapInterval(0);
-      if (primary_monitor_video_mode->refreshRate > 100) {
-        BootVideoMode = VideoMode::FPS150;
-        Gfx::g_global_settings.target_fps = 150;
-      } else if (primary_monitor_video_mode->refreshRate > 60) {
-        BootVideoMode = VideoMode::FPS100;
-        Gfx::g_global_settings.target_fps = 100;
-      }
-    } else {
-      // enable vsync
-      Gfx::g_global_settings.framelimiter = false;
-      Gfx::g_global_settings.vsync = true;
-      // glfwSwapInterval(1);
-      glfwSwapInterval(settings.vsync);
-    }
   } else {
-    // enable vsync
+    lg::error("Could not load icon for OpenGL window");
-    Gfx::g_global_settings.framelimiter = false;
-    Gfx::g_global_settings.vsync = true;
-    // glfwSwapInterval(1);
-    glfwSwapInterval(settings.vsync);
   }
-  */
 
   SetDisplayCallbacks(window);
   Pad::initialize();

game/main.cpp

@@ -3,6 +3,8 @@
  * Main for the game. Launches the runtime.
  */
 
+#define STBI_WINDOWS_UTF8
+
 #include <string>
 
 #include "runtime.h"
@@ -40,14 +42,13 @@ void setup_logging(bool verbose) {
  * Entry point for the game.
  */
 int main(int argc, char** argv) {
-#ifdef _WIN32
+  fs::u8arguments u8guard(argc, argv);
-  auto args = get_widechar_cli_args();
+  if (!u8guard.valid()) {
-  std::vector<char*> string_ptrs;
+    exit(EXIT_FAILURE);
-  for (auto& str : args) {
-    string_ptrs.push_back(str.data());
   }
-  argv = string_ptrs.data();
+
-#endif
+  // TODO - replace with CLI11 and just propagate args through
+  // - https://github.com/CLIUtils/CLI11/issues/744
 
   // Figure out if the CPU has AVX2 to enable higher performance AVX2 versions of functions.
   setup_cpu_info();
@@ -125,17 +126,21 @@ int main(int argc, char** argv) {
 
     // run the runtime in a loop so we can reset the game and have it restart cleanly
     lg::info("OpenGOAL Runtime {}.{}", versions::GOAL_VERSION_MAJOR, versions::GOAL_VERSION_MINOR);
-    auto exit_status = exec_runtime(ptrs.size(), ptrs.data());
+    try {
-
+      auto exit_status = exec_runtime(ptrs.size(), ptrs.data());
-    switch (exit_status) {
+      switch (exit_status) {
-      case RuntimeExitStatus::EXIT:
+        case RuntimeExitStatus::EXIT:
-        return 0;
+          return 0;
-      case RuntimeExitStatus::RESTART_RUNTIME:
+        case RuntimeExitStatus::RESTART_RUNTIME:
-      case RuntimeExitStatus::RUNNING:
+        case RuntimeExitStatus::RUNNING:
-        break;
+          break;
-      case RuntimeExitStatus::RESTART_IN_DEBUG:
+        case RuntimeExitStatus::RESTART_IN_DEBUG:
-        force_debug_next_time = true;
+          force_debug_next_time = true;
-        break;
+          break;
+      }
+    } catch (std::exception& ex) {
+      lg::error("Unexpected exception occurred - {}", ex.what());
+      throw ex;
     }
   }
   return 0;

test/offline/offline_test_main.cpp

@@ -389,14 +389,11 @@ std::optional<OfflineTestConfig> parse_config(const std::string_view& game_name)
 }
 
 int main(int argc, char* argv[]) {
-#ifdef _WIN32
+  fs::u8arguments u8guard(argc, argv);
-  auto utf8_args = get_widechar_cli_args();
+  if (!u8guard.valid()) {
-  std::vector<char*> string_ptrs;
+    std::cerr << "Bad encoding, needs UTF-8." << std::endl;
-  for (auto& str : utf8_args) {
+    exit(EXIT_FAILURE);
-    string_ptrs.push_back(str.data());
   }
-  argv = string_ptrs.data();
-#endif
 
   lg::initialize();
 

test/test_main.cpp

@@ -19,14 +19,11 @@
 // to make it easier to test a subset of tests
 
 int main(int argc, char** argv) {
-#ifdef _WIN32
+  fs::u8arguments u8guard(argc, argv);
-  auto args = get_widechar_cli_args();
+  if (!u8guard.valid()) {
-  std::vector<char*> string_ptrs;
+    std::cerr << "Bad encoding, needs UTF-8." << std::endl;
-  for (auto& str : args) {
+    exit(EXIT_FAILURE);
-    string_ptrs.push_back(str.data());
   }
-  argv = string_ptrs.data();
-#endif
 
   // hopefully get a debug print on github actions
   setup_cpu_info();

third-party/fpng/fpng.cpp

@@ -1707,6 +1707,31 @@ do_literals:
 	}
 
 #ifndef FPNG_NO_STDIO
+#ifdef _WIN32
+        bool utf8_string_to_wide_string(std::wstring& dest, const std::string& str) {
+          int wlen = MultiByteToWideChar(CP_UTF8, 0, str.data(), static_cast<int>(str.length()),
+                                         nullptr, 0);
+          if (wlen < 0)
+            return false;
+
+          dest.resize(wlen);
+          if (wlen > 0 &&
+              MultiByteToWideChar(CP_UTF8, 0, str.data(), static_cast<int>(str.length()),
+                                  dest.data(), wlen) < 0)
+            return false;
+
+          return true;
+        }
+
+        std::wstring utf8_string_to_wide_string(const std::string& str) {
+          std::wstring ret;
+          if (!utf8_string_to_wide_string(ret, str))
+            return {};
+
+          return ret;
+        }
+#endif
+
 	bool fpng_encode_image_to_file(const char* pFilename, const void* pImage, uint32_t w, uint32_t h, uint32_t num_chans, uint32_t flags)
 	{
 		std::vector<uint8_t> out_buf;
@@ -1715,7 +1740,14 @@ do_literals:
 
 		FILE* pFile = nullptr;
 #ifdef _MSC_VER
-		fopen_s(&pFile, pFilename, "wb");
+        // NOTE - Manual fix by us to support unicode....
+        std::wstring converted_path = utf8_string_to_wide_string(pFilename);
+        if (converted_path.empty()) {
+          printf("bad path - %s", converted_path.data());
+          return false;
+        }
+
+        pFile = _wfopen(converted_path.data(), L"wb");
 #else
 		pFile = fopen(pFilename, "wb");
 #endif

third-party/stb_image/stb_image.cpp

@@ -1,4 +1,5 @@
+#define STBI_WINDOWS_UTF8
 #define STB_IMAGE_IMPLEMENTATION
 #define STB_IMAGE_WRITE_IMPLEMENTATION
 #include "stb_image_write.h"
-#include "stb_image.h"
+#include "stb_image.h"

third-party/tiny_gltf/tiny_gltf.h

@@ -1554,6 +1554,7 @@ class TinyGLTF {
 
 #ifndef TINYGLTF_NO_STB_IMAGE
 #ifndef TINYGLTF_NO_INCLUDE_STB_IMAGE
+#define STBI_WINDOWS_UTF8
 #include "stb_image.h"
 #endif
 #endif

tools/MemoryDumpTool/main.cpp

@@ -558,14 +558,10 @@ void inspect_symbols(const Ram& ram,
 }
 
 int main(int argc, char** argv) {
-#ifdef _WIN32
+  fs::u8arguments u8guard(argc, argv);
-  auto args = get_widechar_cli_args();
+  if (!u8guard.valid()) {
-  std::vector<char*> string_ptrs;
+    exit(EXIT_FAILURE);
-  for (auto& str : args) {
-    string_ptrs.push_back(str.data());
   }
-  argv = string_ptrs.data();
-#endif
 
   fmt::print("MemoryDumpTool\n");
 

tools/dgo_packer.cpp

@@ -6,14 +6,10 @@
 #include <common/util/unicode_util.h>
 
 int main(int argc, char** argv) {
-#ifdef _WIN32
+  fs::u8arguments u8guard(argc, argv);
-  auto args = get_widechar_cli_args();
+  if (!u8guard.valid()) {
-  std::vector<char*> string_ptrs;
+    exit(EXIT_FAILURE);
-  for (auto& str : args) {
-    string_ptrs.push_back(str.data());
   }
-  argv = string_ptrs.data();
-#endif
 
   printf("OpenGOAL version %d.%d\n", versions::GOAL_VERSION_MAJOR, versions::GOAL_VERSION_MINOR);
   printf("DGO Packing Tool\n");

tools/dgo_unpacker.cpp

@@ -49,14 +49,10 @@ int run(int argc, char** argv) {
 }  // namespace
 
 int main(int argc, char** argv) {
-#ifdef _WIN32
+  fs::u8arguments u8guard(argc, argv);
-  auto args = get_widechar_cli_args();
+  if (!u8guard.valid()) {
-  std::vector<char*> string_ptrs;
+    exit(EXIT_FAILURE);
-  for (auto& str : args) {
-    string_ptrs.push_back(str.data());
   }
-  argv = string_ptrs.data();
-#endif
 
   try {
     return run(argc, argv);

tools/level_tools/level_dump/main.cpp

@@ -56,14 +56,10 @@ bool is_valid_bsp(const decompiler::LinkedObjectFile& file) {
 }
 
 int main(int argc, char** argv) {
-#ifdef _WIN32
+  fs::u8arguments u8guard(argc, argv);
-  auto args = get_widechar_cli_args();
+  if (!u8guard.valid()) {
-  std::vector<char*> string_ptrs;
+    exit(EXIT_FAILURE);
-  for (auto& str : args) {
-    string_ptrs.push_back(str.data());
   }
-  argv = string_ptrs.data();
-#endif
 
   try {
     fmt::print("Level Dump Tool\n");

vendor.yaml

@@ -8,9 +8,17 @@ third-party/discord-rpc:
   sha: 963aa9f3e5ce81a4682c6ca3d136cddda614db33
 third-party/fpng:
   sha: bfe5f9c69e93b99b31268c10db8e645c9125a07f
+  modifications:
+    - "PR #1632 - Fixes Unicode path saving on windows"
 third-party/CLI11.hpp:
   git: https://github.com/CLIUtils/CLI11/tree/v2.2.0
 third-party/xxhash.hpp:
   git: https://github.com/RedSpah/xxhash_cpp/tree/0.7.3
 third-party/xdelta3:
   sha: 7508fd2a823443b1f0173ca361620f21d62a7d37
+third-party/stb_image:
+  modifications:
+    - "PR #1632 - Uses UTF-8 on windows"
+third-party/tiny_gltf:
+  modifications:
+    - "PR #1632 - Ensure stb_image is using UTF-8 on windows"