goalc/repl: Allow hot-loading files via ml with just the object name (#2036)

This allows you to not have to define the entire file path to a source
file to re-compile and load it. Technically a stop-gap until editor
tools are developed around writing OpenGOAL.


![image](https://user-images.githubusercontent.com/13153231/203196148-de61cf4b-42c8-43dc-a7fd-80e6ba6f5ac2.png)

As opposed to `(ml "goal_src/jak2/engine/game/main.gc")` (which still
works)

This is accomplished via the following config (connection attempts is
irrelevant):
```json
{
  "numConnectToTargetAttempts": 1,
  "jak2": {
    "asmFileSearchDirs": [
      "goal_src/jak2"
    ]
  }
}
```

This also provides a way to make game-specific configurations for the
REPL fairly easily.

common/CMakeLists.txt

@@ -46,6 +46,7 @@ add_library(common
         util/json_util.cpp
         util/read_iso_file.cpp
         util/SimpleThreadGroup.cpp
+        util/StringUtil.cpp
         util/Timer.cpp
         util/os.cpp
         util/print_float.cpp

common/goos/Reader.cpp

@@ -239,17 +239,9 @@ Object Reader::read_from_string(const std::string& str,
  * Read a file
  */
 Object Reader::read_from_file(const std::vector<std::string>& file_path, bool check_encoding) {
-  std::string joined_name;
+  std::string joined_path = fmt::format("{}", fmt::join(file_path, "/"));
 
-  for (const auto& thing : file_path) {
-    if (!joined_name.empty()) {
-      joined_name += '/';
-    }
-
-    joined_name += thing;
-  }
-
-  auto textFrag = std::make_shared<FileText>(file_util::get_file_path(file_path), joined_name);
+  auto textFrag = std::make_shared<FileText>(file_util::get_file_path(file_path), joined_path);
   db.insert(textFrag);
 
   auto result = internal_read(textFrag, check_encoding);

common/util/FileUtil.cpp

@@ -321,10 +321,23 @@ std::string base_name(const std::string& filename) {
       break;
     }
   }
-
   return filename.substr(pos);
 }
 
+std::string base_name_no_ext(const std::string& filename) {
+  size_t pos = 0;
+  ASSERT(!filename.empty());
+  for (size_t i = filename.size() - 1; i-- > 0;) {
+    if (filename.at(i) == '/' || filename.at(i) == '\\') {
+      pos = (i + 1);
+      break;
+    }
+  }
+  std::string file_name = filename.substr(pos);
+  return file_name.substr(0, file_name.find_last_of('.'));
+  ;
+}
+
 void ISONameFromAnimationName(char* dst, const char* src) {
   // The Animation Name is a bunch of words separated by dashes
 
@@ -529,7 +542,7 @@ std::vector<fs::path> find_files_recursively(const fs::path& base_dir, const std
   std::vector<fs::path> files = {};
   for (auto& p : fs::recursive_directory_iterator(base_dir)) {
     if (p.is_regular_file()) {
-      if (std::regex_match(fs::path(p.path()).filename().string(), pattern)) {
+      if (std::regex_match(p.path().filename().string(), pattern)) {
         files.push_back(p.path());
       }
     }

common/util/FileUtil.h

@@ -52,6 +52,7 @@ bool is_printable_char(char c);
 std::string combine_path(const std::string& parent, const std::string& child);
 bool file_exists(const std::string& path);
 std::string base_name(const std::string& filename);
+std::string base_name_no_ext(const std::string& filename);
 void MakeISOName(char* dst, const char* src);
 void ISONameFromAnimationName(char* dst, const char* src);
 void assert_file_exists(const char* path, const char* error_message);

common/util/StringUtil.cpp

@@ -0,0 +1,24 @@
+#include "StringUtil.h"
+
+namespace str_util {
+
+const std::string WHITESPACE = " \n\r\t\f\v";
+
+bool starts_with(const std::string& s, const std::string& prefix) {
+  return s.rfind(prefix) == 0;
+}
+
+std::string ltrim(const std::string& s) {
+  size_t start = s.find_first_not_of(WHITESPACE);
+  return (start == std::string::npos) ? "" : s.substr(start);
+}
+
+std::string rtrim(const std::string& s) {
+  size_t end = s.find_last_not_of(WHITESPACE);
+  return (end == std::string::npos) ? "" : s.substr(0, end + 1);
+}
+
+std::string trim(const std::string& s) {
+  return rtrim(ltrim(s));
+}
+}  // namespace str_util

common/util/StringUtil.h

@@ -0,0 +1,8 @@
+#include <string>
+
+namespace str_util {
+bool starts_with(const std::string& s, const std::string& prefix);
+std::string ltrim(const std::string& s);
+std::string rtrim(const std::string& s);
+std::string trim(const std::string& s);
+}  // namespace str_util

goalc/compiler/Compiler.cpp

@@ -67,8 +67,8 @@ Compiler::Compiler(GameVersion version,
     auto regex_colors = m_repl->regex_colors;
     m_repl->init_default_settings();
     using namespace std::placeholders;
-    m_repl->get_repl().set_completion_callback(
-        std::bind(&Compiler::find_symbols_by_prefix, this, _1, _2, std::cref(examples)));
+    m_repl->get_repl().set_completion_callback(std::bind(
+        &Compiler::find_symbols_or_object_file_by_prefix, this, _1, _2, std::cref(examples)));
     m_repl->get_repl().set_hint_callback(
         std::bind(&Compiler::find_hints_by_prefix, this, _1, _2, _3, std::cref(examples)));
     m_repl->get_repl().set_highlighter_callback(
@@ -382,14 +382,57 @@ void Compiler::setup_goos_forms() {
 }
 
 void Compiler::asm_file(const CompilationOptions& options) {
-  auto code = m_goos.reader.read_from_file({options.filename});
+  // If the filename provided is not a valid path but it's a name (with or without an extension)
+  // attempt to find it in the defined `asmFileSearchDirs`
+  //
+  // For example - (ml "process-drawable.gc")
+  // - This allows you to load a file without precisely defining the entire path
+  //
+  // If multiple candidates are found, abort
 
-  std::string obj_file_name = options.filename;
+  std::string file_name = options.filename;
+  std::string file_path = file_util::get_file_path({file_name});
+
+  if (!file_util::file_exists(file_path)) {
+    if (file_path.empty()) {
+      lg::print("ERROR - can't load a file without a providing a path\n");
+      return;
+    } else if (m_asm_file_search_dirs.empty()) {
+      lg::print(
+          "ERROR - can't load a file that doesn't exist - '{}' and no search dirs are defined\n",
+          file_path);
+      return;
+    }
+    std::string base_name = file_util::base_name_no_ext(file_path);
+    // Attempt the find the full path of the file (ignore extension)
+    std::vector<fs::path> candidate_paths = {};
+    for (const auto& dir : m_asm_file_search_dirs) {
+      std::string base_dir = file_util::get_file_path({dir});
+      const auto& results = file_util::find_files_recursively(
+          base_dir, std::regex(fmt::format("^{}(\\..*)?$", base_name)));
+      for (const auto& result : results) {
+        candidate_paths.push_back(result);
+      }
+    }
+    if (candidate_paths.empty()) {
+      lg::print("ERROR - attempt to find object file automatically, but found nothing\n");
+      return;
+    } else if (candidate_paths.size() > 1) {
+      lg::print("ERROR - attempt to find object file automatically, but found multiple\n");
+      return;
+    }
+    // Found the file!, use it!
+    file_path = candidate_paths.at(0).string();
+  }
+
+  auto code = m_goos.reader.read_from_file({file_path});
+
+  std::string obj_file_name = file_path;
 
   // Extract object name from file name.
-  for (int idx = int(options.filename.size()) - 1; idx-- > 0;) {
-    if (options.filename.at(idx) == '\\' || options.filename.at(idx) == '/') {
-      obj_file_name = options.filename.substr(idx + 1);
+  for (int idx = int(file_path.size()) - 1; idx-- > 0;) {
+    if (file_path.at(idx) == '\\' || file_path.at(idx) == '/') {
+      obj_file_name = file_path.substr(idx + 1);
       break;
     }
   }

goalc/compiler/Compiler.h

@@ -81,7 +81,8 @@ class Compiler {
   listener::Listener& listener() { return m_listener; }
   void poke_target() { m_listener.send_poke(); }
   bool connect_to_target();
-  Replxx::completions_t find_symbols_by_prefix(std::string const& context,
+  Replxx::completions_t find_symbols_or_object_file_by_prefix(
+      std::string const& context,
       int& contextLen,
       std::vector<std::string> const& user_data);
   Replxx::hints_t find_hints_by_prefix(std::string const& context,
@@ -93,7 +94,8 @@ class Compiler {
                      std::vector<std::pair<std::string, Replxx::Color>> const& user_data);
   bool knows_object_file(const std::string& name);
   MakeSystem& make_system() { return m_make; }
-  void update_via_config_file(const std::string& json);
+  void update_via_config_file(const std::string& json,
+                              const std::optional<std::string> game_name = {});
 
  private:
   GameVersion m_version;
@@ -114,7 +116,10 @@ class Compiler {
   SymbolInfoMap m_symbol_info;
   std::unique_ptr<ReplWrapper> m_repl;
   MakeSystem m_make;
+
+  // Configurable fields
   int m_target_connect_attempts = 30;
+  std::vector<std::string> m_asm_file_search_dirs = {};
 
   struct DebugStats {
     int num_spills = 0;

goalc/compiler/Env.cpp

@@ -108,6 +108,16 @@ FileEnv* GlobalEnv::add_file(std::string name) {
   return m_files.back().get();
 }
 
+std::vector<std::string> GlobalEnv::list_files_with_prefix(const std::string& prefix) {
+  std::vector<std::string> matches = {};
+  for (const auto& file : m_files) {
+    if (file->name().rfind(prefix) == 0) {
+      matches.push_back(file->name());
+    }
+  }
+  return matches;
+}
+
 ///////////////////
 // BlockEnv
 ///////////////////

goalc/compiler/Env.h

@@ -84,6 +84,8 @@ class GlobalEnv : public Env {
   ~GlobalEnv() = default;
 
   FileEnv* add_file(std::string name);
+  // TODO - consider refactoring to use a Trie
+  std::vector<std::string> list_files_with_prefix(const std::string& prefix);
 
  private:
   std::vector<std::unique_ptr<FileEnv>> m_files;

goalc/compiler/Util.cpp

@@ -154,11 +154,20 @@ bool Compiler::knows_object_file(const std::string& name) {
   return m_debugger.knows_object(name);
 }
 
-void Compiler::update_via_config_file(const std::string& json) {
+void Compiler::update_via_config_file(const std::string& json,
+                                      const std::optional<std::string> game_name) {
   auto cfg = parse_commented_json(json, "repl-config.json");
   if (cfg.contains("numConnectToTargetAttempts")) {
     m_target_connect_attempts = cfg.at("numConnectToTargetAttempts").get<int>();
   }
+  if (cfg.contains("asmFileSearchDirs")) {
+    m_asm_file_search_dirs = cfg.at("asmFileSearchDirs").get<std::vector<std::string>>();
+  }
+  // If there are any game specific config entries, set or override with them
+  if (game_name && cfg.contains(game_name.value())) {
+    auto game_cfg = cfg.at(game_name.value()).get<nlohmann::json>();
+    update_via_config_file(game_cfg.dump(), {});
+  }
 }
 
 /*!

goalc/compiler/compilation/CompilerControl.cpp

@@ -9,6 +9,7 @@
 #include "common/goos/ReplUtils.h"
 #include "common/util/DgoWriter.h"
 #include "common/util/FileUtil.h"
+#include "common/util/StringUtil.h"
 #include "common/util/Timer.h"
 
 #include "goalc/compiler/Compiler.h"
@@ -378,17 +379,42 @@ Val* Compiler::compile_get_info(const goos::Object& form, const goos::Object& re
   return get_none();
 }
 
-Replxx::completions_t Compiler::find_symbols_by_prefix(std::string const& context,
+Replxx::completions_t Compiler::find_symbols_or_object_file_by_prefix(
+    std::string const& context,
     int& contextLen,
     std::vector<std::string> const& user_data) {
   (void)contextLen;
   (void)user_data;
-  auto token = m_repl->get_current_repl_token(context);
-  auto possible_forms = lookup_symbol_infos_starting_with(token.first);
   Replxx::completions_t completions;
-  for (auto& x : possible_forms) {
-    completions.push_back(token.second ? "(" + x : x);
+
+  // If we are trying to execute a `(ml ...)` we can automatically get the object file
+  // insert quotes if needed as well.
+  if (str_util::starts_with(context, "(ml ")) {
+    std::string file_name_prefix = context.substr(4);
+    // Trim string just incase, extra whitespace is valid LISP
+    file_name_prefix = str_util::trim(file_name_prefix);
+    // Remove quotes
+    file_name_prefix.erase(remove(file_name_prefix.begin(), file_name_prefix.end(), '"'),
+                           file_name_prefix.end());
+    if (file_name_prefix.empty()) {
+      return completions;
     }
+
+    // Get all the potential object file names
+    const auto& matches = m_global_env->list_files_with_prefix(file_name_prefix);
+    for (const auto& match : matches) {
+      completions.push_back(fmt::format("\"{}\")", match));
+    }
+  } else {
+    const auto [token, stripped_leading_paren] = m_repl->get_current_repl_token(context);
+    // Otherwise, look for symbols
+    auto possible_forms = lookup_symbol_infos_starting_with(token);
+
+    for (auto& x : possible_forms) {
+      completions.push_back(stripped_leading_paren ? "(" + x : x);
+    }
+  }
+
   return completions;
 }
 
@@ -403,6 +429,8 @@ Replxx::hints_t Compiler::find_hints_by_prefix(std::string const& context,
 
   Replxx::hints_t hints;
 
+  // TODO - hints for `(ml ...` as well
+
   // Only show hints if there are <= 3 possibilities
   if (possible_forms.size() <= 3) {
     for (auto& x : possible_forms) {

goalc/main.cpp

@@ -134,7 +134,7 @@ int main(int argc, char** argv) {
     }
     compiler = std::make_unique<Compiler>(game_version, username, std::make_unique<ReplWrapper>());
     if (repl_config) {
-      compiler->update_via_config_file(repl_config.value());
+      compiler->update_via_config_file(repl_config.value(), game);
     }
     // Start nREPL Server
     if (repl_server_ok) {
@@ -180,7 +180,7 @@ int main(int argc, char** argv) {
         compiler =
             std::make_unique<Compiler>(game_version, username, std::make_unique<ReplWrapper>());
         if (repl_config) {
-          compiler->update_via_config_file(repl_config.value());
+          compiler->update_via_config_file(repl_config.value(), game);
         }
         if (!startup_cmd.empty()) {
           compiler->handle_repl_string(startup_cmd);