Support "game count" and v4 objects (#140)

* generate object, but not supported in linker yet

* add link and tests

* update types

boot_game.sh

@@ -3,4 +3,4 @@
 # Directory of this script
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 
-$DIR/build/game/gk -fakeiso -debug
+$DIR/build/game/gk -boot -fakeiso -debug "$@"

common/goos/CMakeLists.txt

@@ -5,5 +5,5 @@ ELSE()
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3")
 ENDIF()
 
-add_library(goos SHARED Object.cpp TextDB.cpp Reader.cpp Interpreter.cpp PrettyPrinter.cpp)
+add_library(goos SHARED Object.cpp ParseHelpers.cpp TextDB.cpp Reader.cpp Interpreter.cpp PrettyPrinter.cpp ParseHelpers.cpp ParseHelpers.h)
 target_link_libraries(goos common_util fmt)

common/goos/Interpreter.cpp

@@ -5,6 +5,7 @@
 
 #include <utility>
 #include "Interpreter.h"
+#include "ParseHelpers.h"
 #include <third-party/fmt/core.h>
 
 namespace goos {
@@ -381,44 +382,9 @@ void Interpreter::vararg_check(
     const Arguments& args,
     const std::vector<std::optional<ObjectType>>& unnamed,
     const std::unordered_map<std::string, std::pair<bool, std::optional<ObjectType>>>& named) {
-  assert(args.rest.empty());
-  if (unnamed.size() != args.unnamed.size()) {
-    throw_eval_error(form, "Got " + std::to_string(args.unnamed.size()) +
-                               " arguments, but expected " + std::to_string(unnamed.size()));
-  }
-
-  for (size_t i = 0; i < unnamed.size(); i++) {
-    if (unnamed[i].has_value() && unnamed[i] != args.unnamed[i].type) {
-      throw_eval_error(form, "Argument " + std::to_string(i) + " has type " +
-                                 object_type_to_string(args.unnamed[i].type) + " but " +
-                                 object_type_to_string(unnamed[i].value()) + " was expected");
-    }
-  }
-
-  for (const auto& kv : named) {
-    auto kv2 = args.named.find(kv.first);
-    if (kv2 == args.named.end()) {
-      // argument not given.
-      if (kv.second.first) {
-        // but was required
-        throw_eval_error(form, "Required named argument \"" + kv.first + "\" was not found");
-      }
-    } else {
-      // argument given.
-      if (kv.second.second.has_value() && kv.second.second != kv2->second.type) {
-        // but is wrong type
-        throw_eval_error(form, "Argument \"" + kv.first + "\" has type " +
-                                   object_type_to_string(kv2->second.type) + " but " +
-                                   object_type_to_string(kv.second.second.value()) +
-                                   " was expected");
-      }
-    }
-  }
-
-  for (const auto& kv : args.named) {
-    if (named.find(kv.first) == named.end()) {
-      throw_eval_error(form, "Got unrecognized keyword argument \"" + kv.first + "\"");
-    }
+  std::string err;
+  if (!va_check(args, unnamed, named, &err)) {
+    throw_eval_error(form, err);
   }
 }
 

common/goos/ParseHelpers.cpp

@@ -0,0 +1,101 @@
+#include "ParseHelpers.h"
+
+namespace goos {
+
+bool get_va(const goos::Object& rest, std::string* err_string, goos::Arguments* result) {
+  goos::Arguments args;
+  // loop over forms in list
+  goos::Object current = rest;
+  while (!current.is_empty_list()) {
+    auto arg = current.as_pair()->car;
+
+    // did we get a ":keyword"
+    if (arg.is_symbol() && arg.as_symbol()->name.at(0) == ':') {
+      auto key_name = arg.as_symbol()->name.substr(1);
+
+      // check for multiple definition of key
+      if (args.named.find(key_name) != args.named.end()) {
+        *err_string = "Key argument " + key_name + " multiply defined";
+        return false;
+      }
+
+      // check for well-formed :key value expression
+      current = current.as_pair()->cdr;
+      if (current.is_empty_list()) {
+        *err_string = "Key argument didn't have a value";
+        return false;
+      }
+
+      args.named[key_name] = current.as_pair()->car;
+    } else {
+      // not a keyword. Add to unnamed or rest, depending on what we expect
+      args.unnamed.push_back(arg);
+    }
+    current = current.as_pair()->cdr;
+  }
+  *result = args;
+  return true;
+}
+
+bool va_check(
+    const goos::Arguments& args,
+    const std::vector<std::optional<goos::ObjectType>>& unnamed,
+    const std::unordered_map<std::string, std::pair<bool, std::optional<goos::ObjectType>>>& named,
+    std::string* err_string) {
+  assert(args.rest.empty());
+  if (unnamed.size() != args.unnamed.size()) {
+    *err_string = "Got " + std::to_string(args.unnamed.size()) + " arguments, but expected " +
+                  std::to_string(unnamed.size());
+    return false;
+  }
+
+  for (size_t i = 0; i < unnamed.size(); i++) {
+    if (unnamed[i].has_value() && unnamed[i] != args.unnamed[i].type) {
+      *err_string = "Argument " + std::to_string(i) + " has type " +
+                    object_type_to_string(args.unnamed[i].type) + " but " +
+                    object_type_to_string(unnamed[i].value()) + " was expected";
+      return false;
+    }
+  }
+
+  for (const auto& kv : named) {
+    auto kv2 = args.named.find(kv.first);
+    if (kv2 == args.named.end()) {
+      // argument not given.
+      if (kv.second.first) {
+        // but was required
+        *err_string = "Required named argument \"" + kv.first + "\" was not found";
+        return false;
+      }
+    } else {
+      // argument given.
+      if (kv.second.second.has_value() && kv.second.second != kv2->second.type) {
+        // but is wrong type
+        *err_string = "Argument \"" + kv.first + "\" has type " +
+                      object_type_to_string(kv2->second.type) + " but " +
+                      object_type_to_string(kv.second.second.value()) + " was expected";
+        return false;
+      }
+    }
+  }
+
+  for (const auto& kv : args.named) {
+    if (named.find(kv.first) == named.end()) {
+      *err_string = "Got unrecognized keyword argument \"" + kv.first + "\"";
+      return false;
+    }
+  }
+
+  return true;
+}
+
+int list_length(const goos::Object& list) {
+  int len = 0;
+  for_each_in_list(list, [&](const goos::Object& x) {
+    (void)x;
+    len++;
+  });
+  return len;
+}
+
+}  // namespace goos

common/goos/ParseHelpers.h

@@ -0,0 +1,30 @@
+#pragma once
+
+#include <optional>
+#include <string>
+#include <vector>
+#include <unordered_map>
+#include "Object.h"
+
+namespace goos {
+bool get_va(const goos::Object& rest, std::string* err_string, goos::Arguments* result);
+bool va_check(
+    const goos::Arguments& args,
+    const std::vector<std::optional<goos::ObjectType>>& unnamed,
+    const std::unordered_map<std::string, std::pair<bool, std::optional<goos::ObjectType>>>& named,
+    std::string* err_string);
+
+template <typename T>
+void for_each_in_list(const goos::Object& list, T f) {
+  const goos::Object* iter = &list;
+  while (iter->is_pair()) {
+    auto lap = iter->as_pair();
+    f(lap->car);
+    iter = &lap->cdr;
+  }
+
+  assert(iter->is_empty_list());
+}
+
+int list_length(const goos::Object& list);
+}  // namespace goos

common/link_types.h

@@ -44,4 +44,12 @@ struct LinkHeaderV2 {
   uint32_t version;   // always 2
 };
 
+// Header for link data used for V4
+struct LinkHeaderV4 {
+  uint32_t type_tag;   // always -1
+  uint32_t length;     // length of V2 link data found after object.
+  uint32_t version;    // always 4
+  uint32_t code_size;  // length of object data before link data starts
+};
+
 #endif  // JAK1_LINK_TYPES_H

decompiler/CMakeLists.txt

@@ -21,7 +21,8 @@ add_executable(decompiler
 		Function/TypeInspector.cpp
 		data/tpage.cpp
 		data/game_text.cpp
-		data/StrFileReader.cpp)
+		data/StrFileReader.cpp
+		data/game_count.cpp data/LinkedWordReader.h)
 
 target_link_libraries(decompiler
         goos

decompiler/ObjectFile/LinkedObjectFileCreation.cpp

@@ -27,14 +27,6 @@ struct LinkHeaderCommon {
   uint16_t version;   // what version (2, 3, 4)
 };
 
-// Header for link data used for V4
-struct LinkHeaderV4 {
-  uint32_t type_tag;   // always -1
-  uint32_t length;     // length of V2 link data found after object.
-  uint32_t version;    // always 4
-  uint32_t code_size;  // length of object data before link data starts
-};
-
 // Per-segment info for V3 and V5 link data
 struct SegmentInfo {
   uint32_t relocs;  // offset of relocation table

decompiler/ObjectFile/ObjectFileDB.cpp

@@ -13,6 +13,7 @@
 #include "decompiler/data/tpage.h"
 #include "decompiler/data/game_text.h"
 #include "decompiler/data/StrFileReader.h"
+#include "decompiler/data/game_count.h"
 #include "LinkedObjectFileCreation.h"
 #include "decompiler/config.h"
 #include "third-party/minilzo/minilzo.h"
@@ -715,6 +716,26 @@ std::string ObjectFileDB::process_game_text() {
   return write_game_text(text_by_language_by_id);
 }
 
+std::string ObjectFileDB::process_game_count() {
+  spdlog::info("- Finding game count file...");
+  bool found = false;
+  std::string result;
+
+  for_each_obj([&](ObjectFileData& data) {
+    if (data.name_in_dgo == "game-cnt") {
+      assert(!found);
+      found = true;
+      result = write_game_count(::process_game_count(data));
+    }
+  });
+
+  if (!found) {
+    spdlog::warn("did not find game-cnt file");
+  }
+
+  return result;
+}
+
 void ObjectFileDB::analyze_functions() {
   spdlog::info("- Analyzing Functions...");
   Timer timer;
@@ -916,3 +937,10 @@ void ObjectFileDB::analyze_functions() {
   //      }
   //    }
 }
+
+void ObjectFileDB::dump_raw_objects(const std::string& output_dir) {
+  for_each_obj([&](ObjectFileData& data) {
+    auto dest = output_dir + "/" + data.to_unique_name();
+    file_util::write_binary_file(dest, data.data.data(), data.data.size());
+  });
+}

decompiler/ObjectFile/ObjectFileDB.h

@@ -55,11 +55,13 @@ class ObjectFileDB {
   void process_labels();
   void find_code();
   void find_and_write_scripts(const std::string& output_dir);
+  void dump_raw_objects(const std::string& output_dir);
 
   void write_object_file_words(const std::string& output_dir, bool dump_v3_only);
   void write_disassembly(const std::string& output_dir, bool disassemble_objects_without_functions);
   void analyze_functions();
   void process_tpages();
+  std::string process_game_count();
   std::string process_game_text();
 
   ObjectFileData& lookup_record(const ObjectFileRecord& rec);

decompiler/config.cpp

@@ -30,6 +30,8 @@ void set_config(const std::string& path_to_config_file) {
   gConfig.analyze_functions = cfg.at("analyze_functions").get<bool>();
   gConfig.process_tpages = cfg.at("process_tpages").get<bool>();
   gConfig.process_game_text = cfg.at("process_game_text").get<bool>();
+  gConfig.process_game_count = cfg.at("process_game_count").get<bool>();
+  gConfig.dump_objs = cfg.at("dump_objs").get<bool>();
 
   std::vector<std::string> asm_functions_by_name =
       cfg.at("asm_functions_by_name").get<std::vector<std::string>>();

decompiler/config.h

@@ -23,6 +23,8 @@ struct Config {
   bool analyze_functions = false;
   bool process_tpages = false;
   bool process_game_text = false;
+  bool process_game_count = false;
+  bool dump_objs = false;
   std::unordered_set<std::string> asm_functions_by_name;
   // ...
 };

decompiler/config/all-types.gc

@@ -10142,20 +10142,21 @@
   ((money-count  int32  :offset-assert 0)
    (buzzer-count int32  :offset-assert 4)
    )
+  :pack-me
   :method-count-assert 9
   :size-assert         #x8
   :flag-assert         #x900000008
   )
 
-; ;; progress-h
-; (deftype game-count-info (basic)
-;   ((length int32            :offset-assert 4)
-;    (data   UNKNOWN :dynamic :offset-assert 8)
-;    )
-;   :method-count-assert 9
-;   :size-assert         #x8
-;   :flag-assert         #x900000008
-;   )
+;; progress-h
+(deftype game-count-info (basic)
+  ((length int32            :offset-assert 4)
+   (data   count-info :inline :dynamic :offset-assert 8)
+   )
+  :method-count-assert 9
+  :size-assert         #x8
+  :flag-assert         #x900000008
+  )
 
 ; ;; progress-h
 ; (deftype task-info-data (basic)
@@ -33563,7 +33564,7 @@
 (define-extern reset-actors function)
 (define-extern get-task-control function)
 ;;(define-extern *kernel-boot-message* object) ;; unknown type
-(define-extern play function)
+(define-extern play (function none))
 (define-extern stop function)
 (define-extern auto-save-check function)
 ;;(define-extern auto-save object) ;; unknown type

decompiler/config/jak1_ntsc_black_label.jsonc

@@ -60,6 +60,8 @@
 
   "process_tpages":true,
   "process_game_text":true,
+  "process_game_count":true,
+  "dump_objs":false,
 
   // to write out data of each object file
   "write_hexdump":false,

decompiler/data/LinkedWordReader.h

@@ -0,0 +1,40 @@
+#pragma once
+#include <cassert>
+#include <cstring>
+#include <vector>
+#include <string>
+#include "common/common_types.h"
+#include "decompiler/ObjectFile/LinkedWord.h"
+
+class LinkedWordReader {
+ public:
+  explicit LinkedWordReader(const std::vector<LinkedWord>* words) : m_words(words) {}
+  const std::string& get_type_tag() {
+    if (m_words->at(m_offset).kind == LinkedWord::TYPE_PTR) {
+      auto& result = m_words->at(m_offset).symbol_name;
+      m_offset++;
+      return result;
+    } else {
+      assert(false);
+    }
+  }
+
+  template <typename T>
+  T get_word() {
+    static_assert(sizeof(T) == 4, "size of word in get_word");
+    T result;
+    assert(m_words->at(m_offset).kind == LinkedWord::PLAIN_DATA);
+    memcpy(&result, &m_words->at(m_offset).data, 4);
+    m_offset++;
+    return result;
+  }
+
+  u32 words_left() {
+    assert(m_words->size() >= m_offset);
+    return m_words->size() - m_offset;
+  }
+
+ private:
+  const std::vector<LinkedWord>* m_words = nullptr;
+  u32 m_offset = 0;
+};

decompiler/data/game_count.cpp

@@ -0,0 +1,40 @@
+#include "third-party/fmt/core.h"
+#include "decompiler/ObjectFile/ObjectFileDB.h"
+#include "game_count.h"
+#include "LinkedWordReader.h"
+
+GameCountResult process_game_count(ObjectFileData& data) {
+  GameCountResult result;
+  auto& words = data.linked_data.words_by_seg.at(0);
+  auto reader = LinkedWordReader(&words);
+  auto type = reader.get_type_tag();
+  assert(type == "game-count-info");
+  auto length = reader.get_word<s32>();
+
+  for (s32 i = 0; i < length; i++) {
+    GameCountResult::CountInfo info;
+    info.money_count = reader.get_word<s32>();
+    info.buzzer_count = reader.get_word<s32>();
+    result.info.push_back(info);
+  }
+
+  result.mystery_data[0] = reader.get_word<u32>();
+  result.mystery_data[1] = reader.get_word<u32>();
+  assert(reader.words_left() == 0);
+  return result;
+}
+
+std::string write_game_count(const GameCountResult& result) {
+  std::string str;
+  str +=
+      ";; this file contains money/buzzer counts for each level.\n;; The last pair is unknown data "
+      "and possibly a bug that it is included\n\n";
+
+  for (auto& x : result.info) {
+    str += fmt::format("(:money {} :buzzer {})\n", x.money_count, x.buzzer_count);
+  }
+
+  str += fmt::format("(:unknown-1 {} :unknown-2 {})\n", result.mystery_data[0],
+                     result.mystery_data[1]);
+  return str;
+}

decompiler/data/game_count.h

@@ -0,0 +1,18 @@
+#pragma once
+
+#include <vector>
+#include "common/common_types.h"
+
+struct GameCountResult {
+  struct CountInfo {
+    s32 money_count;
+    s32 buzzer_count;
+  };
+
+  std::vector<CountInfo> info;
+  u32 mystery_data[2];
+};
+
+struct ObjectFileData;
+GameCountResult process_game_count(ObjectFileData& data);
+std::string write_game_count(const GameCountResult& result);

decompiler/data/game_text.h

@@ -2,7 +2,7 @@
 #include <string>
 #include <unordered_map>
 
-class ObjectFileData;
+struct ObjectFileData;
 
 struct GameTextResult {
   int total_text = 0;

decompiler/main.cpp

@@ -46,6 +46,12 @@ int main(int argc, char** argv) {
   file_util::write_text_file(file_util::combine_path(out_folder, "obj.txt"),
                              db.generate_obj_listing());
 
+  if (get_config().dump_objs) {
+    auto path = file_util::combine_path(out_folder, "raw_obj");
+    file_util::create_dir_if_needed(path);
+    db.dump_raw_objects(path);
+  }
+
   db.process_link_data();
   db.find_code();
   db.process_labels();
@@ -71,6 +77,11 @@ int main(int argc, char** argv) {
     db.process_tpages();
   }
 
+  if (get_config().process_game_count) {
+    auto result = db.process_game_count();
+    file_util::write_text_file(file_util::get_file_path({"assets", "game_count.txt"}), result);
+  }
+
   if (get_config().write_disassembly) {
     db.write_disassembly(out_folder, get_config().disassemble_objects_without_functions);
   }

game/kernel/kboot.cpp

@@ -184,5 +184,5 @@ void KernelCheckAndDispatch() {
  * DONE, EXACT
  */
 void KernelShutdown() {
-  MasterExit = 1;  // GOAL Kernel Dispatch loop will stop now.
+  MasterExit = 2;  // GOAL Kernel Dispatch loop will stop now.
 }

game/kernel/klink.cpp

@@ -217,8 +217,12 @@ void link_control::begin(Ptr<uint8_t> object_file,
       }
 
     } else {
-      // not yet implemented
-      assert(false);
+      auto header_v4 = (const LinkHeaderV4*)header;
+      auto old_object_data = m_object_data;
+      m_link_block_ptr =
+          old_object_data + header_v4->code_size + sizeof(LinkHeaderV4) + BASIC_OFFSET;
+      m_object_data = old_object_data + sizeof(LinkHeaderV4);
+      m_code_size = header_v4->code_size;
     }
 
     if ((m_flags & LINK_FLAG_FORCE_DEBUG) && MasterDebug && !DiskBoot) {
@@ -768,7 +772,14 @@ void link_control::finish() {
       output_segment_load(m_object_name, m_link_block_ptr, m_flags);
     }
   } else {
-    printf("UNHANDELD OBJECT FILE VERSION IN FINISH\n");
+    if (m_flags & LINK_FLAG_EXECUTE) {
+      auto entry = m_entry;
+      auto name = basename_goal(m_object_name);
+      strcpy(Ptr<char>(LINK_CONTROL_NAME_ADDR).c(), name);
+      call_method_of_type_arg2(entry.offset, Ptr<Type>(*((entry - 4).cast<u32>())),
+                               GOAL_RELOC_METHOD, m_heap.offset,
+                               Ptr<char>(LINK_CONTROL_NAME_ADDR).offset);
+    }
   }
 
   *EnableMethodSet = *EnableMethodSet - m_keep_debug;

game/kernel/kmachine.h

@@ -23,6 +23,7 @@ constexpr u32 GLOBAL_HEAP_END = 0x1ffc000;
 //! Location of kglobalheap, kdebugheap kheapinfo structures.
 constexpr u32 GLOBAL_HEAP_INFO_ADDR = 0x13AD00;
 constexpr u32 DEBUG_HEAP_INFO_ADDR = 0x13AD10;
+constexpr u32 LINK_CONTROL_NAME_ADDR = 0x13AD80;
 
 //! Where to place the debug heap
 constexpr u32 DEBUG_HEAP_START = 0x5000000;

game/kernel/kscheme.h

@@ -102,6 +102,7 @@ u64 loado(u32 file_name_in, u32 heap_in);
 u64 unload(u32 name);
 Ptr<Function> make_function_symbol_from_c(const char* name, void* f);
 u64 call_goal_function_by_name(const char* name);
+u64 call_method_of_type_arg2(u32 arg, Ptr<Type> type, u32 method_id, u32 a1, u32 a2);
 Ptr<Type> alloc_and_init_type(Ptr<Symbol> sym, u32 method_count);
 Ptr<Symbol> set_fixed_symbol(u32 offset, const char* name, u32 value);
 

game/overlord/iso.cpp

@@ -649,7 +649,7 @@ u32 RunDGOStateMachine(IsoMessage* _cmd, IsoBufferHeader* buffer) {
     }
   }
 
-  printf("[DGO State Machine Complete] Out of things to read!\n");
+  //  printf("[DGO State Machine Complete] Out of things to read!\n");
 
 cleanup_and_return:
   if (return_value == 0) {

goal_src/engine/level/level.gc

@@ -5,3 +5,8 @@
 ;; name in dgo: level
 ;; dgos: GAME, ENGINE
 
+(defun play ()
+  (format #t "Play has been called!~%")
+  (format 0 "Play has been called!~%")
+  (kernel-shutdown)
+  )

goal_src/engine/ui/progress-h.gc

@@ -5,3 +5,23 @@
 ;; name in dgo: progress-h
 ;; dgos: GAME, ENGINE
 
+;; progress-h
+(deftype count-info (structure)
+  ((money-count  int32  :offset-assert 0)
+   (buzzer-count int32  :offset-assert 4)
+   )
+  :pack-me
+  :method-count-assert 9
+  :size-assert         #x8
+  :flag-assert         #x900000008
+  )
+
+;; progress-h
+(deftype game-count-info (basic)
+  ((length int32            :offset-assert 4)
+   (data   count-info :inline :dynamic :offset-assert 8)
+   )
+  :method-count-assert 9
+  :size-assert         #x8
+  :flag-assert         #x900000008
+  )

goal_src/engine/ui/progress/progress-static.gc

@@ -5,3 +5,4 @@
 ;; name in dgo: progress-static
 ;; dgos: GAME, ENGINE
 
+(define *game-counts* (the game-count-info '#f))

goal_src/engine/ui/progress/progress.gc

@@ -5,3 +5,7 @@
 ;; name in dgo: progress
 ;; dgos: GAME, ENGINE
 
+(defmethod relocate game-count-info ((this game-count-info) (offset int))
+  "Load in the game-count-info. This is a bit of a hack."
+  (set! *game-counts* this)
+  )

goal_src/goal-lib.gc

@@ -36,6 +36,7 @@
 (defmacro build-data ()
   `(begin
      (asm-data-file game-text "assets/game_text.txt")
+     (asm-data-file game-count "assets/game_count.txt")
      )
   )
 

goal_src/kernel-defs.gc

@@ -138,7 +138,7 @@
 ;; scf-get-timeout
 ;; scf-get-inactive-timeout
 ;; dma-to-iop
-;; kernel-shutdown
+(define-extern kernel-shutdown (function none))
 ;; aybabtu
 ;; *stack-top*
 ;; *stack-base*

goalc/CMakeLists.txt

@@ -26,6 +26,7 @@ add_library(compiler
         compiler/Util.cpp
         data_compiler/game_text.cpp
         data_compiler/DataObjectGenerator.cpp
+        data_compiler/game_count.cpp
         debugger/Debugger.cpp
         debugger/DebugInfo.cpp
         logger/Logger.cpp
@@ -38,7 +39,6 @@ add_library(compiler
         compiler/Compiler.cpp)
 
 add_executable(goalc main.cpp)
-add_executable(data_compiler data_compiler.cpp)
 
 IF (WIN32)
     target_link_libraries(compiler goos type_system mman common_util spdlog cross_os_debug cross_sockets Zydis)
@@ -46,5 +46,4 @@ ELSE ()
     target_link_libraries(compiler goos type_system common_util spdlog cross_os_debug cross_sockets Zydis)
 ENDIF ()
 
-target_link_libraries(goalc goos compiler type_system)
-target_link_libraries(data_compiler goos compiler type_system)
+target_link_libraries(goalc goos compiler type_system)

goalc/compiler/Util.cpp

@@ -1,36 +1,14 @@
 #include "goalc/compiler/Compiler.h"
 #include "goalc/compiler/IR.h"
+#include "common/goos/ParseHelpers.h"
 
 goos::Arguments Compiler::get_va(const goos::Object& form, const goos::Object& rest) {
   goos::Arguments args;
-  // loop over forms in list
-  goos::Object current = rest;
-  while (!current.is_empty_list()) {
-    auto arg = current.as_pair()->car;
 
-    // did we get a ":keyword"
-    if (arg.is_symbol() && arg.as_symbol()->name.at(0) == ':') {
-      auto key_name = arg.as_symbol()->name.substr(1);
-
-      // check for multiple definition of key
-      if (args.named.find(key_name) != args.named.end()) {
-        throw_compile_error(form, "Key argument " + key_name + " multiply defined");
-      }
-
-      // check for well-formed :key value expression
-      current = current.as_pair()->cdr;
-      if (current.is_empty_list()) {
-        throw_compile_error(form, "Key argument didn't have a value");
-      }
-
-      args.named[key_name] = current.as_pair()->car;
-    } else {
-      // not a keyword. Add to unnamed or rest, depending on what we expect
-      args.unnamed.push_back(arg);
-    }
-    current = current.as_pair()->cdr;
+  std::string err;
+  if (!goos::get_va(rest, &err, &args)) {
+    throw_compile_error(form, err);
   }
-
   return args;
 }
 
@@ -40,44 +18,9 @@ void Compiler::va_check(
     const std::vector<std::optional<goos::ObjectType>>& unnamed,
     const std::unordered_map<std::string, std::pair<bool, std::optional<goos::ObjectType>>>&
         named) {
-  assert(args.rest.empty());
-  if (unnamed.size() != args.unnamed.size()) {
-    throw_compile_error(form, "Got " + std::to_string(args.unnamed.size()) +
-                                  " arguments, but expected " + std::to_string(unnamed.size()));
-  }
-
-  for (size_t i = 0; i < unnamed.size(); i++) {
-    if (unnamed[i].has_value() && unnamed[i] != args.unnamed[i].type) {
-      throw_compile_error(form, "Argument " + std::to_string(i) + " has type " +
-                                    object_type_to_string(args.unnamed[i].type) + " but " +
-                                    object_type_to_string(unnamed[i].value()) + " was expected");
-    }
-  }
-
-  for (const auto& kv : named) {
-    auto kv2 = args.named.find(kv.first);
-    if (kv2 == args.named.end()) {
-      // argument not given.
-      if (kv.second.first) {
-        // but was required
-        throw_compile_error(form, "Required named argument \"" + kv.first + "\" was not found");
-      }
-    } else {
-      // argument given.
-      if (kv.second.second.has_value() && kv.second.second != kv2->second.type) {
-        // but is wrong type
-        throw_compile_error(form, "Argument \"" + kv.first + "\" has type " +
-                                      object_type_to_string(kv2->second.type) + " but " +
-                                      object_type_to_string(kv.second.second.value()) +
-                                      " was expected");
-      }
-    }
-  }
-
-  for (const auto& kv : args.named) {
-    if (named.find(kv.first) == named.end()) {
-      throw_compile_error(form, "Got unrecognized keyword argument \"" + kv.first + "\"");
-    }
+  std::string err;
+  if (!goos::va_check(args, unnamed, named, &err)) {
+    throw_compile_error(form, err);
   }
 }
 

goalc/compiler/compilation/CompilerControl.cpp

@@ -3,12 +3,14 @@
  * Compiler implementation for forms which actually control the compiler.
  */
 
+#include <filesystem>
 #include "goalc/compiler/Compiler.h"
 #include "goalc/compiler/IR.h"
 #include "common/util/Timer.h"
 #include "common/util/DgoWriter.h"
 #include "common/util/FileUtil.h"
 #include "goalc/data_compiler/game_text.h"
+#include "goalc/data_compiler/game_count.h"
 
 /*!
  * Exit the compiler. Disconnects the listener and tells the target to reset itself.
@@ -57,6 +59,8 @@ Val* Compiler::compile_asm_data_file(const goos::Object& form, const goos::Objec
   auto kind = symbol_string(args.unnamed.at(0));
   if (kind == "game-text") {
     compile_game_text(as_string(args.unnamed.at(1)));
+  } else if (kind == "game-count") {
+    compile_game_count(as_string(args.unnamed.at(1)));
   } else {
     throw_compile_error(form, "Unknown asm data file mode");
   }
@@ -290,8 +294,13 @@ Val* Compiler::compile_build_dgo(const goos::Object& form, const goos::Object& r
       DgoDescription::DgoEntry o;
       o.file_name = as_string(e_arg.unnamed.at(0));
       o.name_in_dgo = as_string(e_arg.unnamed.at(1));
-      if (o.file_name.substr(o.file_name.length() - 3) != ".go") {  // kill v2's for now.
+      if (o.file_name.substr(o.file_name.length() - 3) != ".go") {
         desc.entries.push_back(o);
+      } else {
+        // allow data objects to be missing.
+        if (std::filesystem::exists(file_util::get_file_path({"out", "obj", o.file_name}))) {
+          desc.entries.push_back(o);
+        }
       }
     });
 

goalc/data_compiler.cpp

@@ -1,46 +0,0 @@
-#include <cstdio>
-#include "goalc/compiler/Compiler.h"
-#include "common/versions.h"
-#include "third-party/spdlog/include/spdlog/spdlog.h"
-#include "third-party/spdlog/include/spdlog/sinks/basic_file_sink.h"
-#include "third-party/spdlog/include/spdlog/sinks/stdout_color_sinks.h"
-
-void setup_logging(bool verbose) {
-  spdlog::set_level(spdlog::level::debug);
-  if (verbose) {
-    auto game_logger = spdlog::stdout_color_mt("GOAL Compiler: Data Mode");
-    spdlog::set_default_logger(game_logger);
-    spdlog::flush_on(spdlog::level::info);
-    spdlog::set_pattern("%v");
-    spdlog::info("Verbose logging enabled");
-  } else {
-    auto game_logger = spdlog::basic_logger_mt("GOAL Compiler", "logs/data_compiler.log");
-    spdlog::set_default_logger(game_logger);
-    spdlog::flush_on(spdlog::level::debug);
-    printf("OpenGOAL Compiler %d.%d: Data Mode\n", versions::GOAL_VERSION_MAJOR,
-           versions::GOAL_VERSION_MINOR);
-  }
-}
-
-int main(int argc, char** argv) {
-  (void)argc;
-  (void)argv;
-
-  bool verbose = false;
-  for (int i = 1; i < argc; i++) {
-    if (std::string("-v") == argv[i]) {
-      verbose = true;
-      break;
-    }
-  }
-  setup_logging(verbose);
-
-  spdlog::info("OpenGOAL Compiler {}.{}: Data Mode", versions::GOAL_VERSION_MAJOR,
-               versions::GOAL_VERSION_MINOR);
-
-  Compiler compiler;
-  compiler.run_front_end_on_string("(build-data)");
-
-  printf("Done!\n");
-  return 0;
-}

goalc/data_compiler/DataObjectGenerator.cpp

@@ -100,8 +100,6 @@ std::vector<u8> DataObjectGenerator::generate_v2() {
   // Generate the link table.
   std::vector<u8> link = generate_link_table();
 
-  // add words
-
   // header
   LinkHeaderV2 header;
   header.type_tag = 0xffffffff;
@@ -124,6 +122,38 @@ std::vector<u8> DataObjectGenerator::generate_v2() {
   return result;
 }
 
+std::vector<u8> DataObjectGenerator::generate_v4() {
+  // add string data at the end.
+  add_strings();
+
+  // Generate the link table.
+  std::vector<u8> link = generate_link_table();
+
+  // header (first)
+  LinkHeaderV4 first_header;
+  first_header.type_tag = 0xffffffff;
+  first_header.length = sizeof(LinkHeaderV2) + link.size();
+  first_header.version = 4;
+  first_header.code_size = 4 * m_words.size();
+
+  LinkHeaderV2 second_header;
+  second_header.version = 2;
+  second_header.type_tag = 0xffffffff;
+  second_header.length = first_header.length;
+
+  std::vector<u8> result;
+  add_data_to_vector(first_header, &result);
+  auto start = result.size();
+  result.resize(result.size() + m_words.size() * 4);
+  memcpy(result.data() + start, m_words.data(), m_words.size() * 4);
+  while (result.size() % 16) {
+    result.push_back(0);
+  }
+  add_data_to_vector(second_header, &result);
+  result.insert(result.end(), link.begin(), link.end());
+  return result;
+}
+
 std::vector<u8> DataObjectGenerator::generate_link_table() {
   std::vector<u8> link;
 

goalc/data_compiler/DataObjectGenerator.h

@@ -14,6 +14,7 @@ class DataObjectGenerator {
   int add_type_tag(const std::string& str);
   int add_symbol_link(const std::string& str);
   std::vector<u8> generate_v2();
+  std::vector<u8> generate_v4();
   void align(int alignment_words);
   int words() const;
 

goalc/data_compiler/game_count.cpp

@@ -0,0 +1,78 @@
+#include <vector>
+#include "DataObjectGenerator.h"
+#include "common/goos/ParseHelpers.h"
+#include "common/goos/Reader.h"
+#include "common/util/FileUtil.h"
+#include "game_count.h"
+
+void compile_game_count(const std::string& filename) {
+  printf("[Build Game Count] %s\n", filename.c_str());
+  struct Count {
+    int buzzer;
+    int money;
+  };
+
+  std::vector<Count> counts;
+  int unknowns[2] = {0, 0};
+
+  goos::Reader reader;
+  auto code = reader.read_from_file({filename});
+  int len = goos::list_length(code) - 1;
+  int i = 0;
+  std::string err;
+
+  // parser
+  goos::for_each_in_list(code.as_pair()->cdr, [&](const goos::Object& obj) {
+    if (obj.is_pair()) {
+      if (i == len - 1) {
+        // last entry should be the unknowns.
+        goos::Arguments args;
+        if (!goos::get_va(obj, &err, &args)) {
+          throw std::runtime_error(err);
+        }
+        if (!goos::va_check(args, {},
+                            {{"unknown-1", {true, goos::ObjectType::INTEGER}},
+                             {"unknown-2", {true, goos::ObjectType::INTEGER}}},
+                            &err)) {
+          throw std::runtime_error(err);
+        }
+        unknowns[0] = args.get_named("unknown-1").as_int();
+        unknowns[1] = args.get_named("unknown-2").as_int();
+      } else {
+        goos::Arguments args;
+        if (!goos::get_va(obj, &err, &args)) {
+          throw std::runtime_error(err);
+        }
+        if (!goos::va_check(args, {},
+                            {{"buzzer", {true, goos::ObjectType::INTEGER}},
+                             {"money", {true, goos::ObjectType::INTEGER}}},
+                            &err)) {
+          throw std::runtime_error(err);
+        }
+        Count c;
+        c.buzzer = args.get_named("buzzer").as_int();
+        c.money = args.get_named("money").as_int();
+        counts.push_back(c);
+      }
+    } else {
+      throw std::runtime_error("Invalid game count file.");
+    }
+    i++;
+  });
+
+  // compiler
+  DataObjectGenerator gen;
+  gen.add_type_tag("game-count-info");
+  gen.add_word(counts.size());
+  for (auto& x : counts) {
+    gen.add_word(x.money);
+    gen.add_word(x.buzzer);
+  }
+  gen.add_word(unknowns[0]);
+  gen.add_word(unknowns[1]);
+  auto result = gen.generate_v4();
+
+  file_util::create_dir_if_needed(file_util::get_file_path({"out", "obj"}));
+  file_util::write_binary_file(file_util::get_file_path({"out", "obj", "game-cnt.go"}),
+                               result.data(), result.size());
+}

goalc/data_compiler/game_count.h

@@ -0,0 +1,4 @@
+#pragma once
+#include <string>
+
+void compile_game_count(const std::string& filename);

goalc/main.cpp

@@ -25,12 +25,17 @@ int main(int argc, char** argv) {
   (void)argc;
   (void)argv;
 
+  std::string argument;
   bool verbose = false;
   for (int i = 1; i < argc; i++) {
     if (std::string("-v") == argv[i]) {
       verbose = true;
       break;
     }
+
+    if (std::string("-cmd") == argv[i] && i < argc - 1) {
+      argument = argv[++i];
+    }
   }
   setup_logging(verbose);
 
@@ -38,7 +43,12 @@ int main(int argc, char** argv) {
                versions::GOAL_VERSION_MINOR);
 
   Compiler compiler;
-  compiler.execute_repl();
+
+  if (argument.empty()) {
+    compiler.execute_repl();
+  } else {
+    compiler.run_front_end_on_string(argument);
+  }
 
   return 0;
 }

offline_test_git_branch.sh

@@ -45,9 +45,15 @@ echo " ================ Decompiling..."
 ../decomp.sh
 
 echo " ================ Building assets..."
-./goalc/data_compiler
+../gc.sh -cmd \(build-data\)
 
 echo " ================ Checking assets..."
 ../check.sh
 
+echo " ================ Building game..."
+../gc.sh -cmd \(build-game\)
+
+echo " ================ Booting game..."
+../boot_game.sh
+
 echo "Offline test has completed successfully!"

out/hash.md5

@@ -5,3 +5,4 @@ abcc25e5d7469dd6a572dc53dbb9671c  iso/3COMMON.TXT
 82eabdb7159f2059fbdbd18bb6fc06aa  iso/4COMMON.TXT
 5d62de2c78b4cf102b9a78f3aa96c8c9  iso/5COMMON.TXT
 9495f80955e6782513fe12f6539fc8e7  iso/6COMMON.TXT
+9765bdc3add08cb06fd3e87ebd5713aa  obj/game-cnt.go

test/goalc/source_templates/with_game/test-game-count.gc

@@ -0,0 +1,8 @@
+(start-test "game-count")
+
+(expect-true (= (-> *game-counts* data 0 money-count) 123))
+(expect-true (= (-> *game-counts* data 0 buzzer-count) 456))
+(expect-true (= (-> *game-counts* data 1 money-count) 789))
+(expect-true (= (-> *game-counts* data 1 buzzer-count) 221))
+
+(finish-test)

test/goalc/test_with_game.cpp

@@ -293,6 +293,15 @@ TEST_F(WithGameTests, GameText) {
                          get_test_pass_string("game-text", 5));
 }
 
+TEST_F(WithGameTests, GameCount) {
+  compiler.run_test_from_string(
+      "(asm-data-file game-count \"test/test_data/test_game_counts.txt\")");
+  compiler.run_test_from_string("(build-dgos \"test/test_data/test_game_count_dgos.txt\")");
+  compiler.run_test_from_string("(dgo-load \"game\" global #xf #x200000)");
+  runner.run_static_test(env, testCategory, "test-game-count.gc",
+                         get_test_pass_string("game-count", 4));
+}
+
 TEST(TypeConsistency, TypeConsistency) {
   Compiler compiler;
   compiler.enable_throw_on_redefines();

test/test_data/test_game_count_dgos.txt

@@ -0,0 +1,6 @@
+("GAME.CGO"
+  ("types-h.o" "types-h")
+  ("vu1-macros.o" "vu1-macros")
+  ("game-cnt.go" "game-cnt")
+  ("math.o" "math")
+  )

test/test_data/test_game_counts.txt

@@ -0,0 +1,3 @@
+(:money 123 :buzzer 456)
+(:money 789 :buzzer 221)
+(:unknown-1 123123 :unknown-2 234234)