formatter: more changes to constant pairs

.vs/launch.vs.json

@@ -402,7 +402,7 @@
       "args": [
         "--write",
         "--file",
-        "C:\\Users\\xtvas\\Repositories\\opengoal\\jak-project\\goal_src\\jak1\\engine\\anim\\joint.gc"
+        "C:\\Users\\xtvas\\Repositories\\opengoal\\jak-project\\goal_src\\jak1\\engine\\camera\\cam-states.gc"
       ]
     },
     {

common/formatter/formatter.cpp

@@ -319,12 +319,13 @@ std::vector<std::string> apply_formatting(const FormatterTreeNode& curr_node,
       // combine the next inline comment or constant pair
       if ((next_ref.metadata.node_type == "comment" && next_ref.metadata.is_inline) ||
           (curr_node.formatting_config.has_constant_pairs &&
-           constant_pairs::is_element_second_in_constant_pair(curr_node, next_ref, i + 1))) {
+           constant_pairs::is_element_second_in_constant_pair(curr_node, next_ref, i + 1)) ||
+          constant_pairs::is_element_second_in_constant_pair_new(curr_node.refs.at(i), next_ref)) {
         // TODO
         // has issues with not consolidating first lines, this should probably just be moved to
         // outside this loop for simplicity, do it later
         if (next_ref.token) {
-          form_lines.at(form_lines.size() - 1) += fmt::format(" {}", next_ref.token.value());
+          form_lines.at(form_lines.size() - 1) += fmt::format(" {}", next_ref.token_str());
           i++;
           // We have to handle hang-consolidation here or else it will never be reached above!
           if (i == (int)curr_node.refs.size() - 1 && form_lines.size() > 1 &&

common/formatter/rules/formatting_rules.cpp

@@ -12,6 +12,7 @@ namespace formatter_rules {
 // differentiate between a quoted symbol and a quoted form
 const std::set<std::string> constant_types = {"kwd_lit",  "num_lit",  "str_lit",
                                               "char_lit", "null_lit", "bool_lit"};
+const std::set<std::string> constant_type_forms = {"meters", "seconds", "degrees"};
 
 namespace constant_list {
 bool is_constant_list(const FormatterTreeNode& node) {
@@ -114,6 +115,34 @@ bool is_element_second_in_constant_pair(const FormatterTreeNode& containing_node
   return true;
 }
 
+// TODO - potentially remove the above
+bool is_element_second_in_constant_pair_new(const FormatterTreeNode& prev_node,
+                                            const FormatterTreeNode& curr_node) {
+  if (prev_node.metadata.node_type == "kwd_lit") {
+    // Handle standard constant types
+    if (constant_types.find(curr_node.metadata.node_type) != constant_types.end()) {
+      if (curr_node.metadata.node_type != "kwd_lit") {
+        // NOTE - there is ambiugity here which cannot be totally solved (i think?)
+        // if the element itself is also a keyword, assume this is two adjacent keywords and they
+        // should not be paired
+        return true;
+      }
+    }
+    // Quoted symbols
+    if (curr_node.metadata.node_type == "sym_name" && curr_node.node_prefix &&
+        curr_node.node_prefix.value() == "'") {
+      return true;
+    }
+    // Constant forms special cases (ie. meters)
+    if (!curr_node.refs.empty() &&
+        constant_type_forms.find(curr_node.refs.at(0).token_str()) !=
+            constant_type_forms.end()) {
+      return true;
+    }
+  }
+  return false;
+}
+
 bool form_should_be_constant_paired(const FormatterTreeNode& node) {
   // Criteria for a list to be constant paired:
   // - needs to start with a non-symbol

common/formatter/rules/formatting_rules.h

@@ -72,6 +72,8 @@ const static int min_pair_amount = 4;
 bool is_element_second_in_constant_pair(const FormatterTreeNode& containing_node,
                                         const FormatterTreeNode& node,
                                         const int index);
+bool is_element_second_in_constant_pair_new(const FormatterTreeNode& prev_node,
+                                            const FormatterTreeNode& curr_node);
 bool form_should_be_constant_paired(const FormatterTreeNode& node);
 }  // namespace constant_pairs
 

common/formatter/rules/rule_config.cpp

@@ -186,6 +186,7 @@ const std::unordered_map<std::string, FormFormattingConfig> opengoal_form_config
     {"defun-debug", new_flow_rule(3)},
     {"defbehavior", new_flow_rule(4)},
     {"if", new_inlineable_flow_rule(2)},
+    {"#if", new_inlineable_flow_rule(2)},
     {"define", new_permissive_flow_rule()},
     {"define-extern", new_permissive_flow_rule()},
     {"defmacro", new_flow_rule(3)},
@@ -198,10 +199,12 @@ const std::unordered_map<std::string, FormFormattingConfig> opengoal_form_config
     {"when", new_flow_rule(2)},
     {"countdown", new_flow_rule(2)},
     {"until", new_flow_rule(2)},
+    {"loop", new_flow_rule(2)},
     {"while", new_flow_rule(2)},
     {"begin", new_flow_rule(0)},
     {"with-pp", new_flow_rule(0)},
     {"local-vars", new_inlinable_simple_flow_rule()},
-    {"with-dma-buffer-add-bucket", new_flow_rule(2)}};
+    {"with-dma-buffer-add-bucket", new_flow_rule(2)},
+    {"dma-bucket-insert-tag", new_flow_rule(2)}};
 }  // namespace config
 }  // namespace formatter_rules

scripts/gsrc/format-gsrc-file.py

@@ -21,24 +21,34 @@ with open("./scripts/gsrc/format-jak1.json", "r") as f:
     formatting_progress = json.load(f)
 
 # find the next file
-index = 0
+curr_file = None
+curr_file_index = 0
+go_to_next = False
+open_file_in_vscode = False
+if apply_status == "next":
+    go_to_next = True
+    open_file_in_vscode = True
+
 for index, file in enumerate(formatting_progress):
     if file['status'] == 'not-formatted':
-        next_file = file['path']
+        if go_to_next:
+            print(f"Marking {file['path']} as formatted")
+            formatting_progress[index]['status'] = 'formatted'
+            go_to_next = False
+        else:
+            curr_file = file['path']
+            curr_file_index = index
+            if open_file_in_vscode:
+                subprocess.run(["C:\\Users\\xtvas\\AppData\\Local\\Programs\\Microsoft VS Code\\bin\\code.cmd", file['path']]) 
             break
 
 # format it
-print(f"Formatting {next_file}")
+print(f"Formatting {curr_file}")
-subprocess.run(["./out/build/Debug/bin/formatter", "--write", "--file", next_file]) 
+subprocess.run(["./out/build/Debug/bin/formatter", "--write", "--file", curr_file]) 
 
 # save status
 if apply_status is not None and (apply_status == "skip" or apply_status == "next"):
-    if apply_status == "next":
+    # TODO - add skip support back if i ever want to use it
-        formatting_progress[index]['status'] = 'formatted'
-        print(f"Marking {next_file} as formatted")
-    else:
-        formatting_progress[index]['status'] = 'skipped'
-        print(f"Marking {next_file} as skipped")
     with open("./scripts/gsrc/format-jak1.json", "w") as f:
         f.write(json.dumps(formatting_progress, indent=2))
 

test/common/formatter/corpus/constant-pairs.test.gc

@@ -100,3 +100,85 @@ Amibiguous List
  :test 456
  :not-paired
  :doit 789)
+
+===
+Within unrelated form
+===
+
+(new 'static
+     'gif-tag64
+     :nloop
+     #x1
+     :eop
+     #x1
+     :pre
+     #x1
+     :prim
+     (new 'static 'gs-prim :prim (gs-prim-type line) :iip #x1 :abe #x1)
+     :nreg
+     #x4)
+
+---
+
+(new 'static
+     'gif-tag64
+     :nloop #x1
+     :eop #x1
+     :pre #x1
+     :prim
+     (new 'static 'gs-prim :prim (gs-prim-type line) :iip #x1 :abe #x1)
+     :nreg #x4)
+
+===
+Quoted Symbol
+===
+
+(new 'static
+     'clm-item
+     :description "adjust"
+     :button-symbol
+     'x
+     :action
+     (new 'static 'clm-item-action :button #x4000 :func #f))
+
+---
+
+(new 'static
+     'clm-item
+     :description "adjust"
+     :button-symbol 'x
+     :action
+     (new 'static 'clm-item-action :button #x4000 :func #f))
+
+===
+Special case forms
+===
+
+(define *CAMERA_MASTER-bank*
+  (new 'static
+       'camera-master-bank
+       :onscreen-head-height
+       (meters 2.65)
+       :onscreen-foot-height
+       (meters -0.5)
+       :target-height
+       (meters 2.15)
+       :up-move-to-pitch-ratio-in-air 1.0
+       :down-move-to-pitch-ratio-in-air 0.5
+       :up-move-to-pitch-on-ground 0.9
+       :down-move-to-pitch-on-ground 0.9
+       :pitch-off-blend 0.5))
+
+---
+
+(define *CAMERA_MASTER-bank*
+  (new 'static
+       'camera-master-bank
+       :onscreen-head-height (meters 2.65)
+       :onscreen-foot-height (meters -0.5)
+       :target-height (meters 2.15)
+       :up-move-to-pitch-ratio-in-air 1.0
+       :down-move-to-pitch-ratio-in-air 0.5
+       :up-move-to-pitch-on-ground 0.9
+       :down-move-to-pitch-on-ground 0.9
+       :pitch-off-blend 0.5))

third-party/tree-sitter/tree-sitter-opengoal/grammar.js

@@ -117,7 +117,7 @@ const CHARACTER =
 // \u205f => <medium mathematical space>
 // \u3000 => <ideographic space>
 const SYMBOL_HEAD =
-  /[^\f\n\r\t \/()\[\]{}"@~^;`\\,:#'0-9\u000B\u001C\u001D\u001E\u001F\u2028\u2029\u1680\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2008\u2009\u200a\u205f\u3000]/;
+  /[^\f\n\r\t \/()\[\]{}"@~^;`\\,:'0-9\u000B\u001C\u001D\u001E\u001F\u2028\u2029\u1680\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2008\u2009\u200a\u205f\u3000]/;
 
 const SYMBOL_BODY =
   choice(SYMBOL_HEAD,
@@ -205,7 +205,7 @@ module.exports = grammar({
       seq(field('numberOfArgs', $._format_token), '*'),
       '?',
       "Newline",
-      seq(repeat(choice($._format_token, ',')), /[$mrRbBdDgGxXeEoOsStTfFHhJjKkLlNnVwWyYzZ]/),
+      seq(repeat(choice($._format_token, ',')), /[$mMrRbBdDgGxXeEoOsStTfFHhJjKkLlNnVwWyYzZ]/),
     ),
     format_specifier: $ =>
       prec.left(seq(

third-party/tree-sitter/tree-sitter-opengoal/grammar.json

@@ -646,7 +646,7 @@
             },
             {
               "type": "PATTERN",
-              "value": "[$mrRbBdDgGxXeEoOsStTfFHhJjKkLlNnVwWyYzZ]"
+              "value": "[$mMrRbBdDgGxXeEoOsStTfFHhJjKkLlNnVwWyYzZ]"
             }
           ]
         }
@@ -846,7 +846,7 @@
                 "members": [
                   {
                     "type": "PATTERN",
-                    "value": "[^\\f\\n\\r\\t \\/()\\[\\]{}\"@~^;`\\\\,:#'0-9\\u000B\\u001C\\u001D\\u001E\\u001F\\u2028\\u2029\\u1680\\u2000\\u2001\\u2002\\u2003\\u2004\\u2005\\u2006\\u2008\\u2009\\u200a\\u205f\\u3000]"
+                    "value": "[^\\f\\n\\r\\t \\/()\\[\\]{}\"@~^;`\\\\,:'0-9\\u000B\\u001C\\u001D\\u001E\\u001F\\u2028\\u2029\\u1680\\u2000\\u2001\\u2002\\u2003\\u2004\\u2005\\u2006\\u2008\\u2009\\u200a\\u205f\\u3000]"
                   },
                   {
                     "type": "REPEAT",
@@ -855,7 +855,7 @@
                       "members": [
                         {
                           "type": "PATTERN",
-                          "value": "[^\\f\\n\\r\\t \\/()\\[\\]{}\"@~^;`\\\\,:#'0-9\\u000B\\u001C\\u001D\\u001E\\u001F\\u2028\\u2029\\u1680\\u2000\\u2001\\u2002\\u2003\\u2004\\u2005\\u2006\\u2008\\u2009\\u200a\\u205f\\u3000]"
+                          "value": "[^\\f\\n\\r\\t \\/()\\[\\]{}\"@~^;`\\\\,:'0-9\\u000B\\u001C\\u001D\\u001E\\u001F\\u2028\\u2029\\u1680\\u2000\\u2001\\u2002\\u2003\\u2004\\u2005\\u2006\\u2008\\u2009\\u200a\\u205f\\u3000]"
                         },
                         {
                           "type": "PATTERN",