jak-project/scripts/gsrc/copy-common-naming.py

# The goal of this script is fairly straight forward
# Given a object file name, decompile it for both jak 2 and jak 3 WITHOUT variable casts
# Get all function definitions and compare the function bodies.
# - if the function bodies are the same, copy the variables from whichever game has them defined to the other
#   - also, if it's a function and a docstring exists on one side but not the other, copy the docstring to the other side's all-types file

import argparse
import glob
import json
import os

from utils import decompile_file, is_file_in_game


parser = argparse.ArgumentParser("copy-common-naming")
parser.add_argument("--file", help="The name of the file", type=str)
parser.add_argument("--decompiler", help="The path to the decompiler", type=str)
parser.add_argument("--update-names-from-refs", help="The decomp config version", action='store_true')
args = parser.parse_args()

def find_all_function_defs(lines):
  store = {}
  in_function_def = False
  in_docstring = False
  passed_potential_docstring = False
  current_function_name = None
  for line in lines:
    if line.startswith("; .function") and "top-level" not in line:
      current_function_name = line.split(".function")[1].strip()
      store[current_function_name] = {
        "docstring": [],
        "definition": [],
      }
      passed_potential_docstring = False
      in_docstring = False
      continue
    if current_function_name is not None and line.startswith(";;-*-OpenGOAL-Start-*-"):
      in_function_def = True
      continue
    if current_function_name is not None and line.startswith(";;-*-OpenGOAL-End-*-"):
      in_function_def = False
      continue
    if line.strip() == "":
      continue
    if in_function_def:
      if not passed_potential_docstring and line.strip().startswith("\""):
        in_docstring = True
      if in_docstring:
        store[current_function_name]["docstring"].append(line.strip())
        if line.strip().endswith("\""):
          in_docstring = False
      else:
        store[current_function_name]["definition"].append(line)
        if len(store[current_function_name]["definition"]) > 1 and line.startswith("   "):
          passed_potential_docstring = True
  return store

def get_var_casts_for_game(game_name):
  return json.load(open("./decompiler/config/{}/ntsc_v1/var_names.jsonc".format(game_name), "r"))

def save_var_casts_for_game(game_name, casts):
  with open("./decompiler/config/{}/ntsc_v1/var_names.jsonc".format(game_name), "w") as f:
    json.dump(casts, f, indent=2)

def get_all_types_for_game(game_name):
  return open("./decompiler/config/{}/all-types.gc".format(game_name), "r").readlines()

jak2_alltypes = get_all_types_for_game("jak2")
jak3_alltypes = get_all_types_for_game("jak3")

file_stats = ""

def update_file_var_name_casts(file_name, modify_alltypes):
  global file_stats
  # Check if the file exists in both games
  if not is_file_in_game("jak3", file_name) or not is_file_in_game("jak2", file_name):
    print("File not found in both games")
    return

  # Decompile the file for both games
  decompile_file(args.decompiler, "jak3/jak3_config.jsonc", "ntsc_v1", "[\"{}\"]".format(file_name), True)
  decompile_file(args.decompiler, "jak2/jak2_config.jsonc", "ntsc_v1", "[\"{}\"]".format(file_name), True)

  # Go grab the contents of each file
  jak2_file_contents = open("./decompiler_out/jak2/{}_ir2.asm".format(file_name), "r").readlines()
  jak3_file_contents = open("./decompiler_out/jak3/{}_ir2.asm".format(file_name), "r").readlines()

  # Read in the function definitions for each file to find which ones match

  jak2_function_defs = find_all_function_defs(jak2_file_contents)
  jak3_function_defs = find_all_function_defs(jak3_file_contents)

  # print(jak2_function_defs["vector-xz-cross!"])
  # print()
  # print(jak3_function_defs["vector-xz-cross!"])

  # Compare functions to see which ones are eligible
  matching_func_names = []
  for func_name in jak2_function_defs:
    if func_name in jak3_function_defs and jak2_function_defs[func_name]["definition"] == jak3_function_defs[func_name]["definition"]:
      matching_func_names.append(func_name)

  # print(matching_func_names)
  file_stats = file_stats + "Found {} matching functions in {}\n".format(len(matching_func_names), file_name)

  # Go grab the var casts for each game
  jak2_var_casts = get_var_casts_for_game("jak2")
  jak3_var_casts = get_var_casts_for_game("jak3")

  # For each eligible matching function, copy over the var casts
  # The assumption is if it exists in jak 3 it's better (done more recently) so we use that
  # else, use jak 2's if it exists
  for func_name in matching_func_names:
    if func_name in jak3_var_casts:
      jak2_var_casts[func_name] = jak3_var_casts[func_name]
    elif func_name in jak2_var_casts:
      jak3_var_casts[func_name] = jak2_var_casts[func_name]

  save_var_casts_for_game("jak2", jak2_var_casts)
  save_var_casts_for_game("jak3", jak3_var_casts)

  # Automatically copy docstrings for functions (methods are way to annoying to do with hack scripts now)
  if modify_alltypes:
    for func_name in matching_func_names:
      if func_name.startswith("("):
        continue
      # handle the case where the jak 3 version has a docstring, but jak 2 does not
      if len(jak3_function_defs[func_name]["docstring"]) != 0 and len(jak2_function_defs[func_name]["docstring"]) == 0:
        for line_no, line in enumerate(jak2_alltypes):
          line = jak2_alltypes[line_no]
          if line.startswith("(define-extern {}".format(func_name)):
            jak2_alltypes[line_no] = line.replace("(define-extern {} ".format(func_name), "(define-extern {}\n  {}\n  ".format(func_name, "\n  ".join(jak3_function_defs[func_name]["docstring"])))
            break
      # handle the case where jak 2 has a docstring but jak 3 does not
      elif len(jak2_function_defs[func_name]["docstring"]) != 0 and len(jak3_function_defs[func_name]["docstring"]) == 0:
        for line_no, line in enumerate(jak3_alltypes):
          line = jak3_alltypes[line_no]
          if line.startswith("(define-extern {}".format(func_name)):
            jak3_alltypes[line_no] = line.replace("(define-extern {} ".format(func_name), "(define-extern {}\n  {}\n  ".format(func_name, "\n  ".join(jak2_function_defs[func_name]["docstring"])))
            break

if args.update_names_from_refs:
  reference_test_files = glob.glob("./test/decompiler/reference/jak3/**/*_REF.gc", recursive=True)
  for file_no, reference_test_file in enumerate(reference_test_files):
    file_name = os.path.basename(reference_test_file).split("_REF.gc")[0]
    print("({}/{}) Checking Var Name Casts for {}...".format(file_no+1, len(reference_test_files), file_name))
    update_file_var_name_casts(file_name, False)
else:
  update_file_var_name_casts(args.file, True)

print(file_stats)

def get_type_docstrings_from_alltypes(lines):
  store = {}
  awaiting_next_docstring = True
  current_type_name = None
  for line in lines:
    if line.startswith("(deftype"):
      current_type_name = line.split("deftype ")[1].split("(")[0].strip()
      awaiting_next_docstring = False
      store[current_type_name] = []
      continue
    if line.strip().startswith("(") and not line.strip().endswith("\""):
      awaiting_next_docstring = True
      continue
    if not awaiting_next_docstring:
      store[current_type_name].append(line.strip())
  return store

jak2_type_docs = get_type_docstrings_from_alltypes(jak2_alltypes)
jak3_type_docs = get_type_docstrings_from_alltypes(jak3_alltypes)

# If a docstring exists in jak3 but not in jak2, copy it back
new_jak2_alltypes = []
for line_no, line in enumerate(jak2_alltypes):
  line = jak2_alltypes[line_no]
  new_jak2_alltypes.append(line)
  if line.startswith("(deftype "):
    current_type_name = line.split("deftype ")[1].split("(")[0].strip()
    if current_type_name in jak3_type_docs and len(jak2_type_docs[current_type_name]) == 0:
      for docstring_line in jak3_type_docs[current_type_name]:
        new_jak2_alltypes.append("  " + docstring_line + "\n")
jak2_alltypes = new_jak2_alltypes

# Save all-types
def get_all_types_for_game(game_name, lines):
  with open("./decompiler/config/{}/all-types.gc".format(game_name), "w") as f:
    f.writelines(lines)

get_all_types_for_game("jak2", jak2_alltypes)
get_all_types_for_game("jak3", jak3_alltypes)