scripts: update scripts and docs to support multiple games (#1584)

scripts: update scripts and docs to support multi-games disable broken scripts for now
This commit is contained in:
Tyler Wilding 2022-06-30 20:17:06 -04:00 committed by GitHub
parent 0cefb366f6
commit 728d234976
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
10 changed files with 181 additions and 282 deletions

1
.gitignore vendored
View file

@ -27,6 +27,7 @@ linux-default/
savestate-out/
failures/
ee-results.json
.env
# graphics debug
debug_out/*

View file

@ -12,16 +12,14 @@
<a href="https://makeapullrequest.com"><img src="https://img.shields.io/badge/PRs-welcome-brightgreen.svg?style=flat-square" alt=PRs Welcome></a>
</p>
## Table of Contents
<!-- toc -->
- [Project Description](#project-description)
- [**Please read the FAQ by clicking here if you have any questions.**](#please-read-the-faq-by-clicking-here-if-you-have-any-questions)
- [Current Status](#current-status)
- [What's Next](#whats-next)
- [Getting Started - Linux](#getting-started---linux)
- [Ubuntu (20.04)](#ubuntu-2004)
- [Arch](#arch)
- [Fedora](#fedora)
- [Getting Started - Windows](#getting-started---windows)
- [Required Software](#required-software)
- [Setting up and Opening the Project](#setting-up-and-opening-the-project)
@ -35,8 +33,6 @@
- [Project Layout](#project-layout)
- [Directory Layout](#directory-layout)
<!-- tocstop -->
## Project Description
This project is to port Jak 1 (NTSC, "black label" version) to PC. Over 98% of this game is written in GOAL, a custom Lisp language developed by Naughty Dog. Our strategy is:
@ -172,7 +168,6 @@ Once Scoop is installed, run the following commands:
```sh
scoop install git llvm nasm python
scoop bucket add extras
scoop install task
```
@ -204,12 +199,23 @@ Getting a running game involves 4 steps:
### Extract Assets
First, setup your settings so the following scripts know which game you are using, and which version. In a terminal, run the following:
```sh
task set-game-jak1
task set-decomp-ntscv1
```
> Run `task --list` to see the other available options
> At the time of writing, only Jak 1 is expected to work end-to-end!
The first step is to extract your ISO file contents into the `iso_data/<game-name>` folder. In the case of Jak 1 this is `iso_data/jak1`.
Once this is done, open a terminal in the `jak-project` folder and run the following:
```sh
task extract-jak1
task extract
```
### Build the Game

View file

@ -1,48 +1,63 @@
version: '3'
version: "3.13"
includes:
build: ./scripts/tasks/Taskfile_{{OS}}.yml
dotenv:
- ./scripts/tasks/.env
- ./scripts/tasks/.env.default
tasks:
# TODO - make it easy to switch between games instead of having a bunch of "jak1/jak2" varients
# SETTINGS / CONFIGURATION
set-game-jak1:
- 'python ./scripts/tasks/update-env.py --game jak1'
set-game-jak2:
- 'python ./scripts/tasks/update-env.py --game jak2'
set-decomp-ntscv1:
desc: "aka black label"
cmds:
- 'python ./scripts/tasks/update-env.py --decomp_config ntscv1'
set-decomp-ntscv2:
desc: "aka red label"
cmds:
- 'python ./scripts/tasks/update-env.py --decomp_config ntscv2'
set-decomp-pal:
- 'python ./scripts/tasks/update-env.py --decomp_config pal'
set-decomp-ntscjp:
- 'python ./scripts/tasks/update-env.py --decomp_config ntscjp'
# GENERAL
extract-jak1:
desc: "Extracts Jak 1 - NTSC - Black Label assets"
extract:
desc: "Extracts the game's assets from './iso_data' with the set decompiler config"
preconditions:
- sh: test -f {{.DECOMP_BIN_RELEASE_DIR}}/decompiler{{.EXE_FILE_EXTENSION}}
msg: "Couldn't locate decompiler executable in '{{.DECOMP_BIN_RELEASE_DIR}}/decompiler'"
cmds:
- '{{.DECOMP_BIN_RELEASE_DIR}}/decompiler "./decompiler/config/jak1_ntsc_black_label.jsonc" "./iso_data" "./decompiler_out" "decompile_code=false"'
- '{{.DECOMP_BIN_RELEASE_DIR}}/decompiler "./decompiler/config/{{.DECOMP_CONFIG}}" "./iso_data" "./decompiler_out" "decompile_code=false"'
boot-game:
desc: "Boots the game"
desc: "Boots the game, it will fail if it's not already booted!"
preconditions:
- sh: test -f {{.GK_BIN_RELEASE_DIR}}/gk{{.EXE_FILE_EXTENSION}}
msg: "Couldn't locate runtime executable in '{{.GK_BIN_RELEASE_DIR}}/gk'"
cmds:
- "{{.GK_BIN_RELEASE_DIR}}/gk -boot -fakeiso -debug -v"
run-game:
desc: "Start the game's runtime"
desc: "Start the game's runtime, to start the game itself the REPL is required"
preconditions:
- sh: test -f {{.GK_BIN_RELEASE_DIR}}/gk{{.EXE_FILE_EXTENSION}}
msg: "Couldn't locate runtime executable in '{{.GK_BIN_RELEASE_DIR}}/gk'"
cmds:
- "{{.GK_BIN_RELEASE_DIR}}/gk -fakeiso -debug -v"
run-game-quiet:
preconditions:
- sh: test -f {{.GK_BIN_RELEASE_DIR}}/gk{{.EXE_FILE_EXTENSION}}
msg: "Couldn't locate runtime executable in '{{.GK_BIN_RELEASE_DIR}}/gk'"
cmds:
- "{{.GK_BIN_RELEASE_DIR}}/gk -fakeiso"
# DEVELOPMENT
repl:
desc: "Start the REPL"
env:
OPENGOAL_DECOMP_DIR: "jak1/"
preconditions:
- sh: test -f {{.GOALC_BIN_RELEASE_DIR}}/goalc{{.EXE_FILE_EXTENSION}}
msg: "Couldn't locate compiler executable in '{{.GOALC_BIN_RELEASE_DIR}}/goalc'"
cmds:
- "{{.GOALC_BIN_RELEASE_DIR}}/goalc"
# DEVELOPMENT
repl-lt:
cmds:
- "{{.GOALC_BIN_RELEASE_DIR}}/goalc --auto-lt"
format:
desc: "Format code"
cmds:
@ -53,84 +68,76 @@ tasks:
run-game-headless:
cmds:
- "{{.GK_BIN_RELEASE_DIR}}/gk -fakeiso -debug -nodisplay"
repl-lt:
env:
OPENGOAL_DECOMP_DIR: "jak1/"
cmds:
- "{{.GOALC_BIN_RELEASE_DIR}}/goalc --auto-lt"
# DECOMPILING
decomp:
cmds:
- '{{.DECOMP_BIN_RELEASE_DIR}}/decompiler "./decompiler/config/jak1_ntsc_black_label.jsonc" "./iso_data" "./decompiler_out"'
decomp-jak2:
cmds:
- '{{.DECOMP_BIN_RELEASE_DIR}}/decompiler "./decompiler/config/jak2_ntsc_v1.jsonc" "./iso_data" "./decompiler_out"'
- '{{.DECOMP_BIN_RELEASE_DIR}}/decompiler "./decompiler/config/{{.DECOMP_CONFIG}}" "./iso_data" "./decompiler_out"'
decomp-clean:
cmds:
- rm ./decompiler_out/**/*.asm
- rm ./decompiler_out/**/*disasm.gc
decomp-file:
cmds:
- python ./scripts/next-decomp-file.py --files "{{.FILES}}"
- task: decomp
decomp-list:
cmds:
- python ./scripts/next-decomp-file.py --list "{{.LIST}}"
vars:
LIST: '{{default "0" .LIST}}'
# python -m pip install -U watchdog[watchmedo]
decomp-watch:
cmds:
- watchmedo shell-command --drop --patterns="*.gc;*.jsonc" --recursive --command='task decomp-file FILES="{{.FILES}}"' ./decompiler/config/
# TODO - the below are broken, need to be updated or replaced
# decomp-file:
# cmds:
# - python ./scripts/next-decomp-file.py --files "{{.FILES}}"
# - task: decomp
# decomp-list:
# cmds:
# - python ./scripts/next-decomp-file.py --list "{{.LIST}}"
# vars:
# LIST: '{{default "0" .LIST}}'
# # python -m pip install -U watchdog[watchmedo]
# decomp-watch:
# cmds:
# - watchmedo shell-command --drop --patterns="*.gc;*.jsonc" --recursive --command='task decomp-file FILES="{{.FILES}}"' ./decompiler/config/
# update-gsrc:
# cmds:
# - python ./scripts/next-decomp-file.py --files "{{.FILES}}"
# - task: decomp
# - task: find-label-types
# - python ./scripts/update-goal-src.py --files "{{.FILES}}"
# - task: type-test
# - task: check-gsrc-file
# TOOLS
# TODO - broken!
# analyze-ee-memory:
# cmds:
# - '{{.MEMDUMP_BIN_RELEASE_DIR}}/memory_dump_tool "{{.FILE}}" ./ > ee-analysis.log'
# watch-pcsx2:
# cmds:
# - watchmedo shell-command --drop --patterns="*.p2s" --recursive --command='task analyze-ee-memory FILE="${watch_src_path}"' "{{.SAVESTATE_DIR}}"
# vars:
# SAVESTATE_DIR: '{{default "." .SAVESTATE_DIR}}'
# TESTS
offline-tests:
cmds:
- '{{.OFFLINETEST_BIN_RELEASE_DIR}}/offline-test "./iso_data/jak1"'
add-reference-test:
cmds:
- task: decomp-file
- python ./scripts/add-reference-test.py --file "{{.FILES}}"
- task: offline-tests
add-reference-test-no-decomp:
cmds:
- python ./scripts/add-reference-test.py --file "{{.FILES}}"
- task: offline-tests
update-reference-tests:
cmds:
- cmd: python ./scripts/default-file-or-folder.py --path failures
- cmd: '{{.OFFLINETEST_BIN_RELEASE_DIR}}/offline-test "./iso_data/jak1" --dump-mode'
ignore_error: true
- python ./scripts/update_decomp_reference.py ./failures ./test/decompiler/reference/
- task: offline-tests
find-label-types:
cmds:
- python ./scripts/next-decomp-file.py --files "{{.FILES}}"
- task: decomp
- python ./scripts/find-label-types.py --file "{{.FILES}}"
- '{{.OFFLINETEST_BIN_RELEASE_DIR}}/offline-test "./iso_data/{{.GAME}}"'
# TODO - update or replace
# add-reference-test:
# cmds:
# - task: decomp-file
# - python ./scripts/add-reference-test.py --file "{{.FILES}}"
# - task: offline-tests
# add-reference-test-no-decomp:
# cmds:
# - python ./scripts/add-reference-test.py --file "{{.FILES}}"
# - task: offline-tests
# update-reference-tests:
# cmds:
# - cmd: python ./scripts/default-file-or-folder.py --path failures
# - cmd: '{{.OFFLINETEST_BIN_RELEASE_DIR}}/offline-test "./iso_data/{{.GAME}}" --dump-mode'
# ignore_error: true
# - python ./scripts/update_decomp_reference.py ./failures ./test/decompiler/reference/
# - task: offline-tests
# find-label-types:
# cmds:
# - python ./scripts/next-decomp-file.py --files "{{.FILES}}"
# - task: decomp
# - python ./scripts/find-label-types.py --file "{{.FILES}}"
# check-gsrc-file:
# cmds:
# - python ./scripts/check-gsrc-file.py --files "{{.FILES}}"
type-test:
cmds:
- cmd: '{{.GOALCTEST_BIN_RELEASE_DIR}}/goalc-test --gtest_brief=0 --gtest_filter="*MANUAL_TEST_TypeConsistencyWithBuildFirst*"'
ignore_error: true
check-gsrc-file:
cmds:
- python ./scripts/check-gsrc-file.py --files "{{.FILES}}"
# TOOLS
analyze-ee-memory:
cmds:
- '{{.MEMDUMP_BIN_RELEASE_DIR}}/memory_dump_tool "{{.FILE}}" ./ > ee-analysis.log'
watch-pcsx2:
cmds:
- watchmedo shell-command --drop --patterns="*.p2s" --recursive --command='task analyze-ee-memory FILE="${watch_src_path}"' "{{.SAVESTATE_DIR}}"
vars:
SAVESTATE_DIR: '{{default "." .SAVESTATE_DIR}}'
update-gsrc:
cmds:
- python ./scripts/next-decomp-file.py --files "{{.FILES}}"
- task: decomp
- task: find-label-types
- python ./scripts/update-goal-src.py --files "{{.FILES}}"
- task: type-test
- task: check-gsrc-file
cast-repl:
cmds:
- cmd: python ./scripts/cast-repl.py

View file

@ -1,188 +0,0 @@
from prompt_toolkit import PromptSession
from prompt_toolkit.history import FileHistory
from jsmin import jsmin
import shlex
import json
import collections
stack_cast_file_path = "./decompiler/config/jak1_ntsc_black_label/stack_structures.jsonc"
type_cast_file_path = "./decompiler/config/jak1_ntsc_black_label/type_casts.jsonc"
lambda_cast_file_path = "./decompiler/config/jak1_ntsc_black_label/anonymous_function_types.jsonc"
global_file_name = ""
global_func_name = ""
def ordered_dict_insert(ordered_dict, index, key, value):
if key in ordered_dict:
raise KeyError("Key already exists")
if index < 0 or index > len(ordered_dict):
raise IndexError("Index out of range")
keys = list(ordered_dict.keys())[index:]
ordered_dict[key] = value
for k in keys:
ordered_dict.move_to_end(k)
# TODO - optional size
def stack_cast(function_name, type_name, offset):
with open(stack_cast_file_path, "r+") as config_file:
minified = jsmin(config_file.read())
json_data = json.JSONDecoder(object_pairs_hook=collections.OrderedDict).decode(minified)
# Check if the function name has already been added
if function_name not in json_data:
ordered_dict_insert(json_data, len(json_data)-1, function_name, [])
# Check if there already exists a cast with the same offset, if so replace it
casts = json_data[function_name]
for cast in casts:
if cast[0] == offset:
print("Found cast with the same offset, replacing!")
cast[1] = type_name
config_file.seek(0)
json.dump(json_data, config_file, indent=2)
config_file.truncate()
return
json_data[function_name].append([offset, type_name])
config_file.seek(0)
json.dump(json_data, config_file, indent=2)
config_file.truncate()
print("Stack Cast Applied!")
def type_cast(function_name, register, type_cast, cast_range):
with open(type_cast_file_path, "r+") as config_file:
minified = jsmin(config_file.read())
json_data = json.JSONDecoder(object_pairs_hook=collections.OrderedDict).decode(minified)
# Check if the function name has already been added
if function_name not in json_data:
ordered_dict_insert(json_data, len(json_data)-1, function_name, [])
range_start = -1
range_end = -1
if "-" in cast_range:
range_start = int(cast_range.split("-")[0])
range_end = int(cast_range.split("-")[1])
else:
range_start = int(cast_range)
# Check if there already exists a cast with the same offset, if so replace it
casts = json_data[function_name]
new_casts = []
for cast in casts:
cast_range_start = -1
cast_range_end = -1
if isinstance(cast[0], list):
cast_range_start = cast[0][0]
cast_range_end = cast[0][1]
else:
cast_range_start = cast[0]
cast_register = cast[1]
if range_end == -1 and cast_range_end == -1 and range_start == cast_range_start and register == cast_register:
print("Found cast with the same range, replacing!")
continue
elif range_end == -1:
if range_start >= cast_range_start and range_start <= cast_range_end and register == cast_register:
print("Found cast with the same range, replacing!")
continue
elif cast_range_end == -1:
if cast_range_start >= range_start and cast_range_start <= range_end and register == cast_register:
print("Found cast with the same range, replacing!")
continue
new_casts.append(cast)
json_data[function_name] = new_casts
if range_end == -1:
json_data[function_name].append([range_start, register, type_cast])
else:
json_data[function_name].append([[range_start, range_end], register, type_cast])
config_file.seek(0)
json.dump(json_data, config_file, indent=2)
config_file.truncate()
print("Type Cast Applied!")
def lambda_cast(file_name, func_sig, func_id):
with open(lambda_cast_file_path, "r+") as config_file:
minified = jsmin(config_file.read())
json_data = json.JSONDecoder(object_pairs_hook=collections.OrderedDict).decode(minified)
# Check if the function name has already been added
if file_name not in json_data:
ordered_dict_insert(json_data, len(json_data)-1, file_name, [])
# Check if there already exists a cast with the same offset, if so replace it
functions = json_data[file_name]
for func in functions:
if func[0] == func_id:
print("Found lambda with the same id, replacing!")
func[1] = func_sig
config_file.seek(0)
json.dump(json_data, config_file, indent=2)
config_file.truncate()
return
json_data[file_name].append([func_id, func_sig])
config_file.seek(0)
json.dump(json_data, config_file, indent=2)
config_file.truncate()
print("Lambda Cast Applied!")
def main():
global global_file_name
global global_func_name
from pathlib import Path
home = str(Path.home())
session = PromptSession(history=FileHistory('{}/.castReplHistory'.format(home)))
# LOOP
while True:
# READ
try:
prompt_string = "castREPL> "
if global_file_name and global_func_name:
prompt_string = "castREPL-{}-{}> ".format(global_file_name, global_func_name)
elif global_file_name:
prompt_string = "castREPL-{}> ".format(global_file_name)
elif global_func_name:
prompt_string = "castREPL-{}> ".format(global_func_name)
text = session.prompt(prompt_string)
except KeyboardInterrupt:
continue
except EOFError:
break
# TODO - help command
# TODO - command to list current casts
# TODO - command to delete casts
# TODO - command to lookup function signature in all-types
# TODO - command to tie into the C++ program I'll right to narrow down an unknown type
# TODO - command to define function signature in all-types
# EVAL / CAST / PRINT
tokens = shlex.split(text)
if len(tokens) < 1:
continue
# PROCESS COMMANDS
command = tokens[0]
# Allows you to persist a file name -- cutting down on command args (useful if working on a big file)
if command == "enter-file" and len(tokens) == 2:
global_file_name = tokens[1]
elif command == "exit-file":
global_file_name = ""
# Allows you to persist a function name -- cutting down on command args (useful if working on a big function)
elif command == "enter-func" and len(tokens) == 2:
global_func_name = tokens[1]
elif command == "exit-func":
global_func_name = ""
elif command == "stack" and len(tokens) == 4:
# Stack casts are in the following format stack <func> <type> <offset>
stack_cast(tokens[1], tokens[2], int(tokens[3]))
elif command == "lambda" and len(tokens) == 4:
# Stack casts are in the following format lambda <file_name> <id> <func_sig>
lambda_cast(tokens[1], tokens[2], int(tokens[3]))
elif command == "type" and len(tokens) == 5:
# Stack casts are in the following format type <func> <register> <type_cast> <range = X | X-Y>
type_cast(tokens[1], tokens[2], tokens[3], tokens[4])
else:
print("Invalid Cast Command!")
# Exit
print('GoodBye!')
if __name__ == '__main__':
main()

View file

@ -0,0 +1,2 @@
GAME=jak1
DECOMP_CONFIG=jak1_ntsc_black_label.jsonc

View file

@ -1,4 +1,4 @@
version: '3'
version: "3.13"
vars:
GOALC_BIN_RELEASE_DIR: './build/goalc'

View file

@ -1,4 +1,4 @@
version: '3'
version: "3.13"
vars:
GOALC_BIN_RELEASE_DIR: './build/goalc'

View file

@ -1,4 +1,4 @@
version: '3'
version: "3.13"
vars:
GOALC_BIN_RELEASE_DIR: './out/build/Release/bin'

View file

@ -0,0 +1,9 @@
$scriptBlock = {
param($commandName, $wordToComplete, $cursorPosition)
$regex = "task(?:.exe)? (.*)$"
$startsWith = $wordToComplete | Select-String $regex -AllMatches | ForEach-Object { $_.Matches.Groups[1].Value }
$listOutput = $(task --list-all --silent)
$listOutput | Where-Object {$_ -like "$startsWith*"}
}
Register-ArgumentCompleter -Native -CommandName task -ScriptBlock $scriptBlock

View file

@ -0,0 +1,62 @@
import argparse
import os
import pprint
import sys
parser = argparse.ArgumentParser("update-env")
parser.add_argument("--game", help="The name of the game", type=str)
parser.add_argument("--decomp_config", help="The decompiler config file", type=str)
args = parser.parse_args()
# TODO - read from defaults
file = {
"GAME": "jak1",
"DECOMP_CONFIG": "jak1_ntsc_black_label.jsonc"
}
env_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), ".env")
if not os.path.exists(env_path):
with open(env_path, 'w') as env_file:
for item in file.items():
env_file.write("{}={}\n".format(item[0], item[1]))
with open(env_path, 'r') as env_file:
flags = env_file.readlines()
for flag in flags:
tokens = flag.split("=")
if tokens[0] in file:
file[tokens[0]] = tokens[1].strip()
valid_games = ["jak1", "jak2"]
decomp_config_map = {
"jak1": {
"ntscv1": "jak1_ntsc_black_label.jsonc",
"ntscv2": "jak1_us2.jsonc",
"pal": "jak1_pal.jsonc",
"ntscjp": "jak1_jp.jsonc"
},
"jak2": {
"ntscv1": "jak2_ntsc_v1.jsonc"
}
}
if args.game:
if args.game not in valid_games:
print("Unsupported game '{}'".format(args.game))
sys.exit(1)
file["GAME"] = args.game
if args.decomp_config:
if args.decomp_config not in decomp_config_map[file["GAME"]]:
print("Unsupported decomp config '{}' for game '{}'".format(args.decomp_config, file["GAME"]))
sys.exit(1)
file["DECOMP_CONFIG"] = decomp_config_map[file["GAME"]][args.decomp_config]
with open(env_path, 'w') as env_file:
for item in file.items():
env_file.write("{}={}\n".format(item[0], item[1]))
print("Task settings updated")
pp = pprint.PrettyPrinter(indent=2)
pp.pprint(file)