diff --git a/.gitignore b/.gitignore index bd3a231d0..0c6acf012 100644 --- a/.gitignore +++ b/.gitignore @@ -27,6 +27,7 @@ linux-default/ savestate-out/ failures/ ee-results.json +.env # graphics debug debug_out/* diff --git a/README.md b/README.md index 259f103ae..2797811e6 100644 --- a/README.md +++ b/README.md @@ -12,16 +12,14 @@ PRs

-## Table of Contents - - - - [Project Description](#project-description) + - [**Please read the FAQ by clicking here if you have any questions.**](#please-read-the-faq-by-clicking-here-if-you-have-any-questions) - [Current Status](#current-status) - [What's Next](#whats-next) - [Getting Started - Linux](#getting-started---linux) - [Ubuntu (20.04)](#ubuntu-2004) - [Arch](#arch) + - [Fedora](#fedora) - [Getting Started - Windows](#getting-started---windows) - [Required Software](#required-software) - [Setting up and Opening the Project](#setting-up-and-opening-the-project) @@ -35,8 +33,6 @@ - [Project Layout](#project-layout) - [Directory Layout](#directory-layout) - - ## Project Description This project is to port Jak 1 (NTSC, "black label" version) to PC. Over 98% of this game is written in GOAL, a custom Lisp language developed by Naughty Dog. Our strategy is: @@ -172,7 +168,6 @@ Once Scoop is installed, run the following commands: ```sh scoop install git llvm nasm python -scoop bucket add extras scoop install task ``` @@ -204,12 +199,23 @@ Getting a running game involves 4 steps: ### Extract Assets +First, setup your settings so the following scripts know which game you are using, and which version. In a terminal, run the following: + +```sh +task set-game-jak1 +task set-decomp-ntscv1 +``` + +> Run `task --list` to see the other available options + +> At the time of writing, only Jak 1 is expected to work end-to-end! + The first step is to extract your ISO file contents into the `iso_data/` folder. In the case of Jak 1 this is `iso_data/jak1`. Once this is done, open a terminal in the `jak-project` folder and run the following: ```sh -task extract-jak1 +task extract ``` ### Build the Game diff --git a/Taskfile.yml b/Taskfile.yml index c471e3524..fea6e5bc2 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -1,48 +1,63 @@ -version: '3' +version: "3.13" includes: build: ./scripts/tasks/Taskfile_{{OS}}.yml +dotenv: + - ./scripts/tasks/.env + - ./scripts/tasks/.env.default + tasks: - # TODO - make it easy to switch between games instead of having a bunch of "jak1/jak2" varients + # SETTINGS / CONFIGURATION + set-game-jak1: + - 'python ./scripts/tasks/update-env.py --game jak1' + set-game-jak2: + - 'python ./scripts/tasks/update-env.py --game jak2' + set-decomp-ntscv1: + desc: "aka black label" + cmds: + - 'python ./scripts/tasks/update-env.py --decomp_config ntscv1' + set-decomp-ntscv2: + desc: "aka red label" + cmds: + - 'python ./scripts/tasks/update-env.py --decomp_config ntscv2' + set-decomp-pal: + - 'python ./scripts/tasks/update-env.py --decomp_config pal' + set-decomp-ntscjp: + - 'python ./scripts/tasks/update-env.py --decomp_config ntscjp' # GENERAL - extract-jak1: - desc: "Extracts Jak 1 - NTSC - Black Label assets" + extract: + desc: "Extracts the game's assets from './iso_data' with the set decompiler config" preconditions: - sh: test -f {{.DECOMP_BIN_RELEASE_DIR}}/decompiler{{.EXE_FILE_EXTENSION}} msg: "Couldn't locate decompiler executable in '{{.DECOMP_BIN_RELEASE_DIR}}/decompiler'" cmds: - - '{{.DECOMP_BIN_RELEASE_DIR}}/decompiler "./decompiler/config/jak1_ntsc_black_label.jsonc" "./iso_data" "./decompiler_out" "decompile_code=false"' + - '{{.DECOMP_BIN_RELEASE_DIR}}/decompiler "./decompiler/config/{{.DECOMP_CONFIG}}" "./iso_data" "./decompiler_out" "decompile_code=false"' boot-game: - desc: "Boots the game" + desc: "Boots the game, it will fail if it's not already booted!" preconditions: - sh: test -f {{.GK_BIN_RELEASE_DIR}}/gk{{.EXE_FILE_EXTENSION}} msg: "Couldn't locate runtime executable in '{{.GK_BIN_RELEASE_DIR}}/gk'" cmds: - "{{.GK_BIN_RELEASE_DIR}}/gk -boot -fakeiso -debug -v" run-game: - desc: "Start the game's runtime" + desc: "Start the game's runtime, to start the game itself the REPL is required" preconditions: - sh: test -f {{.GK_BIN_RELEASE_DIR}}/gk{{.EXE_FILE_EXTENSION}} msg: "Couldn't locate runtime executable in '{{.GK_BIN_RELEASE_DIR}}/gk'" cmds: - "{{.GK_BIN_RELEASE_DIR}}/gk -fakeiso -debug -v" - run-game-quiet: - preconditions: - - sh: test -f {{.GK_BIN_RELEASE_DIR}}/gk{{.EXE_FILE_EXTENSION}} - msg: "Couldn't locate runtime executable in '{{.GK_BIN_RELEASE_DIR}}/gk'" - cmds: - - "{{.GK_BIN_RELEASE_DIR}}/gk -fakeiso" + # DEVELOPMENT repl: desc: "Start the REPL" - env: - OPENGOAL_DECOMP_DIR: "jak1/" preconditions: - sh: test -f {{.GOALC_BIN_RELEASE_DIR}}/goalc{{.EXE_FILE_EXTENSION}} msg: "Couldn't locate compiler executable in '{{.GOALC_BIN_RELEASE_DIR}}/goalc'" cmds: - "{{.GOALC_BIN_RELEASE_DIR}}/goalc" - # DEVELOPMENT + repl-lt: + cmds: + - "{{.GOALC_BIN_RELEASE_DIR}}/goalc --auto-lt" format: desc: "Format code" cmds: @@ -53,84 +68,76 @@ tasks: run-game-headless: cmds: - "{{.GK_BIN_RELEASE_DIR}}/gk -fakeiso -debug -nodisplay" - repl-lt: - env: - OPENGOAL_DECOMP_DIR: "jak1/" - cmds: - - "{{.GOALC_BIN_RELEASE_DIR}}/goalc --auto-lt" # DECOMPILING decomp: cmds: - - '{{.DECOMP_BIN_RELEASE_DIR}}/decompiler "./decompiler/config/jak1_ntsc_black_label.jsonc" "./iso_data" "./decompiler_out"' - decomp-jak2: - cmds: - - '{{.DECOMP_BIN_RELEASE_DIR}}/decompiler "./decompiler/config/jak2_ntsc_v1.jsonc" "./iso_data" "./decompiler_out"' + - '{{.DECOMP_BIN_RELEASE_DIR}}/decompiler "./decompiler/config/{{.DECOMP_CONFIG}}" "./iso_data" "./decompiler_out"' decomp-clean: cmds: - rm ./decompiler_out/**/*.asm - rm ./decompiler_out/**/*disasm.gc - decomp-file: - cmds: - - python ./scripts/next-decomp-file.py --files "{{.FILES}}" - - task: decomp - decomp-list: - cmds: - - python ./scripts/next-decomp-file.py --list "{{.LIST}}" - vars: - LIST: '{{default "0" .LIST}}' - # python -m pip install -U watchdog[watchmedo] - decomp-watch: - cmds: - - watchmedo shell-command --drop --patterns="*.gc;*.jsonc" --recursive --command='task decomp-file FILES="{{.FILES}}"' ./decompiler/config/ + # TODO - the below are broken, need to be updated or replaced + # decomp-file: + # cmds: + # - python ./scripts/next-decomp-file.py --files "{{.FILES}}" + # - task: decomp + # decomp-list: + # cmds: + # - python ./scripts/next-decomp-file.py --list "{{.LIST}}" + # vars: + # LIST: '{{default "0" .LIST}}' + # # python -m pip install -U watchdog[watchmedo] + # decomp-watch: + # cmds: + # - watchmedo shell-command --drop --patterns="*.gc;*.jsonc" --recursive --command='task decomp-file FILES="{{.FILES}}"' ./decompiler/config/ + # update-gsrc: + # cmds: + # - python ./scripts/next-decomp-file.py --files "{{.FILES}}" + # - task: decomp + # - task: find-label-types + # - python ./scripts/update-goal-src.py --files "{{.FILES}}" + # - task: type-test + # - task: check-gsrc-file + # TOOLS + # TODO - broken! + # analyze-ee-memory: + # cmds: + # - '{{.MEMDUMP_BIN_RELEASE_DIR}}/memory_dump_tool "{{.FILE}}" ./ > ee-analysis.log' + # watch-pcsx2: + # cmds: + # - watchmedo shell-command --drop --patterns="*.p2s" --recursive --command='task analyze-ee-memory FILE="${watch_src_path}"' "{{.SAVESTATE_DIR}}" + # vars: + # SAVESTATE_DIR: '{{default "." .SAVESTATE_DIR}}' # TESTS offline-tests: cmds: - - '{{.OFFLINETEST_BIN_RELEASE_DIR}}/offline-test "./iso_data/jak1"' - add-reference-test: - cmds: - - task: decomp-file - - python ./scripts/add-reference-test.py --file "{{.FILES}}" - - task: offline-tests - add-reference-test-no-decomp: - cmds: - - python ./scripts/add-reference-test.py --file "{{.FILES}}" - - task: offline-tests - update-reference-tests: - cmds: - - cmd: python ./scripts/default-file-or-folder.py --path failures - - cmd: '{{.OFFLINETEST_BIN_RELEASE_DIR}}/offline-test "./iso_data/jak1" --dump-mode' - ignore_error: true - - python ./scripts/update_decomp_reference.py ./failures ./test/decompiler/reference/ - - task: offline-tests - find-label-types: - cmds: - - python ./scripts/next-decomp-file.py --files "{{.FILES}}" - - task: decomp - - python ./scripts/find-label-types.py --file "{{.FILES}}" + - '{{.OFFLINETEST_BIN_RELEASE_DIR}}/offline-test "./iso_data/{{.GAME}}"' + # TODO - update or replace + # add-reference-test: + # cmds: + # - task: decomp-file + # - python ./scripts/add-reference-test.py --file "{{.FILES}}" + # - task: offline-tests + # add-reference-test-no-decomp: + # cmds: + # - python ./scripts/add-reference-test.py --file "{{.FILES}}" + # - task: offline-tests + # update-reference-tests: + # cmds: + # - cmd: python ./scripts/default-file-or-folder.py --path failures + # - cmd: '{{.OFFLINETEST_BIN_RELEASE_DIR}}/offline-test "./iso_data/{{.GAME}}" --dump-mode' + # ignore_error: true + # - python ./scripts/update_decomp_reference.py ./failures ./test/decompiler/reference/ + # - task: offline-tests + # find-label-types: + # cmds: + # - python ./scripts/next-decomp-file.py --files "{{.FILES}}" + # - task: decomp + # - python ./scripts/find-label-types.py --file "{{.FILES}}" + # check-gsrc-file: + # cmds: + # - python ./scripts/check-gsrc-file.py --files "{{.FILES}}" type-test: cmds: - cmd: '{{.GOALCTEST_BIN_RELEASE_DIR}}/goalc-test --gtest_brief=0 --gtest_filter="*MANUAL_TEST_TypeConsistencyWithBuildFirst*"' ignore_error: true - check-gsrc-file: - cmds: - - python ./scripts/check-gsrc-file.py --files "{{.FILES}}" - # TOOLS - analyze-ee-memory: - cmds: - - '{{.MEMDUMP_BIN_RELEASE_DIR}}/memory_dump_tool "{{.FILE}}" ./ > ee-analysis.log' - watch-pcsx2: - cmds: - - watchmedo shell-command --drop --patterns="*.p2s" --recursive --command='task analyze-ee-memory FILE="${watch_src_path}"' "{{.SAVESTATE_DIR}}" - vars: - SAVESTATE_DIR: '{{default "." .SAVESTATE_DIR}}' - update-gsrc: - cmds: - - python ./scripts/next-decomp-file.py --files "{{.FILES}}" - - task: decomp - - task: find-label-types - - python ./scripts/update-goal-src.py --files "{{.FILES}}" - - task: type-test - - task: check-gsrc-file - cast-repl: - cmds: - - cmd: python ./scripts/cast-repl.py diff --git a/scripts/cast-repl.py b/scripts/cast-repl.py deleted file mode 100644 index f4b5d033e..000000000 --- a/scripts/cast-repl.py +++ /dev/null @@ -1,188 +0,0 @@ -from prompt_toolkit import PromptSession -from prompt_toolkit.history import FileHistory -from jsmin import jsmin -import shlex -import json -import collections - -stack_cast_file_path = "./decompiler/config/jak1_ntsc_black_label/stack_structures.jsonc" -type_cast_file_path = "./decompiler/config/jak1_ntsc_black_label/type_casts.jsonc" -lambda_cast_file_path = "./decompiler/config/jak1_ntsc_black_label/anonymous_function_types.jsonc" - -global_file_name = "" -global_func_name = "" - -def ordered_dict_insert(ordered_dict, index, key, value): - if key in ordered_dict: - raise KeyError("Key already exists") - if index < 0 or index > len(ordered_dict): - raise IndexError("Index out of range") - - keys = list(ordered_dict.keys())[index:] - ordered_dict[key] = value - for k in keys: - ordered_dict.move_to_end(k) - -# TODO - optional size -def stack_cast(function_name, type_name, offset): - with open(stack_cast_file_path, "r+") as config_file: - minified = jsmin(config_file.read()) - json_data = json.JSONDecoder(object_pairs_hook=collections.OrderedDict).decode(minified) - # Check if the function name has already been added - if function_name not in json_data: - ordered_dict_insert(json_data, len(json_data)-1, function_name, []) - # Check if there already exists a cast with the same offset, if so replace it - casts = json_data[function_name] - for cast in casts: - if cast[0] == offset: - print("Found cast with the same offset, replacing!") - cast[1] = type_name - config_file.seek(0) - json.dump(json_data, config_file, indent=2) - config_file.truncate() - return - json_data[function_name].append([offset, type_name]) - config_file.seek(0) - json.dump(json_data, config_file, indent=2) - config_file.truncate() - print("Stack Cast Applied!") - -def type_cast(function_name, register, type_cast, cast_range): - with open(type_cast_file_path, "r+") as config_file: - minified = jsmin(config_file.read()) - json_data = json.JSONDecoder(object_pairs_hook=collections.OrderedDict).decode(minified) - # Check if the function name has already been added - if function_name not in json_data: - ordered_dict_insert(json_data, len(json_data)-1, function_name, []) - - range_start = -1 - range_end = -1 - if "-" in cast_range: - range_start = int(cast_range.split("-")[0]) - range_end = int(cast_range.split("-")[1]) - else: - range_start = int(cast_range) - - # Check if there already exists a cast with the same offset, if so replace it - casts = json_data[function_name] - new_casts = [] - for cast in casts: - cast_range_start = -1 - cast_range_end = -1 - if isinstance(cast[0], list): - cast_range_start = cast[0][0] - cast_range_end = cast[0][1] - else: - cast_range_start = cast[0] - cast_register = cast[1] - - if range_end == -1 and cast_range_end == -1 and range_start == cast_range_start and register == cast_register: - print("Found cast with the same range, replacing!") - continue - elif range_end == -1: - if range_start >= cast_range_start and range_start <= cast_range_end and register == cast_register: - print("Found cast with the same range, replacing!") - continue - elif cast_range_end == -1: - if cast_range_start >= range_start and cast_range_start <= range_end and register == cast_register: - print("Found cast with the same range, replacing!") - continue - new_casts.append(cast) - - json_data[function_name] = new_casts - if range_end == -1: - json_data[function_name].append([range_start, register, type_cast]) - else: - json_data[function_name].append([[range_start, range_end], register, type_cast]) - config_file.seek(0) - json.dump(json_data, config_file, indent=2) - config_file.truncate() - print("Type Cast Applied!") - -def lambda_cast(file_name, func_sig, func_id): - with open(lambda_cast_file_path, "r+") as config_file: - minified = jsmin(config_file.read()) - json_data = json.JSONDecoder(object_pairs_hook=collections.OrderedDict).decode(minified) - # Check if the function name has already been added - if file_name not in json_data: - ordered_dict_insert(json_data, len(json_data)-1, file_name, []) - # Check if there already exists a cast with the same offset, if so replace it - functions = json_data[file_name] - for func in functions: - if func[0] == func_id: - print("Found lambda with the same id, replacing!") - func[1] = func_sig - config_file.seek(0) - json.dump(json_data, config_file, indent=2) - config_file.truncate() - return - json_data[file_name].append([func_id, func_sig]) - config_file.seek(0) - json.dump(json_data, config_file, indent=2) - config_file.truncate() - print("Lambda Cast Applied!") - -def main(): - global global_file_name - global global_func_name - - from pathlib import Path - home = str(Path.home()) - session = PromptSession(history=FileHistory('{}/.castReplHistory'.format(home))) - - # LOOP - while True: - # READ - try: - prompt_string = "castREPL> " - if global_file_name and global_func_name: - prompt_string = "castREPL-{}-{}> ".format(global_file_name, global_func_name) - elif global_file_name: - prompt_string = "castREPL-{}> ".format(global_file_name) - elif global_func_name: - prompt_string = "castREPL-{}> ".format(global_func_name) - text = session.prompt(prompt_string) - except KeyboardInterrupt: - continue - except EOFError: - break - - # TODO - help command - # TODO - command to list current casts - # TODO - command to delete casts - # TODO - command to lookup function signature in all-types - # TODO - command to tie into the C++ program I'll right to narrow down an unknown type - # TODO - command to define function signature in all-types - - # EVAL / CAST / PRINT - tokens = shlex.split(text) - if len(tokens) < 1: - continue - # PROCESS COMMANDS - command = tokens[0] - # Allows you to persist a file name -- cutting down on command args (useful if working on a big file) - if command == "enter-file" and len(tokens) == 2: - global_file_name = tokens[1] - elif command == "exit-file": - global_file_name = "" - # Allows you to persist a function name -- cutting down on command args (useful if working on a big function) - elif command == "enter-func" and len(tokens) == 2: - global_func_name = tokens[1] - elif command == "exit-func": - global_func_name = "" - elif command == "stack" and len(tokens) == 4: - # Stack casts are in the following format stack - stack_cast(tokens[1], tokens[2], int(tokens[3])) - elif command == "lambda" and len(tokens) == 4: - # Stack casts are in the following format lambda - lambda_cast(tokens[1], tokens[2], int(tokens[3])) - elif command == "type" and len(tokens) == 5: - # Stack casts are in the following format type - type_cast(tokens[1], tokens[2], tokens[3], tokens[4]) - else: - print("Invalid Cast Command!") - # Exit - print('GoodBye!') - -if __name__ == '__main__': - main() diff --git a/scripts/tasks/.env.default b/scripts/tasks/.env.default new file mode 100644 index 000000000..f2865201d --- /dev/null +++ b/scripts/tasks/.env.default @@ -0,0 +1,2 @@ +GAME=jak1 +DECOMP_CONFIG=jak1_ntsc_black_label.jsonc diff --git a/scripts/tasks/Taskfile_darwin.yml b/scripts/tasks/Taskfile_darwin.yml index 0131351b9..cd2ffd928 100644 --- a/scripts/tasks/Taskfile_darwin.yml +++ b/scripts/tasks/Taskfile_darwin.yml @@ -1,4 +1,4 @@ -version: '3' +version: "3.13" vars: GOALC_BIN_RELEASE_DIR: './build/goalc' diff --git a/scripts/tasks/Taskfile_linux.yml b/scripts/tasks/Taskfile_linux.yml index 0131351b9..cd2ffd928 100644 --- a/scripts/tasks/Taskfile_linux.yml +++ b/scripts/tasks/Taskfile_linux.yml @@ -1,4 +1,4 @@ -version: '3' +version: "3.13" vars: GOALC_BIN_RELEASE_DIR: './build/goalc' diff --git a/scripts/tasks/Taskfile_windows.yml b/scripts/tasks/Taskfile_windows.yml index 613e9ec7d..1ec940b20 100644 --- a/scripts/tasks/Taskfile_windows.yml +++ b/scripts/tasks/Taskfile_windows.yml @@ -1,4 +1,4 @@ -version: '3' +version: "3.13" vars: GOALC_BIN_RELEASE_DIR: './out/build/Release/bin' diff --git a/scripts/tasks/completions/task.ps1 b/scripts/tasks/completions/task.ps1 new file mode 100644 index 000000000..0f6f2d4a1 --- /dev/null +++ b/scripts/tasks/completions/task.ps1 @@ -0,0 +1,9 @@ +$scriptBlock = { + param($commandName, $wordToComplete, $cursorPosition) + $regex = "task(?:.exe)? (.*)$" + $startsWith = $wordToComplete | Select-String $regex -AllMatches | ForEach-Object { $_.Matches.Groups[1].Value } + $listOutput = $(task --list-all --silent) + $listOutput | Where-Object {$_ -like "$startsWith*"} +} + +Register-ArgumentCompleter -Native -CommandName task -ScriptBlock $scriptBlock diff --git a/scripts/tasks/update-env.py b/scripts/tasks/update-env.py new file mode 100644 index 000000000..fe29bb9b7 --- /dev/null +++ b/scripts/tasks/update-env.py @@ -0,0 +1,62 @@ +import argparse +import os +import pprint +import sys + +parser = argparse.ArgumentParser("update-env") +parser.add_argument("--game", help="The name of the game", type=str) +parser.add_argument("--decomp_config", help="The decompiler config file", type=str) +args = parser.parse_args() + +# TODO - read from defaults +file = { + "GAME": "jak1", + "DECOMP_CONFIG": "jak1_ntsc_black_label.jsonc" +} + +env_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), ".env") + +if not os.path.exists(env_path): + with open(env_path, 'w') as env_file: + for item in file.items(): + env_file.write("{}={}\n".format(item[0], item[1])) + +with open(env_path, 'r') as env_file: + flags = env_file.readlines() + for flag in flags: + tokens = flag.split("=") + if tokens[0] in file: + file[tokens[0]] = tokens[1].strip() + +valid_games = ["jak1", "jak2"] + +decomp_config_map = { + "jak1": { + "ntscv1": "jak1_ntsc_black_label.jsonc", + "ntscv2": "jak1_us2.jsonc", + "pal": "jak1_pal.jsonc", + "ntscjp": "jak1_jp.jsonc" + }, + "jak2": { + "ntscv1": "jak2_ntsc_v1.jsonc" + } +} + +if args.game: + if args.game not in valid_games: + print("Unsupported game '{}'".format(args.game)) + sys.exit(1) + file["GAME"] = args.game +if args.decomp_config: + if args.decomp_config not in decomp_config_map[file["GAME"]]: + print("Unsupported decomp config '{}' for game '{}'".format(args.decomp_config, file["GAME"])) + sys.exit(1) + file["DECOMP_CONFIG"] = decomp_config_map[file["GAME"]][args.decomp_config] + +with open(env_path, 'w') as env_file: + for item in file.items(): + env_file.write("{}={}\n".format(item[0], item[1])) + +print("Task settings updated") +pp = pprint.PrettyPrinter(indent=2) +pp.pprint(file)