From 2a315419de1b7484d00dd6a075993e9c2087d754 Mon Sep 17 00:00:00 2001 From: Tyler Wilding Date: Sun, 9 May 2021 17:03:58 -0400 Subject: [PATCH] tests: Automate the offline reference tests better (#427) * tests: Move all files to new directories * scripts: Update decomp scripts * tests: Remove hard-coded list for offline tests * linting --- Taskfile.yml | 15 ++- decompiler/config/jak1_ntsc_black_label.jsonc | 2 +- .../build/{all_objs.txt => all_objs.json} | 0 .../build/{game_dgos.txt => game_dgos.gc} | 0 .../build/{kernel_dgos.txt => kernel_dgos.gc} | 0 goal_src/goal-lib.gc | 4 +- scripts/next-decomp-file.py | 77 +++++++---- .../{ => engine/anim}/mspace-h_REF.gc | 0 .../{ => engine/camera}/math-camera-h_REF.gc | 0 .../{ => engine/camera}/pov-camera-h_REF.gc | 0 .../{ => engine/debug}/memory-usage-h_REF.gc | 0 .../{ => engine/dma}/dma-bucket_REF.gc | 0 .../{ => engine/dma}/dma-buffer_REF.gc | 0 .../{ => engine/dma}/dma-disasm_REF.gc | 0 .../reference/{ => engine/dma}/dma-h_REF.gc | 0 .../reference/{ => engine/dma}/dma_REF.gc | 0 .../{ => engine/draw}/draw-node-h_REF.gc | 0 .../{ => engine/draw}/drawable-actor-h_REF.gc | 0 .../draw}/drawable-ambient-h_REF.gc | 0 .../{ => engine/draw}/drawable-group-h_REF.gc | 0 .../{ => engine/draw}/drawable-h_REF.gc | 0 .../draw}/drawable-inline-array-h_REF.gc | 0 .../{ => engine/draw}/drawable-tree-h_REF.gc | 0 .../{ => engine/engine}/connect_REF.gc | 0 .../reference/{ => engine/game}/main-h_REF.gc | 0 .../{ => engine/game}/settings-h_REF.gc | 0 .../{ => engine/game/task}/game-task-h_REF.gc | 0 .../game/task}/hint-control-h_REF.gc | 0 .../geometry}/bounding-box-h_REF.gc | 0 .../{ => engine/geometry}/bounding-box_REF.gc | 0 .../{ => engine/geometry}/geometry-h_REF.gc | 0 .../reference/{ => engine/gfx}/capture_REF.gc | 0 .../{ => engine/gfx}/decomp-h_REF.gc | 0 .../reference/{ => engine/gfx}/font-h_REF.gc | 0 .../{ => engine/gfx/generic}/generic-h_REF.gc | 0 .../{ => engine/gfx/hw}/display-h_REF.gc | 0 .../{ => engine/gfx/hw}/display_REF.gc | 0 .../reference/{ => engine/gfx/hw}/gs_REF.gc | 0 .../{ => engine/gfx/hw}/video-h_REF.gc | 0 .../{ => engine/gfx/hw}/vu1-user-h_REF.gc | 0 .../{ => engine/gfx}/lights-h_REF.gc | 9 +- .../reference/{ => engine/gfx}/lights_REF.gc | 10 +- .../reference/{ => engine/gfx}/mood-h_REF.gc | 0 .../{ => engine/gfx/ocean}/ocean-h_REF.gc | 0 .../gfx/ocean}/ocean-trans-tables_REF.gc | 0 .../{ => engine/gfx/sky}/sky-h_REF.gc | 0 .../{ => engine/gfx}/texture-h_REF.gc | 0 .../{ => engine/level}/level-h_REF.gc | 0 .../{ => engine/load}/file-io_REF.gc | 0 .../{ => engine/load}/loader-h_REF.gc | 0 .../{ => engine/math}/euler-h_REF.gc | 0 .../reference/{ => engine/math}/euler_REF.gc | 0 .../reference/{ => engine/math}/math_REF.gc | 0 .../{ => engine/math}/matrix-h_REF.gc | 0 .../reference/{ => engine/math}/matrix_REF.gc | 0 .../{ => engine/math}/quaternion-h_REF.gc | 0 .../{ => engine/math}/quaternion_REF.gc | 0 .../{ => engine/math}/transform-h_REF.gc | 0 .../{ => engine/math}/transform_REF.gc | 0 .../{ => engine/math}/trigonometry-h_REF.gc | 0 .../{ => engine/math}/vector-h_REF.gc | 0 .../reference/{ => engine/math}/vector_REF.gc | 0 .../reference/{ => engine/ps2}/pad_REF.gc | 0 .../reference/{ => engine/ps2}/timer-h_REF.gc | 0 .../reference/{ => engine/ps2}/timer_REF.gc | 0 .../reference/{ => engine/ps2}/vif-h_REF.gc | 0 .../{ => engine/ps2}/vu1-macros_REF.gc | 0 .../{ => engine/sound}/gsound-h_REF.gc | 0 .../reference/{ => engine/ui}/hud-h_REF.gc | 0 .../reference/{ => engine/ui}/text-h_REF.gc | 0 .../{ => engine/util}/sync-info-h_REF.gc | 0 .../{ => engine/util}/sync-info_REF.gc | 0 .../{ => engine/util}/types-h_REF.gc | 0 .../reference/{ => kernel}/dgo-h_REF.gc | 0 .../reference/{ => kernel}/gcommon_REF.gc | 0 .../reference/{ => kernel}/gkernel-h_REF.gc | 0 .../reference/{ => kernel}/gkernel_REF.gc | 0 .../reference/{ => kernel}/gstate_REF.gc | 0 .../reference/{ => kernel}/gstring-h_REF.gc | 0 .../reference/{ => kernel}/gstring_REF.gc | 0 test/offline/offline_test_main.cpp | 125 ++++++++++-------- 81 files changed, 148 insertions(+), 94 deletions(-) rename goal_src/build/{all_objs.txt => all_objs.json} (100%) rename goal_src/build/{game_dgos.txt => game_dgos.gc} (100%) rename goal_src/build/{kernel_dgos.txt => kernel_dgos.gc} (100%) rename test/decompiler/reference/{ => engine/anim}/mspace-h_REF.gc (100%) rename test/decompiler/reference/{ => engine/camera}/math-camera-h_REF.gc (100%) rename test/decompiler/reference/{ => engine/camera}/pov-camera-h_REF.gc (100%) rename test/decompiler/reference/{ => engine/debug}/memory-usage-h_REF.gc (100%) rename test/decompiler/reference/{ => engine/dma}/dma-bucket_REF.gc (100%) rename test/decompiler/reference/{ => engine/dma}/dma-buffer_REF.gc (100%) rename test/decompiler/reference/{ => engine/dma}/dma-disasm_REF.gc (100%) rename test/decompiler/reference/{ => engine/dma}/dma-h_REF.gc (100%) rename test/decompiler/reference/{ => engine/dma}/dma_REF.gc (100%) rename test/decompiler/reference/{ => engine/draw}/draw-node-h_REF.gc (100%) rename test/decompiler/reference/{ => engine/draw}/drawable-actor-h_REF.gc (100%) rename test/decompiler/reference/{ => engine/draw}/drawable-ambient-h_REF.gc (100%) rename test/decompiler/reference/{ => engine/draw}/drawable-group-h_REF.gc (100%) rename test/decompiler/reference/{ => engine/draw}/drawable-h_REF.gc (100%) rename test/decompiler/reference/{ => engine/draw}/drawable-inline-array-h_REF.gc (100%) rename test/decompiler/reference/{ => engine/draw}/drawable-tree-h_REF.gc (100%) rename test/decompiler/reference/{ => engine/engine}/connect_REF.gc (100%) rename test/decompiler/reference/{ => engine/game}/main-h_REF.gc (100%) rename test/decompiler/reference/{ => engine/game}/settings-h_REF.gc (100%) rename test/decompiler/reference/{ => engine/game/task}/game-task-h_REF.gc (100%) rename test/decompiler/reference/{ => engine/game/task}/hint-control-h_REF.gc (100%) rename test/decompiler/reference/{ => engine/geometry}/bounding-box-h_REF.gc (100%) rename test/decompiler/reference/{ => engine/geometry}/bounding-box_REF.gc (100%) rename test/decompiler/reference/{ => engine/geometry}/geometry-h_REF.gc (100%) rename test/decompiler/reference/{ => engine/gfx}/capture_REF.gc (100%) rename test/decompiler/reference/{ => engine/gfx}/decomp-h_REF.gc (100%) rename test/decompiler/reference/{ => engine/gfx}/font-h_REF.gc (100%) rename test/decompiler/reference/{ => engine/gfx/generic}/generic-h_REF.gc (100%) rename test/decompiler/reference/{ => engine/gfx/hw}/display-h_REF.gc (100%) rename test/decompiler/reference/{ => engine/gfx/hw}/display_REF.gc (100%) rename test/decompiler/reference/{ => engine/gfx/hw}/gs_REF.gc (100%) rename test/decompiler/reference/{ => engine/gfx/hw}/video-h_REF.gc (100%) rename test/decompiler/reference/{ => engine/gfx/hw}/vu1-user-h_REF.gc (100%) rename test/decompiler/reference/{ => engine/gfx}/lights-h_REF.gc (95%) rename test/decompiler/reference/{ => engine/gfx}/lights_REF.gc (92%) rename test/decompiler/reference/{ => engine/gfx}/mood-h_REF.gc (100%) rename test/decompiler/reference/{ => engine/gfx/ocean}/ocean-h_REF.gc (100%) rename test/decompiler/reference/{ => engine/gfx/ocean}/ocean-trans-tables_REF.gc (100%) rename test/decompiler/reference/{ => engine/gfx/sky}/sky-h_REF.gc (100%) rename test/decompiler/reference/{ => engine/gfx}/texture-h_REF.gc (100%) rename test/decompiler/reference/{ => engine/level}/level-h_REF.gc (100%) rename test/decompiler/reference/{ => engine/load}/file-io_REF.gc (100%) rename test/decompiler/reference/{ => engine/load}/loader-h_REF.gc (100%) rename test/decompiler/reference/{ => engine/math}/euler-h_REF.gc (100%) rename test/decompiler/reference/{ => engine/math}/euler_REF.gc (100%) rename test/decompiler/reference/{ => engine/math}/math_REF.gc (100%) rename test/decompiler/reference/{ => engine/math}/matrix-h_REF.gc (100%) rename test/decompiler/reference/{ => engine/math}/matrix_REF.gc (100%) rename test/decompiler/reference/{ => engine/math}/quaternion-h_REF.gc (100%) rename test/decompiler/reference/{ => engine/math}/quaternion_REF.gc (100%) rename test/decompiler/reference/{ => engine/math}/transform-h_REF.gc (100%) rename test/decompiler/reference/{ => engine/math}/transform_REF.gc (100%) rename test/decompiler/reference/{ => engine/math}/trigonometry-h_REF.gc (100%) rename test/decompiler/reference/{ => engine/math}/vector-h_REF.gc (100%) rename test/decompiler/reference/{ => engine/math}/vector_REF.gc (100%) rename test/decompiler/reference/{ => engine/ps2}/pad_REF.gc (100%) rename test/decompiler/reference/{ => engine/ps2}/timer-h_REF.gc (100%) rename test/decompiler/reference/{ => engine/ps2}/timer_REF.gc (100%) rename test/decompiler/reference/{ => engine/ps2}/vif-h_REF.gc (100%) rename test/decompiler/reference/{ => engine/ps2}/vu1-macros_REF.gc (100%) rename test/decompiler/reference/{ => engine/sound}/gsound-h_REF.gc (100%) rename test/decompiler/reference/{ => engine/ui}/hud-h_REF.gc (100%) rename test/decompiler/reference/{ => engine/ui}/text-h_REF.gc (100%) rename test/decompiler/reference/{ => engine/util}/sync-info-h_REF.gc (100%) rename test/decompiler/reference/{ => engine/util}/sync-info_REF.gc (100%) rename test/decompiler/reference/{ => engine/util}/types-h_REF.gc (100%) rename test/decompiler/reference/{ => kernel}/dgo-h_REF.gc (100%) rename test/decompiler/reference/{ => kernel}/gcommon_REF.gc (100%) rename test/decompiler/reference/{ => kernel}/gkernel-h_REF.gc (100%) rename test/decompiler/reference/{ => kernel}/gkernel_REF.gc (100%) rename test/decompiler/reference/{ => kernel}/gstate_REF.gc (100%) rename test/decompiler/reference/{ => kernel}/gstring-h_REF.gc (100%) rename test/decompiler/reference/{ => kernel}/gstring_REF.gc (100%) diff --git a/Taskfile.yml b/Taskfile.yml index a1fdd0882..c859cb84a 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -16,4 +16,17 @@ tasks: decomp-clean: cmds: - rm ./decompiler_out/**/*.asm - - rm ./decompiler_out/**/*disasm.gc \ No newline at end of file + - rm ./decompiler_out/**/*disasm.gc + decomp-next: + cmds: + - python ./scripts/next-decomp-file.py --skip "{{.SKIP}}" + - task: decomp + vars: + SKIP: '{{default "0" .SKIP}}' + decomp-file: + cmds: + - python ./scripts/next-decomp-file.py --file "{{.FILE}}" + - task: decomp + decomp-list: + cmds: + - python ./scripts/next-decomp-file.py --list \ No newline at end of file diff --git a/decompiler/config/jak1_ntsc_black_label.jsonc b/decompiler/config/jak1_ntsc_black_label.jsonc index 79bed70a9..7748dda5b 100644 --- a/decompiler/config/jak1_ntsc_black_label.jsonc +++ b/decompiler/config/jak1_ntsc_black_label.jsonc @@ -86,7 +86,7 @@ "write_scripts":false, // optional: a predetermined object file name map from a file. Useful if you want to run only on some DGOs but have consistent names - "obj_file_name_map_file":"goal_src/build/all_objs.txt", + "obj_file_name_map_file":"goal_src/build/all_objs.json", "types_with_bad_inspect_methods":[ diff --git a/goal_src/build/all_objs.txt b/goal_src/build/all_objs.json similarity index 100% rename from goal_src/build/all_objs.txt rename to goal_src/build/all_objs.json diff --git a/goal_src/build/game_dgos.txt b/goal_src/build/game_dgos.gc similarity index 100% rename from goal_src/build/game_dgos.txt rename to goal_src/build/game_dgos.gc diff --git a/goal_src/build/kernel_dgos.txt b/goal_src/build/kernel_dgos.gc similarity index 100% rename from goal_src/build/kernel_dgos.txt rename to goal_src/build/kernel_dgos.gc diff --git a/goal_src/goal-lib.gc b/goal_src/goal-lib.gc index 8067a9477..51cbf41ba 100644 --- a/goal_src/goal-lib.gc +++ b/goal_src/goal-lib.gc @@ -39,7 +39,7 @@ "Build kernel and create the KERNEL CGO" `(begin ,@(apply make-build-command all-kernel-goal-files) - (build-dgos "goal_src/build/kernel_dgos.txt") + (build-dgos "goal_src/build/kernel_dgos.gc") ) ) @@ -48,7 +48,7 @@ `(begin (build-kernel) ,@(apply make-build-command all-goal-files) - (build-dgos "goal_src/build/game_dgos.txt") + (build-dgos "goal_src/build/game_dgos.gc") ) ) diff --git a/scripts/next-decomp-file.py b/scripts/next-decomp-file.py index 9c7639876..0aab89b89 100644 --- a/scripts/next-decomp-file.py +++ b/scripts/next-decomp-file.py @@ -1,35 +1,54 @@ import sys from jak1_file_list import file_list +import argparse +parser = argparse.ArgumentParser() +parser.add_argument("--skip", type=int) +parser.add_argument("--file") +parser.add_argument("--list", action="store_true") +args = parser.parse_args() + skip_count = 0 -if len(sys.argv) >= 2: - skip_count = int(sys.argv[1]) - -print(skip_count) - -new_file_contents = [] - -for file in file_list: - with open("goal_src/" + file[4] + "/" + file[0] + ".gc") as f: - lines = f.readlines() - # print("goal_src/" + file[4] + "/" + file[0] + ".gc" + " " + str(len(lines))) - if skip_count <= 0 and len(lines) <= 7: - print("Next file to decompile is - " + file[0]) - print("Target is - " + "goal_src/" + file[4] + "/" + file[0] + ".gc") - print("Uses the following CGO / DGO - " + str(file[3])) - # TODO, update the CGO/DGO - with open("decompiler\config\jak1_ntsc_black_label.jsonc", "r") as config_file: - cfg_lines = config_file.readlines() - for line in cfg_lines: - if "allowed_objects" in line: - line = " \"allowed_objects\": [\"" + file[0] + "\"],\n" - new_file_contents.append(line) - if len(new_file_contents) > 0: - with open("decompiler\config\jak1_ntsc_black_label.jsonc", "w") as f: - f.writelines(new_file_contents) - print("Allowed objects list updated") - break - elif len(lines) <= 7: - skip_count = skip_count - 1 +if args.skip: + skip_count = args.skip +def update_file(file): + new_file_contents = [] + print("Next file to decompile is - " + file[0]) + print("Target is - " + "goal_src/" + file[4] + "/" + file[0] + ".gc") + print("Uses the following CGO / DGO - " + str(file[3])) + # TODO, update the CGO/DGO + with open("decompiler\config\jak1_ntsc_black_label.jsonc", "r") as config_file: + cfg_lines = config_file.readlines() + for line in cfg_lines: + if "allowed_objects" in line: + line = " \"allowed_objects\": [\"" + file[0] + "\"],\n" + new_file_contents.append(line) + if len(new_file_contents) > 0: + with open("decompiler\config\jak1_ntsc_black_label.jsonc", "w") as f: + f.writelines(new_file_contents) + print("Allowed objects list updated") +if args.file: + for file in file_list: + if file[0] == args.file: + update_file(file) +else: + list_of_eligible = [] + for file in file_list: + with open("goal_src/" + file[4] + "/" + file[0] + ".gc") as f: + lines = f.readlines() + if skip_count <= 0 and len(lines) <= 7: + if args.list: + list_of_eligible.append(file[0]) + if len(list_of_eligible) >= 10: + break + else: + update_file(file) + break + elif not args.list and len(lines) <= 7: + print("wat") + skip_count = skip_count - 1 + if len(list_of_eligible) > 0: + print("The next 10 files that need to be decompiled:") + print(*list_of_eligible, sep = "\n") diff --git a/test/decompiler/reference/mspace-h_REF.gc b/test/decompiler/reference/engine/anim/mspace-h_REF.gc similarity index 100% rename from test/decompiler/reference/mspace-h_REF.gc rename to test/decompiler/reference/engine/anim/mspace-h_REF.gc diff --git a/test/decompiler/reference/math-camera-h_REF.gc b/test/decompiler/reference/engine/camera/math-camera-h_REF.gc similarity index 100% rename from test/decompiler/reference/math-camera-h_REF.gc rename to test/decompiler/reference/engine/camera/math-camera-h_REF.gc diff --git a/test/decompiler/reference/pov-camera-h_REF.gc b/test/decompiler/reference/engine/camera/pov-camera-h_REF.gc similarity index 100% rename from test/decompiler/reference/pov-camera-h_REF.gc rename to test/decompiler/reference/engine/camera/pov-camera-h_REF.gc diff --git a/test/decompiler/reference/memory-usage-h_REF.gc b/test/decompiler/reference/engine/debug/memory-usage-h_REF.gc similarity index 100% rename from test/decompiler/reference/memory-usage-h_REF.gc rename to test/decompiler/reference/engine/debug/memory-usage-h_REF.gc diff --git a/test/decompiler/reference/dma-bucket_REF.gc b/test/decompiler/reference/engine/dma/dma-bucket_REF.gc similarity index 100% rename from test/decompiler/reference/dma-bucket_REF.gc rename to test/decompiler/reference/engine/dma/dma-bucket_REF.gc diff --git a/test/decompiler/reference/dma-buffer_REF.gc b/test/decompiler/reference/engine/dma/dma-buffer_REF.gc similarity index 100% rename from test/decompiler/reference/dma-buffer_REF.gc rename to test/decompiler/reference/engine/dma/dma-buffer_REF.gc diff --git a/test/decompiler/reference/dma-disasm_REF.gc b/test/decompiler/reference/engine/dma/dma-disasm_REF.gc similarity index 100% rename from test/decompiler/reference/dma-disasm_REF.gc rename to test/decompiler/reference/engine/dma/dma-disasm_REF.gc diff --git a/test/decompiler/reference/dma-h_REF.gc b/test/decompiler/reference/engine/dma/dma-h_REF.gc similarity index 100% rename from test/decompiler/reference/dma-h_REF.gc rename to test/decompiler/reference/engine/dma/dma-h_REF.gc diff --git a/test/decompiler/reference/dma_REF.gc b/test/decompiler/reference/engine/dma/dma_REF.gc similarity index 100% rename from test/decompiler/reference/dma_REF.gc rename to test/decompiler/reference/engine/dma/dma_REF.gc diff --git a/test/decompiler/reference/draw-node-h_REF.gc b/test/decompiler/reference/engine/draw/draw-node-h_REF.gc similarity index 100% rename from test/decompiler/reference/draw-node-h_REF.gc rename to test/decompiler/reference/engine/draw/draw-node-h_REF.gc diff --git a/test/decompiler/reference/drawable-actor-h_REF.gc b/test/decompiler/reference/engine/draw/drawable-actor-h_REF.gc similarity index 100% rename from test/decompiler/reference/drawable-actor-h_REF.gc rename to test/decompiler/reference/engine/draw/drawable-actor-h_REF.gc diff --git a/test/decompiler/reference/drawable-ambient-h_REF.gc b/test/decompiler/reference/engine/draw/drawable-ambient-h_REF.gc similarity index 100% rename from test/decompiler/reference/drawable-ambient-h_REF.gc rename to test/decompiler/reference/engine/draw/drawable-ambient-h_REF.gc diff --git a/test/decompiler/reference/drawable-group-h_REF.gc b/test/decompiler/reference/engine/draw/drawable-group-h_REF.gc similarity index 100% rename from test/decompiler/reference/drawable-group-h_REF.gc rename to test/decompiler/reference/engine/draw/drawable-group-h_REF.gc diff --git a/test/decompiler/reference/drawable-h_REF.gc b/test/decompiler/reference/engine/draw/drawable-h_REF.gc similarity index 100% rename from test/decompiler/reference/drawable-h_REF.gc rename to test/decompiler/reference/engine/draw/drawable-h_REF.gc diff --git a/test/decompiler/reference/drawable-inline-array-h_REF.gc b/test/decompiler/reference/engine/draw/drawable-inline-array-h_REF.gc similarity index 100% rename from test/decompiler/reference/drawable-inline-array-h_REF.gc rename to test/decompiler/reference/engine/draw/drawable-inline-array-h_REF.gc diff --git a/test/decompiler/reference/drawable-tree-h_REF.gc b/test/decompiler/reference/engine/draw/drawable-tree-h_REF.gc similarity index 100% rename from test/decompiler/reference/drawable-tree-h_REF.gc rename to test/decompiler/reference/engine/draw/drawable-tree-h_REF.gc diff --git a/test/decompiler/reference/connect_REF.gc b/test/decompiler/reference/engine/engine/connect_REF.gc similarity index 100% rename from test/decompiler/reference/connect_REF.gc rename to test/decompiler/reference/engine/engine/connect_REF.gc diff --git a/test/decompiler/reference/main-h_REF.gc b/test/decompiler/reference/engine/game/main-h_REF.gc similarity index 100% rename from test/decompiler/reference/main-h_REF.gc rename to test/decompiler/reference/engine/game/main-h_REF.gc diff --git a/test/decompiler/reference/settings-h_REF.gc b/test/decompiler/reference/engine/game/settings-h_REF.gc similarity index 100% rename from test/decompiler/reference/settings-h_REF.gc rename to test/decompiler/reference/engine/game/settings-h_REF.gc diff --git a/test/decompiler/reference/game-task-h_REF.gc b/test/decompiler/reference/engine/game/task/game-task-h_REF.gc similarity index 100% rename from test/decompiler/reference/game-task-h_REF.gc rename to test/decompiler/reference/engine/game/task/game-task-h_REF.gc diff --git a/test/decompiler/reference/hint-control-h_REF.gc b/test/decompiler/reference/engine/game/task/hint-control-h_REF.gc similarity index 100% rename from test/decompiler/reference/hint-control-h_REF.gc rename to test/decompiler/reference/engine/game/task/hint-control-h_REF.gc diff --git a/test/decompiler/reference/bounding-box-h_REF.gc b/test/decompiler/reference/engine/geometry/bounding-box-h_REF.gc similarity index 100% rename from test/decompiler/reference/bounding-box-h_REF.gc rename to test/decompiler/reference/engine/geometry/bounding-box-h_REF.gc diff --git a/test/decompiler/reference/bounding-box_REF.gc b/test/decompiler/reference/engine/geometry/bounding-box_REF.gc similarity index 100% rename from test/decompiler/reference/bounding-box_REF.gc rename to test/decompiler/reference/engine/geometry/bounding-box_REF.gc diff --git a/test/decompiler/reference/geometry-h_REF.gc b/test/decompiler/reference/engine/geometry/geometry-h_REF.gc similarity index 100% rename from test/decompiler/reference/geometry-h_REF.gc rename to test/decompiler/reference/engine/geometry/geometry-h_REF.gc diff --git a/test/decompiler/reference/capture_REF.gc b/test/decompiler/reference/engine/gfx/capture_REF.gc similarity index 100% rename from test/decompiler/reference/capture_REF.gc rename to test/decompiler/reference/engine/gfx/capture_REF.gc diff --git a/test/decompiler/reference/decomp-h_REF.gc b/test/decompiler/reference/engine/gfx/decomp-h_REF.gc similarity index 100% rename from test/decompiler/reference/decomp-h_REF.gc rename to test/decompiler/reference/engine/gfx/decomp-h_REF.gc diff --git a/test/decompiler/reference/font-h_REF.gc b/test/decompiler/reference/engine/gfx/font-h_REF.gc similarity index 100% rename from test/decompiler/reference/font-h_REF.gc rename to test/decompiler/reference/engine/gfx/font-h_REF.gc diff --git a/test/decompiler/reference/generic-h_REF.gc b/test/decompiler/reference/engine/gfx/generic/generic-h_REF.gc similarity index 100% rename from test/decompiler/reference/generic-h_REF.gc rename to test/decompiler/reference/engine/gfx/generic/generic-h_REF.gc diff --git a/test/decompiler/reference/display-h_REF.gc b/test/decompiler/reference/engine/gfx/hw/display-h_REF.gc similarity index 100% rename from test/decompiler/reference/display-h_REF.gc rename to test/decompiler/reference/engine/gfx/hw/display-h_REF.gc diff --git a/test/decompiler/reference/display_REF.gc b/test/decompiler/reference/engine/gfx/hw/display_REF.gc similarity index 100% rename from test/decompiler/reference/display_REF.gc rename to test/decompiler/reference/engine/gfx/hw/display_REF.gc diff --git a/test/decompiler/reference/gs_REF.gc b/test/decompiler/reference/engine/gfx/hw/gs_REF.gc similarity index 100% rename from test/decompiler/reference/gs_REF.gc rename to test/decompiler/reference/engine/gfx/hw/gs_REF.gc diff --git a/test/decompiler/reference/video-h_REF.gc b/test/decompiler/reference/engine/gfx/hw/video-h_REF.gc similarity index 100% rename from test/decompiler/reference/video-h_REF.gc rename to test/decompiler/reference/engine/gfx/hw/video-h_REF.gc diff --git a/test/decompiler/reference/vu1-user-h_REF.gc b/test/decompiler/reference/engine/gfx/hw/vu1-user-h_REF.gc similarity index 100% rename from test/decompiler/reference/vu1-user-h_REF.gc rename to test/decompiler/reference/engine/gfx/hw/vu1-user-h_REF.gc diff --git a/test/decompiler/reference/lights-h_REF.gc b/test/decompiler/reference/engine/gfx/lights-h_REF.gc similarity index 95% rename from test/decompiler/reference/lights-h_REF.gc rename to test/decompiler/reference/engine/gfx/lights-h_REF.gc index 78c7a9eb0..f11b57883 100644 --- a/test/decompiler/reference/lights-h_REF.gc +++ b/test/decompiler/reference/engine/gfx/lights-h_REF.gc @@ -184,10 +184,11 @@ ;; definition of type light-group (deftype light-group (structure) - ((dir0 light :inline :offset-assert 0) - (dir1 light :inline :offset-assert 48) - (dir2 light :inline :offset-assert 96) - (ambi light :inline :offset-assert 144) + ((dir0 light :inline :offset-assert 0) + (dir1 light :inline :offset-assert 48) + (dir2 light :inline :offset-assert 96) + (ambi light :inline :offset-assert 144) + (lights light 4 :inline :offset 0) ) :method-count-assert 9 :size-assert #xc0 diff --git a/test/decompiler/reference/lights_REF.gc b/test/decompiler/reference/engine/gfx/lights_REF.gc similarity index 92% rename from test/decompiler/reference/lights_REF.gc rename to test/decompiler/reference/engine/gfx/lights_REF.gc index 1cf554c99..abeaa6f88 100644 --- a/test/decompiler/reference/lights_REF.gc +++ b/test/decompiler/reference/engine/gfx/lights_REF.gc @@ -26,9 +26,9 @@ ((arg0 light-group) (arg1 light-group) (arg2 light-group) (arg3 float)) (dotimes (s2-0 4) (light-slerp - (the-as light (+ (+ (* 48 s2-0) 0) (the-as int arg0))) - (the-as light (+ (+ (* 48 s2-0) 0) (the-as int arg1))) - (the-as light (+ (+ (* 48 s2-0) 0) (the-as int arg2))) + (-> arg0 lights s2-0) + (-> arg1 lights s2-0) + (-> arg2 lights s2-0) arg3 ) ) @@ -94,3 +94,7 @@ ) arg0 ) + + + + diff --git a/test/decompiler/reference/mood-h_REF.gc b/test/decompiler/reference/engine/gfx/mood-h_REF.gc similarity index 100% rename from test/decompiler/reference/mood-h_REF.gc rename to test/decompiler/reference/engine/gfx/mood-h_REF.gc diff --git a/test/decompiler/reference/ocean-h_REF.gc b/test/decompiler/reference/engine/gfx/ocean/ocean-h_REF.gc similarity index 100% rename from test/decompiler/reference/ocean-h_REF.gc rename to test/decompiler/reference/engine/gfx/ocean/ocean-h_REF.gc diff --git a/test/decompiler/reference/ocean-trans-tables_REF.gc b/test/decompiler/reference/engine/gfx/ocean/ocean-trans-tables_REF.gc similarity index 100% rename from test/decompiler/reference/ocean-trans-tables_REF.gc rename to test/decompiler/reference/engine/gfx/ocean/ocean-trans-tables_REF.gc diff --git a/test/decompiler/reference/sky-h_REF.gc b/test/decompiler/reference/engine/gfx/sky/sky-h_REF.gc similarity index 100% rename from test/decompiler/reference/sky-h_REF.gc rename to test/decompiler/reference/engine/gfx/sky/sky-h_REF.gc diff --git a/test/decompiler/reference/texture-h_REF.gc b/test/decompiler/reference/engine/gfx/texture-h_REF.gc similarity index 100% rename from test/decompiler/reference/texture-h_REF.gc rename to test/decompiler/reference/engine/gfx/texture-h_REF.gc diff --git a/test/decompiler/reference/level-h_REF.gc b/test/decompiler/reference/engine/level/level-h_REF.gc similarity index 100% rename from test/decompiler/reference/level-h_REF.gc rename to test/decompiler/reference/engine/level/level-h_REF.gc diff --git a/test/decompiler/reference/file-io_REF.gc b/test/decompiler/reference/engine/load/file-io_REF.gc similarity index 100% rename from test/decompiler/reference/file-io_REF.gc rename to test/decompiler/reference/engine/load/file-io_REF.gc diff --git a/test/decompiler/reference/loader-h_REF.gc b/test/decompiler/reference/engine/load/loader-h_REF.gc similarity index 100% rename from test/decompiler/reference/loader-h_REF.gc rename to test/decompiler/reference/engine/load/loader-h_REF.gc diff --git a/test/decompiler/reference/euler-h_REF.gc b/test/decompiler/reference/engine/math/euler-h_REF.gc similarity index 100% rename from test/decompiler/reference/euler-h_REF.gc rename to test/decompiler/reference/engine/math/euler-h_REF.gc diff --git a/test/decompiler/reference/euler_REF.gc b/test/decompiler/reference/engine/math/euler_REF.gc similarity index 100% rename from test/decompiler/reference/euler_REF.gc rename to test/decompiler/reference/engine/math/euler_REF.gc diff --git a/test/decompiler/reference/math_REF.gc b/test/decompiler/reference/engine/math/math_REF.gc similarity index 100% rename from test/decompiler/reference/math_REF.gc rename to test/decompiler/reference/engine/math/math_REF.gc diff --git a/test/decompiler/reference/matrix-h_REF.gc b/test/decompiler/reference/engine/math/matrix-h_REF.gc similarity index 100% rename from test/decompiler/reference/matrix-h_REF.gc rename to test/decompiler/reference/engine/math/matrix-h_REF.gc diff --git a/test/decompiler/reference/matrix_REF.gc b/test/decompiler/reference/engine/math/matrix_REF.gc similarity index 100% rename from test/decompiler/reference/matrix_REF.gc rename to test/decompiler/reference/engine/math/matrix_REF.gc diff --git a/test/decompiler/reference/quaternion-h_REF.gc b/test/decompiler/reference/engine/math/quaternion-h_REF.gc similarity index 100% rename from test/decompiler/reference/quaternion-h_REF.gc rename to test/decompiler/reference/engine/math/quaternion-h_REF.gc diff --git a/test/decompiler/reference/quaternion_REF.gc b/test/decompiler/reference/engine/math/quaternion_REF.gc similarity index 100% rename from test/decompiler/reference/quaternion_REF.gc rename to test/decompiler/reference/engine/math/quaternion_REF.gc diff --git a/test/decompiler/reference/transform-h_REF.gc b/test/decompiler/reference/engine/math/transform-h_REF.gc similarity index 100% rename from test/decompiler/reference/transform-h_REF.gc rename to test/decompiler/reference/engine/math/transform-h_REF.gc diff --git a/test/decompiler/reference/transform_REF.gc b/test/decompiler/reference/engine/math/transform_REF.gc similarity index 100% rename from test/decompiler/reference/transform_REF.gc rename to test/decompiler/reference/engine/math/transform_REF.gc diff --git a/test/decompiler/reference/trigonometry-h_REF.gc b/test/decompiler/reference/engine/math/trigonometry-h_REF.gc similarity index 100% rename from test/decompiler/reference/trigonometry-h_REF.gc rename to test/decompiler/reference/engine/math/trigonometry-h_REF.gc diff --git a/test/decompiler/reference/vector-h_REF.gc b/test/decompiler/reference/engine/math/vector-h_REF.gc similarity index 100% rename from test/decompiler/reference/vector-h_REF.gc rename to test/decompiler/reference/engine/math/vector-h_REF.gc diff --git a/test/decompiler/reference/vector_REF.gc b/test/decompiler/reference/engine/math/vector_REF.gc similarity index 100% rename from test/decompiler/reference/vector_REF.gc rename to test/decompiler/reference/engine/math/vector_REF.gc diff --git a/test/decompiler/reference/pad_REF.gc b/test/decompiler/reference/engine/ps2/pad_REF.gc similarity index 100% rename from test/decompiler/reference/pad_REF.gc rename to test/decompiler/reference/engine/ps2/pad_REF.gc diff --git a/test/decompiler/reference/timer-h_REF.gc b/test/decompiler/reference/engine/ps2/timer-h_REF.gc similarity index 100% rename from test/decompiler/reference/timer-h_REF.gc rename to test/decompiler/reference/engine/ps2/timer-h_REF.gc diff --git a/test/decompiler/reference/timer_REF.gc b/test/decompiler/reference/engine/ps2/timer_REF.gc similarity index 100% rename from test/decompiler/reference/timer_REF.gc rename to test/decompiler/reference/engine/ps2/timer_REF.gc diff --git a/test/decompiler/reference/vif-h_REF.gc b/test/decompiler/reference/engine/ps2/vif-h_REF.gc similarity index 100% rename from test/decompiler/reference/vif-h_REF.gc rename to test/decompiler/reference/engine/ps2/vif-h_REF.gc diff --git a/test/decompiler/reference/vu1-macros_REF.gc b/test/decompiler/reference/engine/ps2/vu1-macros_REF.gc similarity index 100% rename from test/decompiler/reference/vu1-macros_REF.gc rename to test/decompiler/reference/engine/ps2/vu1-macros_REF.gc diff --git a/test/decompiler/reference/gsound-h_REF.gc b/test/decompiler/reference/engine/sound/gsound-h_REF.gc similarity index 100% rename from test/decompiler/reference/gsound-h_REF.gc rename to test/decompiler/reference/engine/sound/gsound-h_REF.gc diff --git a/test/decompiler/reference/hud-h_REF.gc b/test/decompiler/reference/engine/ui/hud-h_REF.gc similarity index 100% rename from test/decompiler/reference/hud-h_REF.gc rename to test/decompiler/reference/engine/ui/hud-h_REF.gc diff --git a/test/decompiler/reference/text-h_REF.gc b/test/decompiler/reference/engine/ui/text-h_REF.gc similarity index 100% rename from test/decompiler/reference/text-h_REF.gc rename to test/decompiler/reference/engine/ui/text-h_REF.gc diff --git a/test/decompiler/reference/sync-info-h_REF.gc b/test/decompiler/reference/engine/util/sync-info-h_REF.gc similarity index 100% rename from test/decompiler/reference/sync-info-h_REF.gc rename to test/decompiler/reference/engine/util/sync-info-h_REF.gc diff --git a/test/decompiler/reference/sync-info_REF.gc b/test/decompiler/reference/engine/util/sync-info_REF.gc similarity index 100% rename from test/decompiler/reference/sync-info_REF.gc rename to test/decompiler/reference/engine/util/sync-info_REF.gc diff --git a/test/decompiler/reference/types-h_REF.gc b/test/decompiler/reference/engine/util/types-h_REF.gc similarity index 100% rename from test/decompiler/reference/types-h_REF.gc rename to test/decompiler/reference/engine/util/types-h_REF.gc diff --git a/test/decompiler/reference/dgo-h_REF.gc b/test/decompiler/reference/kernel/dgo-h_REF.gc similarity index 100% rename from test/decompiler/reference/dgo-h_REF.gc rename to test/decompiler/reference/kernel/dgo-h_REF.gc diff --git a/test/decompiler/reference/gcommon_REF.gc b/test/decompiler/reference/kernel/gcommon_REF.gc similarity index 100% rename from test/decompiler/reference/gcommon_REF.gc rename to test/decompiler/reference/kernel/gcommon_REF.gc diff --git a/test/decompiler/reference/gkernel-h_REF.gc b/test/decompiler/reference/kernel/gkernel-h_REF.gc similarity index 100% rename from test/decompiler/reference/gkernel-h_REF.gc rename to test/decompiler/reference/kernel/gkernel-h_REF.gc diff --git a/test/decompiler/reference/gkernel_REF.gc b/test/decompiler/reference/kernel/gkernel_REF.gc similarity index 100% rename from test/decompiler/reference/gkernel_REF.gc rename to test/decompiler/reference/kernel/gkernel_REF.gc diff --git a/test/decompiler/reference/gstate_REF.gc b/test/decompiler/reference/kernel/gstate_REF.gc similarity index 100% rename from test/decompiler/reference/gstate_REF.gc rename to test/decompiler/reference/kernel/gstate_REF.gc diff --git a/test/decompiler/reference/gstring-h_REF.gc b/test/decompiler/reference/kernel/gstring-h_REF.gc similarity index 100% rename from test/decompiler/reference/gstring-h_REF.gc rename to test/decompiler/reference/kernel/gstring-h_REF.gc diff --git a/test/decompiler/reference/gstring_REF.gc b/test/decompiler/reference/kernel/gstring_REF.gc similarity index 100% rename from test/decompiler/reference/gstring_REF.gc rename to test/decompiler/reference/kernel/gstring_REF.gc diff --git a/test/offline/offline_test_main.cpp b/test/offline/offline_test_main.cpp index 1a39ce766..4232eafe1 100644 --- a/test/offline/offline_test_main.cpp +++ b/test/offline/offline_test_main.cpp @@ -7,52 +7,20 @@ #include "decompiler/ObjectFile/ObjectFileDB.h" #include "goalc/compiler/Compiler.h" #include "common/util/Timer.h" +#include + +namespace fs = std::filesystem; namespace { -// the object files to test -const std::unordered_set g_object_files_to_decompile = { - "gcommon", "gstring-h", "gkernel-h", "gkernel", - /*"pskernel",*/ "gstring", "dgo-h", "gstate", "types-h", "vu1-macros", "math", "vector-h", - "bounding-box-h", "matrix-h", "quaternion-h", "euler-h", "transform-h", "geometry-h", - "trigonometry-h", /* transformq-h */ "matrix", "transform", "quaternion", - "euler", /* geometry, trigonometry, */ - "gsound-h", "timer-h", "timer", "vif-h", "dma-h", "video-h", "vu1-user-h", "dma", "dma-buffer", - "dma-bucket", "dma-disasm", "pad", "gs", "display-h", "vector", "file-io", "loader-h", - "texture-h", "level-h", "math-camera-h", /* math-camera, */ "font-h", "decomp-h", "display", - "connect", "text-h", "settings-h", "capture", "memory-usage-h", /* "texture", */ "main-h", - "mspace-h", "drawable-h", "drawable-group-h", "drawable-inline-array-h", "draw-node-h", - "drawable-tree-h", "drawable-actor-h", "drawable-ambient-h", "game-task-h", "hint-control-h", - "generic-h", "lights-h", "lights", "ocean-h", "pov-camera-h", - "ocean-trans-tables", /* "ocean-tables", "ocean-frames", */ - "sky-h", "mood-h", /* "time-of-day-h", */ - "hud-h", - /* gap */ - "bounding-box", - /* gap */ - "sync-info-h", "sync-info"}; -// the object files to check against a reference in test/decompiler/reference -const std::vector g_object_files_to_check_against_reference = { - "gcommon", "gstring-h", "gkernel-h", "gkernel", - /*"pskernel",*/ "gstring", "dgo-h", "gstate", "types-h", "vu1-macros", "math", "vector-h", - "bounding-box-h", "matrix-h", "quaternion-h", "euler-h", "transform-h", "geometry-h", - "trigonometry-h", /* transformq-h, */ "matrix", "transform", "quaternion", - "euler", /* geometry, trigonometry */ - "gsound-h", "timer-h", /* timer, */ "vif-h", "dma-h", "video-h", "vu1-user-h", "dma", - "dma-buffer", "dma-bucket", "dma-disasm", "pad", "gs", "display-h", "vector", "file-io", - "loader-h", "texture-h", "level-h", "math-camera-h", /* math-camera, */ "font-h", "decomp-h", - "display", "connect", "text-h", "settings-h", "capture", "memory-usage-h", - /* "texture", */ "main-h", "mspace-h", "drawable-h", "drawable-group-h", - "drawable-inline-array-h", "draw-node-h", "drawable-tree-h", "drawable-actor-h", - "drawable-ambient-h", "game-task-h", "hint-control-h", "generic-h", "lights-h", "lights", - "ocean-h", "ocean-trans-tables", /* "ocean-tables", "ocean-frames", */ "pov-camera-h", "sky-h", - "mood-h", /* "time-of-day-h", */ - /* gap */ "bounding-box", "hud-h", - /* gap */ - "sync-info-h", "sync-info"}; +// list of object files to ignore during reference checks +const std::unordered_set g_object_files_to_ignore_ref_checks = { + "pskernel", "transformq-h", "geometry", "trigonometry", "math-camera", "timer", + "texture", "ocean-tables", "ocean-frames", "time-of-day", "display"}; -const std::unordered_set skip_files_in_compiling = { - "display" // interrupt handler setup +const std::unordered_set g_object_files_to_ignore_decompiling = { + // TODO - not implemented, if you want to ignore decompiling something currently, don't include + // it in the reference folder }; // the functions we expect the decompiler to skip @@ -167,10 +135,50 @@ std::string g_iso_data_path = ""; bool g_dump_mode = false; +std::vector> g_object_files_to_decompile_or_ref_check; + } // namespace + +std::string replaceFirstOccurrence(std::string& s, + const std::string& toReplace, + const std::string& replaceWith) { + std::size_t pos = s.find(toReplace); + if (pos == std::string::npos) + return s; + return s.replace(pos, toReplace.length(), replaceWith); +} + int main(int argc, char** argv) { lg::initialize(); + // Determine the files to decompile and reference check by scanning the reference directory + // All relevant files are assumed to end with `_REF.g[c|d]` + // First rough order them + std::vector> reference_files_rough_order; + for (auto& p : fs::recursive_directory_iterator( + file_util::get_file_path({"test", "decompiler", "reference"}))) { + if (p.is_regular_file()) { + std::string file_name = fs::path(p.path()).replace_extension().filename().string(); + if (file_name.find("_REF") == std::string::npos) { + continue; + } + std::string object_name = replaceFirstOccurrence(file_name, "_REF", ""); + reference_files_rough_order.push_back({object_name, p.path()}); + } + } + // use the all_objs.json file to place them in the correct build order + auto j = parse_commented_json( + file_util::read_text_file(file_util::get_file_path({"goal_src", "build", "all_objs.json"}))); + for (auto& x : j) { + auto mapped_name = x[0].get(); + for (auto& p : reference_files_rough_order) { + if (p.first == mapped_name) { + g_object_files_to_decompile_or_ref_check.push_back(p); + break; + } + } + } + // look for an argument that's not a gtest option bool got_arg = false; for (int i = 1; i < argc; i++) { @@ -198,7 +206,6 @@ int main(int argc, char** argv) { class OfflineDecompilation : public ::testing::Test { protected: static std::unique_ptr db; - static void SetUpTestCase() { // global setup file_util::init_crc(); @@ -206,7 +213,11 @@ class OfflineDecompilation : public ::testing::Test { decompiler::set_config( file_util::get_file_path({"decompiler", "config", "jak1_ntsc_black_label.jsonc"})); - decompiler::get_config().allowed_objects = g_object_files_to_decompile; + std::unordered_set object_files; + for (auto& p : g_object_files_to_decompile_or_ref_check) { + object_files.insert(p.first); + } + decompiler::get_config().allowed_objects = object_files; std::vector dgos = {"CGO/KERNEL.CGO", "CGO/ENGINE.CGO"}; std::vector dgo_paths; @@ -428,8 +439,13 @@ void strip_trailing_newlines(std::string& in) { } // namespace TEST_F(OfflineDecompilation, Reference) { - for (auto& file : g_object_files_to_check_against_reference) { - auto& obj_l = db->obj_files_by_name.at(file); + for (auto& file : g_object_files_to_decompile_or_ref_check) { + if (g_object_files_to_ignore_ref_checks.find(file.first) != + g_object_files_to_ignore_ref_checks.end()) { + continue; + } + + auto& obj_l = db->obj_files_by_name.at(file.first); ASSERT_EQ(obj_l.size(), 1); std::string src = db->ir2_final_out(obj_l.at(0)); @@ -438,17 +454,17 @@ TEST_F(OfflineDecompilation, Reference) { fmt::print("{}\n", src); }*/ - lg::info("Comparing {}...", file); + lg::info("Comparing {}...", file.first); - auto reference = file_util::read_text_file(file_util::get_file_path( - {"test", "decompiler", "reference", fmt::format("{}_REF.gc", file)})); + // NOTE - currently only handles .gc files! + auto reference = file_util::read_text_file(file.second.string()); strip_trailing_newlines(reference); strip_trailing_newlines(src); if (g_dump_mode) { if (reference != src) { - fmt::print("----------------- {}\n", file); + fmt::print("----------------- {}\n", file.first); fmt::print("{}\n", src); EXPECT_TRUE(false); } @@ -478,14 +494,15 @@ TEST_F(OfflineDecompilation, Compile) { Timer timer; int total_lines = 0; - for (auto& file : g_object_files_to_check_against_reference) { - if (skip_files_in_compiling.find(file) != skip_files_in_compiling.end()) { + for (auto& file : g_object_files_to_decompile_or_ref_check) { + if (g_object_files_to_ignore_ref_checks.find(file.first) != + g_object_files_to_ignore_ref_checks.end()) { continue; } - lg::info("Compiling {}...", file); + lg::info("Compiling {}...", file.first); - auto& obj_l = db->obj_files_by_name.at(file); + auto& obj_l = db->obj_files_by_name.at(file.first); ASSERT_EQ(obj_l.size(), 1); std::string src = db->ir2_final_out(obj_l.at(0), skip_in_compiling);