Archive/jak-project
1
0
Fork 0
mirror of https://github.com/open-goal/jak-project.git synced 2024-10-20 00:57:44 -04:00
Code Issues Actions 3 Packages Projects Releases Wiki Activity

Process Text Files in Decompiler (#122)

Browse source 
* begin support for v2

* export game text

* generate text files

* working text load

* fix windows

* add test and clean up game tests a bit

* load the right file

* add separate program to launch the data compiler

* add offline test script
This commit is contained in:
water111 2020-11-19 21:22:16 -05:00 committed by GitHub
parent ae053870c3
commit 953c1512db
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
48 changed files with 1794 additions and 200 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
check.sh Executable file
View file

@ -0,0 +1,6 @@
#!/bin/bash
# Directory of this script
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
cd ${DIR}/out
md5sum --check hash.md5

8
common/link_types.h
View file

@ -36,4 +36,12 @@ struct ObjectHeader {
char name[60];
};
// Header for link data used for V2 linking data
// used in GOAL and OpenGOAL
struct LinkHeaderV2 {
uint32_t type_tag; // always -1
uint32_t length; // length of link data
uint32_t version; // always 2
};
#endif // JAK1_LINK_TYPES_H

3
decompiler/CMakeLists.txt
View file

@ -19,7 +19,8 @@ add_executable(decompiler
IR/IR.cpp
IR/IR_TypeAnalysis.cpp
Function/TypeInspector.cpp
data/tpage.cpp)
data/tpage.cpp
data/game_text.cpp)
target_link_libraries(decompiler
goos

1
decompiler/ObjectFile/LinkedObjectFile.cpp
View file

@ -823,6 +823,7 @@ std::string LinkedObjectFile::get_goal_string(int seg, int word_idx, bool with_q
char cword[4];
memcpy(cword, &word.data, 4);
result += cword[byte_offset];
assert(result.back() != 0);
}
if (with_quotes) {
result += "\"";

72
decompiler/ObjectFile/LinkedObjectFileCreation.cpp
View file

@ -9,6 +9,7 @@
#include "LinkedObjectFileCreation.h"
#include "decompiler/config.h"
#include "decompiler/util/DecompilerTypeSystem.h"
#include "common/link_types.h"
// There are three link versions:
// V2 - not really in use anymore, but V4 will resue logic from it (and the game didn't rename the
@ -26,13 +27,6 @@ struct LinkHeaderCommon {
uint16_t version; // what version (2, 3, 4)
};
// Header for link data used for V2 linking data
struct LinkHeaderV2 {
uint32_t type_tag; // always -1
uint32_t length; // length of link data
uint32_t version; // always 2
};
// Header for link data used for V4
struct LinkHeaderV4 {
uint32_t type_tag; // always -1
@ -101,12 +95,15 @@ static uint32_t c_symlink2(LinkedObjectFile& f,
uint32_t next_reloc = link_ptr_offset + 1;
if (seek & 3) {
// 0b01, 0b10
seek = (relocPtr[1] << 8) | table_value;
next_reloc = link_ptr_offset + 2;
if (seek & 2) {
// 0b10
seek = (relocPtr[2] << 16) | seek;
next_reloc = link_ptr_offset + 3;
if (seek & 1) {
// 0b11
seek = (relocPtr[3] << 24) | seek;
next_reloc = link_ptr_offset + 4;
}
@ -216,24 +213,50 @@ static uint32_t align16(uint32_t in) {
}
/*!
* Process link data for a "V4" object file.
* Process link data for a "V4" or "V2" object file.
* In reality a V4 seems to be just a V2 object, but with the link data after the real data.
* There's a V4 header at the very beginning, but another V2 header/link data at the end
* -----------------------------------------------
* | V4 header | data | V2 header | V2 link data |
* -----------------------------------------------
*
* V2
* -----------------------------------
* | V2 header | V2 link data | data |
* -----------------------------------
* The V4 format avoids having to copy the data to the left once the V2 link data is discarded.
* Presumably once they decided that data could never be relocated after being loaded in,
* it became worth it to throw away the link data, and avoid the memcpy of the data.
* The memcpy is surprisingly expensive, when you consider the linker ran for ~3% of a frame each
* frame and level data is ~10 MB.
*/
static void link_v4(LinkedObjectFile& f,
const std::vector<uint8_t>& data,
const std::string& name,
DecompilerTypeSystem& dts) {
// read the V4 header to find where the link data really is
static void link_v2_or_v4(LinkedObjectFile& f,
const std::vector<uint8_t>& data,
const std::string& name,
DecompilerTypeSystem& dts) {
const auto* header = (const LinkHeaderV4*)&data.at(0);
uint32_t link_data_offset = header->code_size + sizeof(LinkHeaderV4); // no basic offset
assert(header->version == 4 || header->version == 2);
// code starts immediately after the header
uint32_t code_offset = sizeof(LinkHeaderV4);
uint32_t code_size = header->code_size;
// these are different depending on the version.
uint32_t code_offset, link_data_offset, code_size;
if (header->version == 4) {
// code starts immediately after the V4 header
code_offset = sizeof(LinkHeaderV4);
// link_data_offset points to a V2 header
link_data_offset = header->code_size + sizeof(LinkHeaderV4);
// code size is specified!
code_size = header->code_size;
} else {
// link data starts immediately
link_data_offset = 0;
// code is after all the link data
code_offset = header->length;
// we have to compute the code size ourself
code_size = data.size() - code_offset;
assert(header->type_tag == 0xffffffff);
}
f.stats.total_code_bytes += code_size;
f.stats.total_v2_code_bytes += code_size;
@ -241,7 +264,7 @@ static void link_v4(LinkedObjectFile& f,
// add all code
const uint8_t* code_start = &data.at(code_offset);
const uint8_t* code_end =
&data.at(code_offset + code_size); // safe because link data is after code.
&data.at(code_offset + code_size - 1) + 1; // get the pointer to one past the end.
assert(((code_end - code_start) % 4) == 0);
f.set_segment_count(1);
for (auto x = code_start; x < code_end; x += 4) {
@ -250,12 +273,13 @@ static void link_v4(LinkedObjectFile& f,
// read v2 header after the code
const uint8_t* link_data = &data.at(link_data_offset);
const auto* link_header_v2 = (const LinkHeaderV2*)(link_data); // subtract off type tag
uint32_t link_ptr_offset = link_data_offset;
link_ptr_offset += sizeof(LinkHeaderV2);
auto* link_header_v2 = (const LinkHeaderV2*)(link_data);
assert(link_header_v2->type_tag == 0xffffffff);
assert(link_header_v2->version == 2);
assert(link_header_v2->length == header->length);
f.stats.total_v2_link_bytes += link_header_v2->length;
uint32_t link_ptr_offset = link_data_offset + sizeof(LinkHeaderV2);
// first "section" of link data is a list of where all the pointer are.
if (data.at(link_ptr_offset) == 0) {
@ -369,7 +393,8 @@ static void link_v4(LinkedObjectFile& f,
// check length
assert(link_header_v2->length == align64(link_ptr_offset - link_data_offset + 1));
while (link_ptr_offset < data.size()) {
size_t expected_end = header->version == 4 ? data.size() : link_header_v2->length;
while (link_ptr_offset < expected_end) {
assert(data.at(link_ptr_offset) == 0);
link_ptr_offset++;
}
@ -790,12 +815,13 @@ LinkedObjectFile to_linked_object_file(const std::vector<uint8_t>& data,
if (header->version == 3) {
assert(header->type_tag == 0);
link_v3(result, data, name, dts);
} else if (header->version == 4) {
} else if (header->version == 4 || header->version == 2) {
assert(header->type_tag == 0xffffffff);
link_v4(result, data, name, dts);
link_v2_or_v4(result, data, name, dts);
} else if (header->version == 5) {
link_v5(result, data, name, dts);
} else {
printf("Unsupported version %d\n", header->version);
assert(false);
}

70
decompiler/ObjectFile/ObjectFileDB.cpp
View file

@ -11,6 +11,7 @@
#include <cstring>
#include <map>
#include "decompiler/data/tpage.h"
#include "decompiler/data/game_text.h"
#include "LinkedObjectFileCreation.h"
#include "decompiler/config.h"
#include "third-party/minilzo/minilzo.h"
@ -32,6 +33,37 @@ std::string strip_dgo_extension(const std::string& x) {
}
return x;
}
/*!
* Get an object name from a file name.
* Strips off the file extension and anything before the last slash.
*/
std::string obj_filename_to_name(const std::string& x) {
auto end = x.length();
// find last dot
auto last_dot = end;
for (; last_dot-- > 0;) {
if (x.at(last_dot) == '.') {
break;
}
}
if (last_dot == 0) {
last_dot = end;
}
auto last_slash = end;
for (; last_slash-- > 0;) {
if (x.at(last_slash) == '\\' || x.at(last_slash) == '/') {
break;
}
}
assert(last_dot > last_slash + 1);
assert(last_slash + 1 < x.length());
return x.substr(last_slash + 1, last_dot - last_slash - 1);
}
} // namespace
std::string ObjectFileData::to_unique_name() const {
@ -53,6 +85,7 @@ std::string ObjectFileData::to_unique_name() const {
return record.name;
}
}
ObjectFileData& ObjectFileDB::lookup_record(const ObjectFileRecord& rec) {
ObjectFileData* result = nullptr;
@ -72,7 +105,8 @@ ObjectFileData& ObjectFileDB::lookup_record(const ObjectFileRecord& rec) {
* Build an object file DB for the given list of DGOs.
*/
ObjectFileDB::ObjectFileDB(const std::vector<std::string>& _dgos,
const std::string& obj_file_name_map_file) {
const std::string& obj_file_name_map_file,
const std::vector<std::string>& object_files) {
Timer timer;
spdlog::info("-Loading types...");
@ -93,6 +127,12 @@ ObjectFileDB::ObjectFileDB(const std::vector<std::string>& _dgos,
get_objs_from_dgo(dgo);
}
for (auto& obj : object_files) {
auto data = file_util::read_binary_file(obj);
auto name = obj_filename_to_name(obj);
add_obj_from_dgo(name, name, data.data(), data.size(), "NO-XGO");
}
spdlog::info("ObjectFileDB Initialized:");
spdlog::info("Total DGOs: {}", int(_dgos.size()));
spdlog::info("Total data: {} bytes", stats.total_dgo_bytes);
@ -629,6 +669,34 @@ void ObjectFileDB::process_tpages() {
100.f * float(success) / float(total), timer.getMs());
}
std::string ObjectFileDB::process_game_text() {
spdlog::info("- Finding game text...");
std::string text_string = "COMMON";
Timer timer;
int file_count = 0;
int string_count = 0;
int char_count = 0;
std::unordered_map<int, std::unordered_map<int, std::string>> text_by_language_by_id;
for_each_obj([&](ObjectFileData& data) {
if (data.name_in_dgo.substr(1) == text_string) {
file_count++;
auto statistics = ::process_game_text(data);
string_count += statistics.total_text;
char_count += statistics.total_chars;
if (text_by_language_by_id.find(statistics.language) != text_by_language_by_id.end()) {
assert(false);
}
text_by_language_by_id[statistics.language] = std::move(statistics.text);
}
});
spdlog::info("Processed {} text files ({} strings, {} characters) in {:.2f} ms", file_count,
string_count, char_count, timer.getMs());
return write_game_text(text_by_language_by_id);
}
void ObjectFileDB::analyze_functions() {
spdlog::info("- Analyzing Functions...");
Timer timer;

5
decompiler/ObjectFile/ObjectFileDB.h
View file

@ -45,7 +45,9 @@ struct ObjectFileData {
class ObjectFileDB {
public:
ObjectFileDB(const std::vector<std::string>& _dgos, const std::string& obj_file_name_map_file);
ObjectFileDB(const std::vector<std::string>& _dgos,
const std::string& obj_file_name_map_file,
const std::vector<std::string>& object_files);
std::string generate_dgo_listing();
std::string generate_obj_listing();
void process_link_data();
@ -57,6 +59,7 @@ class ObjectFileDB {
void write_disassembly(const std::string& output_dir, bool disassemble_objects_without_functions);
void analyze_functions();
void process_tpages();
std::string process_game_text();
ObjectFileData& lookup_record(const ObjectFileRecord& rec);
DecompilerTypeSystem dts;

2
decompiler/config.cpp
View file

@ -15,6 +15,7 @@ void set_config(const std::string& path_to_config_file) {
gConfig.game_version = cfg.at("game_version").get<int>();
gConfig.dgo_names = cfg.at("dgo_names").get<std::vector<std::string>>();
gConfig.object_file_names = cfg.at("object_file_names").get<std::vector<std::string>>();
if (cfg.contains("obj_file_name_map_file")) {
gConfig.obj_file_name_map_file = cfg.at("obj_file_name_map_file").get<std::string>();
}
@ -27,6 +28,7 @@ void set_config(const std::string& path_to_config_file) {
gConfig.write_hex_near_instructions = cfg.at("write_hex_near_instructions").get<bool>();
gConfig.analyze_functions = cfg.at("analyze_functions").get<bool>();
gConfig.process_tpages = cfg.at("process_tpages").get<bool>();
gConfig.process_game_text = cfg.at("process_game_text").get<bool>();
std::vector<std::string> asm_functions_by_name =
cfg.at("asm_functions_by_name").get<std::vector<std::string>>();

2
decompiler/config.h
View file

@ -10,6 +10,7 @@
struct Config {
int game_version = -1;
std::vector<std::string> dgo_names;
std::vector<std::string> object_file_names;
std::unordered_set<std::string> bad_inspect_types;
std::string obj_file_name_map_file;
bool write_disassembly = false;
@ -20,6 +21,7 @@ struct Config {
bool write_hex_near_instructions = false;
bool analyze_functions = false;
bool process_tpages = false;
bool process_game_text = false;
std::unordered_set<std::string> asm_functions_by_name;
// ...
};

7
decompiler/config/all-types.gc
View file

@ -2570,8 +2570,9 @@
(deftype game-text (structure)
((id uint32 :offset-assert 0)
(text basic :offset-assert 4)
(text string :offset-assert 4)
)
:pack-me
:method-count-assert 9
:size-assert #x8
:flag-assert #x900000008
@ -2580,8 +2581,8 @@
(deftype game-text-info (basic)
((length int32 :offset-assert 4)
(language-id int32 :offset-assert 8)
(group-name basic :offset-assert 12)
(data uint8 :dynamic :offset-assert 16)
(group-name string :offset-assert 12)
(data game-text :dynamic :inline :offset-assert 16)
)
:method-count-assert 10
:size-assert #x10

6
decompiler/config/jak1_ntsc_black_label.jsonc
View file

@ -2,7 +2,7 @@
{
"game_version":1,
// the order here matters. KERNEL and GAME should go first
// the order here matters (not sure that this is true any more...). KERNEL and GAME should go first
"dgo_names":["CGO/KERNEL.CGO","CGO/GAME.CGO",
"CGO/ENGINE.CGO"
, "CGO/ART.CGO", "DGO/BEA.DGO", "DGO/CIT.DGO", "CGO/COMMON.CGO", "DGO/DAR.DGO", "DGO/DEM.DGO",
@ -12,6 +12,9 @@
"DGO/VI2.DGO", "DGO/VI3.DGO", "CGO/VILLAGEP.CGO", "CGO/WATER-AN.CGO"
],
"object_file_names":["TEXT/0COMMON.TXT", "TEXT/1COMMON.TXT", "TEXT/2COMMON.TXT", "TEXT/3COMMON.TXT", "TEXT/4COMMON.TXT",
"TEXT/5COMMON.TXT", "TEXT/6COMMON.TXT"],
"analyze_functions":true,
"write_disassembly":true,
@ -21,6 +24,7 @@
"disassemble_objects_without_functions":false,
"process_tpages":true,
"process_game_text":true,
// to write out data of each object file
"write_hexdump":false,

162
decompiler/data/game_text.cpp Normal file
View file

@ -0,0 +1,162 @@
#include <cstring>
#include <map>
#include <vector>
#include <algorithm>
#include "third-party/fmt/core.h"
#include "game_text.h"
#include "decompiler/ObjectFile/ObjectFileDB.h"
#include "common/goos/Reader.h"
namespace {
template <typename T>
T get_word(const LinkedWord& word) {
T result;
assert(word.kind == LinkedWord::PLAIN_DATA);
static_assert(sizeof(T) == 4, "bad get_word size");
memcpy(&result, &word.data, 4);
return result;
}
Label get_label(ObjectFileData& data, const LinkedWord& word) {
assert(word.kind == LinkedWord::PTR);
return data.linked_data.labels.at(word.label_id);
}
int align16(int in) {
return (in + 15) & ~15;
}
} // namespace
/*
(deftype game-text (structure)
((id uint32 :offset-assert 0)
(text basic :offset-assert 4)
)
)
(deftype game-text-info (basic)
((length int32 :offset-assert 4)
(language-id int32 :offset-assert 8)
(group-name basic :offset-assert 12)
(data game-text :dynamic :offset-assert 16)
)
)
*/
GameTextResult process_game_text(ObjectFileData& data) {
GameTextResult result;
auto& words = data.linked_data.words_by_seg.at(0);
std::vector<int> read_words(words.size(), false);
int offset = 0;
// type tage for game-text-info
if (words.at(offset).kind != LinkedWord::TYPE_PTR ||
words.front().symbol_name != "game-text-info") {
assert(false);
}
read_words.at(offset)++;
offset++;
// length field
read_words.at(offset)++;
u32 text_count = get_word<u32>(words.at(offset++));
// language-id field
read_words.at(offset)++;
u32 language = get_word<u32>(words.at(offset++));
result.language = language;
// group-name field
read_words.at(offset)++;
auto group_label = get_label(data, words.at(offset++));
auto group_name = data.linked_data.get_goal_string_by_label(group_label);
assert(group_name == "common");
// remember that we read these bytes
auto group_start = (group_label.offset / 4) - 1;
for (int j = 0; j < align16(8 + 1 + group_name.length()) / 4; j++) {
read_words.at(group_start + j)++;
}
// read each text...
for (u32 i = 0; i < text_count; i++) {
// id number
read_words.at(offset)++;
auto text_id = get_word<u32>(words.at(offset++));
// label to string
read_words.at(offset)++;
auto text_label = get_label(data, words.at(offset++));
// actual string
auto text = data.linked_data.get_goal_string_by_label(text_label);
result.total_text++;
result.total_chars += text.length();
// no duplicate ids
if (result.text.find(text_id) != result.text.end()) {
assert(false);
}
// escape characters
result.text[text_id] = goos::get_readable_string(text.c_str());
// remember what we read (-1 for the type tag)
auto string_start = (text_label.offset / 4) - 1;
// 8 for type tag and length fields, 1 for null char.
for (int j = 0; j < align16(8 + 1 + text.length()) / 4; j++) {
read_words.at(string_start + j)++;
}
}
// alignment to the string section.
while (offset & 3) {
read_words.at(offset)++;
offset++;
}
// make sure we read each thing at least once.
// reading more than once is ok, some text is duplicated.
for (int i = 0; i < int(words.size()); i++) {
if (read_words[i] < 1) {
std::string debug;
data.linked_data.append_word_to_string(debug, words.at(i));
printf("[%d] %d 0x%s\n", i, int(read_words[i]), debug.c_str());
assert(false);
}
}
return result;
}
std::string write_game_text(
const std::unordered_map<int, std::unordered_map<int, std::string>>& data) {
// first sort languages:
std::vector<int> langauges;
for (const auto& lang : data) {
langauges.push_back(lang.first);
}
std::sort(langauges.begin(), langauges.end());
// build map
std::map<int, std::vector<std::string>> text_by_id;
for (auto lang : langauges) {
for (auto text : data.at(lang)) {
text_by_id[text.first].push_back(text.second);
}
}
// write!
std::string result = fmt::format("(language-count {})\n", langauges.size());
result += "(group-name \"common\")\n";
for (auto& x : text_by_id) {
result += fmt::format("(#x{:04x}\n ", x.first);
for (auto& y : x.second) {
result += fmt::format("\"{}\"\n ", y);
}
result += ")\n\n";
}
return result;
}

16
decompiler/data/game_text.h Normal file
View file

@ -0,0 +1,16 @@
#pragma once
#include <string>
#include <unordered_map>
class ObjectFileData;
struct GameTextResult {
int total_text = 0;
int language = -1;
std::unordered_map<int, std::string> text;
int total_chars = 0;
};
GameTextResult process_game_text(ObjectFileData& data);
std::string write_game_text(
const std::unordered_map<int, std::unordered_map<int, std::string>>& data);

17
decompiler/main.cpp
View file

@ -11,8 +11,8 @@ int main(int argc, char** argv) {
spdlog::info("Beginning disassembly. This may take a few minutes...");
spdlog::set_level(spdlog::level::debug);
auto lu = spdlog::basic_logger_mt("GOAL Decompiler", "logs/decompiler.log");
spdlog::set_default_logger(lu);
// auto lu = spdlog::basic_logger_mt("GOAL Decompiler", "logs/decompiler.log");
// spdlog::set_default_logger(lu);
spdlog::flush_on(spdlog::level::info);
file_util::init_crc();
@ -27,12 +27,16 @@ int main(int argc, char** argv) {
std::string in_folder = argv[2];
std::string out_folder = argv[3];
std::vector<std::string> dgos;
std::vector<std::string> dgos, objs;
for (const auto& dgo_name : get_config().dgo_names) {
dgos.push_back(file_util::combine_path(in_folder, dgo_name));
}
ObjectFileDB db(dgos, get_config().obj_file_name_map_file);
for (const auto& obj_name : get_config().object_file_names) {
objs.push_back(file_util::combine_path(in_folder, obj_name));
}
ObjectFileDB db(dgos, get_config().obj_file_name_map_file, objs);
file_util::write_text_file(file_util::combine_path(out_folder, "dgo.txt"),
db.generate_dgo_listing());
file_util::write_text_file(file_util::combine_path(out_folder, "obj.txt"),
@ -54,6 +58,11 @@ int main(int argc, char** argv) {
db.analyze_functions();
}
if (get_config().process_game_text) {
auto result = db.process_game_text();
file_util::write_text_file(file_util::get_file_path({"assets", "game_text.txt"}), result);
}
if (get_config().process_tpages) {
db.process_tpages();
}

6
game/kernel/fileio.cpp
View file

@ -7,7 +7,7 @@
#include <cassert>
#include <cstring>
#include <cstdio>
#include "game/sce/stubs.h"
#include "game/sce/sif_ee.h"
#include "fileio.h"
#include "kprint.h"
#include "common/versions.h"
@ -255,6 +255,8 @@ char* DecodeFileName(const char* name) {
result = MakeFileName(CODE_FILE_TYPE, name + 6, 0);
} else if (!strncmp(name, "$RES/", 5)) {
result = MakeFileName(RES_FILE_TYPE, name + 5, 0);
} else if (!strncmp(name, "$JAK-PROJECT/", 13)) {
result = MakeFileName(JAK_PROJECT_FILE_TYPE, name + 13, 0);
} else {
printf("[ERROR] DecodeFileName: UNKNOWN FILE NAME %s\n", name);
result = nullptr;
@ -378,6 +380,8 @@ char* MakeFileName(int type, const char* name, int new_string) {
// REFPLANT? no idea
static char nextDir[] = "/";
sprintf(buf, "%sconfig_data/refplant/%s", nextDir, name);
} else if (type == JAK_PROJECT_FILE_TYPE) {
sprintf(buffer_633, "/%s", name);
} else {
printf("UNKNOWN FILE TYPE %d\n", type);
}

2
game/kernel/fileio.h
View file

@ -42,6 +42,8 @@ enum GoalFileType {
CNT_FILE_TYPE = 0x3a,
RES_FILE_TYPE = 0x3b,
REFPLANT_FILE_TYPE = 0x301,
// added this, allows access directly to jak-project/ from within the game.
JAK_PROJECT_FILE_TYPE = 0x302
};
constexpr char FOLDER_PREFIX[] = "";

500
game/kernel/klink.cpp
View file

@ -20,6 +20,11 @@
namespace {
// turn on printf's for debugging linking issues.
constexpr bool link_debug_printfs = false;
bool is_opengoal_object(const void* data) {
auto* header = (const LinkHeaderV2*)data;
return !(header->type_tag == 0xffffffff && (header->version == 2 || header->version == 4));
}
} // namespace
// space to store a single in-progress linking state.
@ -41,105 +46,184 @@ void link_control::begin(Ptr<uint8_t> object_file,
int32_t size,
Ptr<kheapinfo> heap,
uint32_t flags) {
// save data from call to begin
m_object_data = object_file;
kstrcpy(m_object_name, name);
m_object_size = size;
m_heap = heap;
m_flags = flags;
if (is_opengoal_object(object_file.c())) {
// save data from call to begin
m_object_data = object_file;
kstrcpy(m_object_name, name);
m_object_size = size;
m_heap = heap;
m_flags = flags;
// initialize link control
m_entry.offset = 0;
m_heap_top = m_heap->top;
m_keep_debug = false;
// initialize link control
m_entry.offset = 0;
m_heap_top = m_heap->top;
m_keep_debug = false;
m_opengoal = true;
if (link_debug_printfs) {
char* goal_name = object_file.cast<char>().c();
printf("link %s\n", goal_name);
printf("link_control::begin %c%c%c%c\n", goal_name[0], goal_name[1], goal_name[2],
goal_name[3]);
}
// points to the beginning of the linking data
m_link_block_ptr = object_file + BASIC_OFFSET;
m_code_size = 0;
m_code_start = object_file;
m_state = 0;
m_segment_process = 0;
ObjectFileHeader* ofh = m_link_block_ptr.cast<ObjectFileHeader>().c();
if (ofh->goal_version_major != versions::GOAL_VERSION_MAJOR) {
fprintf(
stderr,
"VERSION ERROR: C Kernel built from GOAL %d.%d, but object file %s is from GOAL %d.%d\n",
versions::GOAL_VERSION_MAJOR, versions::GOAL_VERSION_MINOR, name, ofh->goal_version_major,
ofh->goal_version_minor);
exit(0);
}
if (link_debug_printfs) {
printf("Object file header:\n");
printf(" GOAL ver %d.%d obj %d len %d\n", ofh->goal_version_major, ofh->goal_version_minor,
ofh->object_file_version, ofh->link_block_length);
printf(" segment count %d\n", ofh->segment_count);
for (int i = 0; i < N_SEG; i++) {
printf(" seg %d link 0x%04x, 0x%04x data 0x%04x, 0x%04x\n", i, ofh->link_infos[i].offset,
ofh->link_infos[i].size, ofh->code_infos[i].offset, ofh->code_infos[i].size);
if (link_debug_printfs) {
char* goal_name = object_file.cast<char>().c();
printf("link %s\n", goal_name);
printf("link_control::begin %c%c%c%c\n", goal_name[0], goal_name[1], goal_name[2],
goal_name[3]);
}
}
m_version = ofh->object_file_version;
if (ofh->object_file_version < 4) {
// three segment file
// points to the beginning of the linking data
m_link_block_ptr = object_file + BASIC_OFFSET;
m_code_size = 0;
m_code_start = object_file;
m_state = 0;
m_segment_process = 0;
// seek past the header
m_object_data.offset += ofh->link_block_length;
// todo, set m_code_size
if (m_link_block_ptr.offset < m_heap->base.offset ||
m_link_block_ptr.offset >= m_heap->top.offset) {
// the link block is outside our heap, or in the top of our heap. It's somebody else's
// problem.
if (link_debug_printfs) {
printf("Link block somebody else's problem\n");
ObjectFileHeader* ofh = m_link_block_ptr.cast<ObjectFileHeader>().c();
if (ofh->goal_version_major != versions::GOAL_VERSION_MAJOR) {
fprintf(
stderr,
"VERSION ERROR: C Kernel built from GOAL %d.%d, but object file %s is from GOAL %d.%d\n",
versions::GOAL_VERSION_MAJOR, versions::GOAL_VERSION_MINOR, name, ofh->goal_version_major,
ofh->goal_version_minor);
exit(0);
}
if (link_debug_printfs) {
printf("Object file header:\n");
printf(" GOAL ver %d.%d obj %d len %d\n", ofh->goal_version_major, ofh->goal_version_minor,
ofh->object_file_version, ofh->link_block_length);
printf(" segment count %d\n", ofh->segment_count);
for (int i = 0; i < N_SEG; i++) {
printf(" seg %d link 0x%04x, 0x%04x data 0x%04x, 0x%04x\n", i, ofh->link_infos[i].offset,
ofh->link_infos[i].size, ofh->code_infos[i].offset, ofh->code_infos[i].size);
}
}
if (m_heap->base.offset <= m_object_data.offset && // above heap base
m_object_data.offset < m_heap->top.offset && // less than heap top (not needed?)
m_object_data.offset < m_heap->current.offset) { // less than heap current
m_version = ofh->object_file_version;
if (ofh->object_file_version < 4) {
// three segment file
// seek past the header
m_object_data.offset += ofh->link_block_length;
// todo, set m_code_size
if (m_link_block_ptr.offset < m_heap->base.offset ||
m_link_block_ptr.offset >= m_heap->top.offset) {
// the link block is outside our heap, or in the top of our heap. It's somebody else's
// problem.
if (link_debug_printfs) {
printf("Code block in the heap, kicking it out for copy into heap\n");
printf("Link block somebody else's problem\n");
}
if (m_heap->base.offset <= m_object_data.offset && // above heap base
m_object_data.offset < m_heap->top.offset && // less than heap top (not needed?)
m_object_data.offset < m_heap->current.offset) { // less than heap current
if (link_debug_printfs) {
printf("Code block in the heap, kicking it out for copy into heap\n");
}
m_heap->current = m_object_data;
}
} else {
// in our heap, we need to move it so we can free up its space later on
if (link_debug_printfs) {
printf("Link block needs to be moved!\n");
}
// allocate space for a new one
auto new_link_block = kmalloc(m_heap, ofh->link_block_length, KMALLOC_TOP, "link-block");
auto old_link_block = m_link_block_ptr - BASIC_OFFSET;
// copy it
ultimate_memcpy(new_link_block.c(), old_link_block.c(), ofh->link_block_length);
m_link_block_ptr = new_link_block + BASIC_OFFSET;
// if we can save some memory here
if (old_link_block.offset < m_heap->current.offset) {
if (link_debug_printfs) {
printf("Kick out old link block\n");
}
m_heap->current = old_link_block;
}
m_heap->current = m_object_data;
}
} else {
// in our heap, we need to move it so we can free up its space later on
if (link_debug_printfs) {
printf("Link block needs to be moved!\n");
}
printf("UNHANDLED OBJECT FILE VERSION\n");
assert(false);
}
// allocate space for a new one
auto new_link_block = kmalloc(m_heap, ofh->link_block_length, KMALLOC_TOP, "link-block");
auto old_link_block = m_link_block_ptr - BASIC_OFFSET;
// copy it
ultimate_memcpy(new_link_block.c(), old_link_block.c(), ofh->link_block_length);
m_link_block_ptr = new_link_block + BASIC_OFFSET;
// if we can save some memory here
if (old_link_block.offset < m_heap->current.offset) {
if (link_debug_printfs) {
printf("Kick out old link block\n");
}
m_heap->current = old_link_block;
}
if ((m_flags & LINK_FLAG_FORCE_DEBUG) && MasterDebug && !DiskBoot) {
m_keep_debug = true;
}
} else {
printf("UNHANDLED OBJECT FILE VERSION\n");
assert(false);
}
m_opengoal = false;
// not an open goal object.
if (link_debug_printfs) {
printf("Linking GOAL style object\n");
}
if ((m_flags & LINK_FLAG_FORCE_DEBUG) && MasterDebug && !DiskBoot) {
m_keep_debug = true;
// initialize
m_object_data = object_file;
kstrcpy(m_object_name, name);
m_object_size = size;
m_heap = heap;
m_flags = flags;
m_entry.offset = 0;
m_heap_top = m_heap->top;
m_keep_debug = false;
m_link_block_ptr = object_file + BASIC_OFFSET;
m_code_size = 0;
m_code_start = object_file;
m_state = 0;
m_segment_process = 0;
const auto* header = (LinkHeaderV2*)(m_link_block_ptr.c() - 4);
m_version = header->version;
if (header->version < 4) {
// seek past header
m_object_data.offset += header->length;
m_code_size = m_object_size - header->length;
if (m_link_block_ptr.offset < m_heap->base.offset ||
m_link_block_ptr.offset >= m_heap->top.offset) {
// the link block is outside our heap, or in the top of our heap. It's somebody else's
// problem.
if (link_debug_printfs) {
printf("Link block somebody else's problem\n");
}
if (m_heap->base.offset <= m_object_data.offset && // above heap base
m_object_data.offset < m_heap->top.offset && // less than heap top (not needed?)
m_object_data.offset < m_heap->current.offset) { // less than heap current
if (link_debug_printfs) {
printf("Code block in the heap, kicking it out for copy into heap\n");
}
m_heap->current = m_object_data;
}
} else {
// in our heap, we need to move it so we can free up its space later on
if (link_debug_printfs) {
printf("Link block needs to be moved!\n");
}
// allocate space for a new one
auto new_link_block = kmalloc(m_heap, header->length, KMALLOC_TOP, "link-block");
auto old_link_block = m_link_block_ptr - BASIC_OFFSET;
// copy it
ultimate_memcpy(new_link_block.c(), old_link_block.c(), header->length);
m_link_block_ptr = new_link_block + BASIC_OFFSET;
// if we can save some memory here
if (old_link_block.offset < m_heap->current.offset) {
if (link_debug_printfs) {
printf("Kick out old link block\n");
}
m_heap->current = old_link_block;
}
}
} else {
// not yet implemented
assert(false);
}
if ((m_flags & LINK_FLAG_FORCE_DEBUG) && MasterDebug && !DiskBoot) {
m_keep_debug = true;
}
}
}
@ -158,9 +242,13 @@ uint32_t link_control::work() {
uint32_t rv;
if (m_version == 3) {
assert(m_opengoal);
rv = work_v3();
} else if (m_version == 2 || m_version == 4) {
assert(!m_opengoal);
rv = work_v2();
} else {
printf("UNHANDLED OBJECT FILE VERSION IN WORK!\n");
printf("UNHANDLED OBJECT FILE VERSION %d IN WORK!\n", m_version);
assert(false);
return 0;
}
@ -409,7 +497,247 @@ uint32_t link_control::work_v3() {
}
}
// TODO - work_v2, once v2 objects are created.
Ptr<u8> c_symlink2(Ptr<u8> objData, Ptr<u8> linkObj, Ptr<u8> relocTable) {
u8* relocPtr = relocTable.c();
Ptr<u8> objPtr = objData;
do {
u8 table_value = *relocPtr;
u32 result = table_value;
u8* next_reloc = relocPtr + 1;
if (result & 3) {
result = (relocPtr[1] << 8) | table_value;
next_reloc = relocPtr + 2;
if (result & 2) {
result = (relocPtr[2] << 16) | result;
next_reloc = relocPtr + 3;
if (result & 1) {
result = (relocPtr[3] << 24) | result;
next_reloc = relocPtr + 4;
}
}
}
relocPtr = next_reloc;
objPtr = objPtr + (result & 0xfffffffc);
u32 objValue = *(objPtr.cast<u32>());
if (objValue == 0xffffffff) {
*(objPtr.cast<u32>()) = linkObj.offset;
} else {
// I don't think we should hit this ever.
assert(false);
}
} while (*relocPtr);
return make_ptr(relocPtr + 1);
}
#define LINK_V2_STATE_INIT_COPY 0
#define LINK_V2_STATE_OFFSETS 1
#define LINK_V2_STATE_SYMBOL_TABLE 2
#define OBJ_V2_CLOSE_ENOUGH 0x90
#define OBJ_V2_MAX_TRANSFER 0x80000
uint32_t link_control::work_v2() {
// u32 startCycle = kernel.read_clock(); todo
if (m_state == LINK_V2_STATE_INIT_COPY) { // initialization and copying to heap
// we move the data segment to eliminate gaps
// very small gaps can be tolerated, as it is not worth the time penalty to move large objects
// many bytes. if this requires copying a large amount of data, we will do it in smaller chunks,
// allowing the copy to be spread over multiple game frames
// state initialization
if (m_segment_process == 0) {
m_heap_gap =
m_object_data - m_heap->current; // distance between end of heap and start of object
if (m_object_data.offset < m_heap->current.offset) {
assert(false);
}
}
if (m_heap_gap <
OBJ_V2_CLOSE_ENOUGH) { // close enough, don't relocate the object, just expand the heap
if (link_debug_printfs) {
printf("[work_v2] close enough, not moving\n");
}
m_heap->current = m_object_data + m_code_size;
if (m_heap->top.offset <= m_heap->current.offset) {
MsgErr("dkernel: heap overflow\n"); // game has ~% instead of \n :P
return 1;
}
} else { // not close enough, need to move the object
// on the first run of this state...
if (m_segment_process == 0) {
m_original_object_location = m_object_data;
// allocate on heap, will have no gap
m_object_data = kmalloc(m_heap, m_code_size, 0, "data-segment");
if (link_debug_printfs) {
printf("[work_v2] moving from 0x%x to 0x%x\n", m_original_object_location.offset,
m_object_data.offset);
}
if (!m_object_data.offset) {
MsgErr("dkernel: unable to malloc %d bytes for data-segment\n", m_code_size);
return 1;
}
}
// the actual copy
Ptr<u8> source = m_original_object_location + m_segment_process;
u32 size = m_code_size - m_segment_process;
if (size > OBJ_V2_MAX_TRANSFER) { // around .5 MB
ultimate_memcpy((m_object_data + m_segment_process).c(), source.c(), OBJ_V2_MAX_TRANSFER);
m_segment_process += OBJ_V2_MAX_TRANSFER;
return 0; // return, don't want to take too long.
}
// if we have bytes to copy, but they are less than the max transfer, do it in one shot!
if (size) {
ultimate_memcpy((m_object_data + m_segment_process).c(), source.c(), size);
if (m_segment_process > 0) { // if we did a previous copy, we return now....
m_state = LINK_V2_STATE_OFFSETS;
m_segment_process = 0;
return 0;
}
}
}
// otherwise go straight into the next state.
m_state = LINK_V2_STATE_OFFSETS;
m_segment_process = 0;
}
// init offset phase
if (m_state == LINK_V2_STATE_OFFSETS && m_segment_process == 0) {
m_reloc_ptr = m_link_block_ptr + 8; // seek to link table
if (*m_reloc_ptr == 0) { // do we have pointer links to do?
m_reloc_ptr.offset++; // if not, seek past the \0, and go to next state
m_state = LINK_V2_STATE_SYMBOL_TABLE;
m_segment_process = 0;
} else {
m_base_ptr = m_object_data; // base address for offsetting.
m_loc_ptr = m_object_data; // pointer which seeks thru the code
m_table_toggle = 0; // are we seeking or fixing?
m_segment_process = 1; // we've done first time setup
}
}
if (m_state == LINK_V2_STATE_OFFSETS) { // pointer fixup
// this state reads through a table. Values alternate between "seek amount" and "number of
// consecutive 4-byte
// words to fix up". The counts are encoded using a variable length encoding scheme. They use
// a very stupid
// method of encoding values which requires O(n) bytes to store the value n.
// to avoid dropping a frame, we check every 0x400 relocations to see if 0.5 milliseconds have
// elapsed.
u32 relocCounter = 0x400;
while (true) { // loop over entire table
while (true) { // loop over current mode
// read and seek table
u8 count = *m_reloc_ptr;
m_reloc_ptr.offset++;
if (!m_table_toggle) { // seek mode
m_loc_ptr.offset +=
4 *
count; // perform seek (MIPS instructions are 4 bytes, so we >> 2 the seek amount)
} else { // offset mode
for (u32 i = 0; i < count; i++) {
if (m_loc_ptr.offset % 4) {
assert(false);
}
u32 code = *(m_loc_ptr.cast<u32>());
code += m_base_ptr.offset;
*(m_loc_ptr.cast<u32>()) = code;
m_loc_ptr.offset += 4;
}
}
if (count != 0xff) {
break;
}
if (*m_reloc_ptr == 0) {
m_reloc_ptr.offset++;
m_table_toggle = m_table_toggle ^ 1;
}
}
// reached the end of the tableToggle mode
m_table_toggle = m_table_toggle ^ 1;
if (*m_reloc_ptr == 0) {
break; // end of the state
}
relocCounter--;
if (relocCounter == 0) {
// u32 clock_value = kernel.read_clock();
// if(clock_value - startCycle > 150000) { // 0.5 milliseconds
// return 0;
// }
relocCounter = 0x400;
}
}
m_reloc_ptr.offset++;
m_state = 2;
m_segment_process = 0;
}
if (m_state == 2) { // GOAL object fixup
if (*m_reloc_ptr == 0) {
m_state = 3;
m_segment_process = 0;
} else {
while (true) {
u32 relocation = *m_reloc_ptr;
m_reloc_ptr.offset++;
Ptr<u8> goalObj;
char* name;
if ((relocation & 0x80) == 0) {
// symbol!
if (relocation > 9) {
m_reloc_ptr.offset--; // no idea what this is.
}
name = m_reloc_ptr.cast<char>().c();
if (link_debug_printfs) {
printf("[work_v2] symlink: %s\n", name);
}
goalObj = intern_from_c(name).cast<u8>();
} else {
// type!
u8 nMethods = relocation & 0x7f;
if (nMethods == 0) {
nMethods = 1;
}
name = m_reloc_ptr.cast<char>().c();
if (link_debug_printfs) {
printf("[work_v2] symlink -type: %s\n", name);
}
goalObj = intern_type_from_c(name, nMethods).cast<u8>();
}
m_reloc_ptr.offset += strlen(name) + 1;
// DECOMPILER->hookStartSymlinkV3(_state - 1, _objectData, std::string(name));
m_reloc_ptr = c_symlink2(m_object_data, goalObj, m_reloc_ptr);
// DECOMPILER->hookFinishSymlinkV3();
if (*m_reloc_ptr == 0) {
break; // done
}
// u32 currentCycle = kernel.read_clock();
// if(currentCycle - startCycle > 150000) {
// return 0;
// }
}
m_state = 3;
m_segment_process = 0;
}
}
m_entry = m_object_data + 4;
return 1;
}
/*!
* Complete linking. This will execute the top-level code for v3 object files, if requested.

11
game/kernel/klink.h
View file

@ -39,6 +39,14 @@ struct link_control {
uint32_t m_state;
uint32_t m_segment_process;
uint32_t m_version;
int m_heap_gap;
Ptr<uint8_t> m_original_object_location;
Ptr<u8> m_reloc_ptr;
Ptr<u8> m_base_ptr;
Ptr<u8> m_loc_ptr;
int m_table_toggle;
bool m_opengoal;
void begin(Ptr<uint8_t> object_file,
const char* name,
int32_t size,
@ -46,6 +54,7 @@ struct link_control {
uint32_t flags);
uint32_t work();
uint32_t work_v3();
uint32_t work_v2();
void finish();
void reset() {
@ -66,11 +75,13 @@ struct link_control {
}
};
// only used in OpenGOAL
struct SegmentInfo {
uint32_t offset;
uint32_t size;
};
// only used in OpenGOAL
struct ObjectFileHeader {
uint16_t goal_version_major;
uint16_t goal_version_minor;

87
game/sce/sif_ee.cpp
View file

@ -1,5 +1,8 @@
#include <cstring>
#include <cassert>
#include <cstdio>
#include <unordered_map>
#include "common/util/FileUtil.h"
#include "sif_ee.h"
#include "game/system/iop_thread.h"
#include "game/runtime.h"
@ -8,7 +11,8 @@ namespace ee {
namespace {
::IOP* iop;
}
std::unordered_map<s32, FILE*> sce_fds;
} // namespace
void LIBRARY_sceSif_register(::IOP* i) {
iop = i;
@ -16,6 +20,10 @@ void LIBRARY_sceSif_register(::IOP* i) {
void LIBRARY_INIT_sceSif() {
iop = nullptr;
for (auto& kv : sce_fds) {
fclose(kv.second);
}
sce_fds.clear();
}
void sceSifInitRpc(unsigned int mode) {
(void)mode;
@ -91,4 +99,81 @@ s32 sceSifBindRpc(sceSifClientData* bd, u32 request, u32 mode) {
bd->serve = (sceSifServeData*)1;
return 0;
}
s32 sceOpen(const char* filename, s32 flag) {
auto name = file_util::get_file_path({filename});
FILE* fp = nullptr;
switch (flag) {
case SCE_RDONLY:
fp = fopen(name.c_str(), "r");
break;
default:
assert(false);
}
if (!fp) {
printf("[SCE] sceOpen(%s) failed.\n", name.c_str());
return -1;
}
s32 fp_idx = sce_fds.size() + 1;
sce_fds[fp_idx] = fp;
return fp_idx;
}
s32 sceClose(s32 fd) {
if (fd < 0) {
// todo, what should we really return?
return 0;
}
auto kv = sce_fds.find(fd);
if (kv != sce_fds.end()) {
fclose(kv->second);
sce_fds.erase(fd);
return 0;
} else {
printf("[SCE] sceClose called on invalid fd\n");
return 0;
}
}
s32 sceRead(s32 fd, void* buf, s32 nbyte) {
auto kv = sce_fds.find(fd);
if (kv == sce_fds.end()) {
return -1;
} else {
return fread(buf, 1, nbyte, kv->second);
}
}
s32 sceWrite(s32 fd, const void* buf, s32 nbyte) {
(void)fd;
(void)buf;
(void)nbyte;
assert(false);
return 0;
}
s32 sceLseek(s32 fd, s32 offset, s32 where) {
auto kv = sce_fds.find(fd);
if (kv == sce_fds.end()) {
return -1;
} else {
switch (where) {
case SCE_SEEK_CUR:
fseek(kv->second, offset, SEEK_CUR);
return ftell(kv->second);
case SCE_SEEK_END:
fseek(kv->second, offset, SEEK_END);
return ftell(kv->second);
case SCE_SEEK_SET:
fseek(kv->second, offset, SEEK_SET);
return ftell(kv->second);
default:
assert(false);
}
}
}
} // namespace ee

27
game/sce/sif_ee.h
View file

@ -49,5 +49,32 @@ s32 sceSifCallRpc(sceSifClientData* bd,
s32 sceSifCheckStatRpc(sceSifRpcData* bd);
s32 sceSifBindRpc(sceSifClientData* bd, u32 request, u32 mode);
#ifndef SCE_SEEK_SET
#define SCE_SEEK_SET (0)
#endif
#ifndef SCE_SEEK_CUR
#define SCE_SEEK_CUR (1)
#endif
#ifndef SCE_SEEK_END
#define SCE_SEEK_END (2)
#endif
#define SCE_RDONLY 0x0001
#define SCE_WRONLY 0x0002
#define SCE_RDWR 0x0003
#define SCE_NBLOCK 0x0010
#define SCE_APPEND 0x0100
#define SCE_CREAT 0x0200
#define SCE_TRUNC 0x0400
#define SCE_EXCL 0x0800
#define SCE_NOBUF 0x4000
#define SCE_NOWAIT 0x8000
s32 sceOpen(const char* filename, s32 flag);
s32 sceClose(s32 fd);
s32 sceRead(s32 fd, void* buf, s32 nbyte);
s32 sceWrite(s32 fd, const void* buf, s32 nbyte);
s32 sceLseek(s32 fd, s32 offset, s32 where);
} // namespace ee
#endif // JAK1_SIF_EE_H

32
game/sce/stubs.cpp
View file

@ -3,38 +3,6 @@
#include "stubs.h"
namespace ee {
s32 sceOpen(const char* filename, s32 flag) {
(void)filename;
(void)flag;
throw std::runtime_error("sceOpen NYI");
}
s32 sceClose(s32 fd) {
(void)fd;
throw std::runtime_error("sceClose NYI");
}
s32 sceRead(s32 fd, void* buf, s32 nbyte) {
(void)fd;
(void)buf;
(void)nbyte;
throw std::runtime_error("sceRead NYI");
}
s32 sceWrite(s32 fd, const void* buf, s32 nbyte) {
(void)fd;
(void)buf;
(void)nbyte;
throw std::runtime_error("sceWrite NYI");
}
s32 sceLseek(s32 fd, s32 offset, s32 where) {
(void)fd;
(void)offset;
(void)where;
throw std::runtime_error("sceLseek NYI");
}
int scePadPortOpen(int port, int slot, void* data) {
(void)port;
(void)slot;

26
game/sce/stubs.h
View file

@ -5,35 +5,9 @@
#include "common/common_types.h"
#ifndef SCE_SEEK_SET
#define SCE_SEEK_SET (0)
#endif
#ifndef SCE_SEEK_CUR
#define SCE_SEEK_CUR (1)
#endif
#ifndef SCE_SEEK_END
#define SCE_SEEK_END (2)
#endif
#define SCE_RDONLY 0x0001
#define SCE_WRONLY 0x0002
#define SCE_RDWR 0x0003
#define SCE_NBLOCK 0x0010
#define SCE_APPEND 0x0100
#define SCE_CREAT 0x0200
#define SCE_TRUNC 0x0400
#define SCE_EXCL 0x0800
#define SCE_NOBUF 0x4000
#define SCE_NOWAIT 0x8000
#define SCE_PAD_DMA_BUFFER_SIZE 0x100
namespace ee {
s32 sceOpen(const char* filename, s32 flag);
s32 sceClose(s32 fd);
s32 sceRead(s32 fd, void* buf, s32 nbyte);
s32 sceWrite(s32 fd, const void* buf, s32 nbyte);
s32 sceLseek(s32 fd, s32 offset, s32 where);
void sceGsSyncV();
void sceGsSyncPath();
void sceGsResetPath();

10
goal_src/build/all_objs.txt
View file

@ -1084,4 +1084,12 @@
["vil3-bridge-36-ag", "vil3-bridge-36", 4, ["VI3"], "levels/village3"],
["water-anim-village3-ag", "water-anim-village3", 4, ["VI3"], "levels/village3"],
["village3-vis", "village3-vis", 4, ["VI3"], "levels/village3"],
["lava", "lava", 3, ["WATER-AN"], "old/lava"]]
["lava", "lava", 3, ["WATER-AN"], "old/lava"],
["0COMMON", "0COMMON", 4, ["NO-XGO"], ""],
["1COMMON", "1COMMON", 4, ["NO-XGO"], ""],
["2COMMON", "2COMMON", 4, ["NO-XGO"], ""],
["3COMMON", "3COMMON", 4, ["NO-XGO"], ""],
["4COMMON", "4COMMON", 4, ["NO-XGO"], ""],
["5COMMON", "5COMMON", 4, ["NO-XGO"], ""],
["6COMMON", "6COMMON", 4, ["NO-XGO"], ""]
]

36
goal_src/engine/ui/text-h.gc
View file

@ -5,3 +5,39 @@
;; name in dgo: text-h
;; dgos: GAME, ENGINE
;; This file contains types related to game text.
;; Each game string is assigned an ID number.
;; This ID is used to lookup the string for the currently selected language.
;; an individual string.
(deftype game-text (structure)
((id uint32 :offset-assert 0)
(text string :offset-assert 4)
)
:pack-me
:method-count-assert 9
:size-assert #x8
:flag-assert #x900000008
)
;; A table of all strings.
(deftype game-text-info (basic)
((length int32 :offset-assert 4)
(language-id int32 :offset-assert 8)
(group-name string :offset-assert 12)
(data game-text :dynamic :inline :offset-assert 16)
)
:method-count-assert 10
:size-assert #x10
:flag-assert #xa00000010
(:methods
(dummy-9 () none 9)
)
)
;; todo, need support for array
;(define *text-group-names* #("common"))
(define *common-text-heap* (new 'global 'kheap))
;; probably some other type.
(define *common-text* #f)

48
goal_src/engine/ui/text.gc
View file

@ -5,3 +5,51 @@
;; name in dgo: text
;; dgos: GAME, ENGINE
(define *game-text-word* (new 'global 'string 256 (the string '#f)))
(define *game-text-line* (new 'global 'string 256 (the string '#f)))
(define *level-text-file-load-flag* '#t)
;; allocate the game text heap if it isn't already allocated.
(when (= 0 (-> *common-text-heap* base))
(let ((heap *common-text-heap*))
(set! (-> heap base) (malloc 'global 34816))
(set! (-> heap current) (-> heap base))
(set! (-> heap top-base) (&+ (-> heap base) 34816))
(set! (-> heap top) (-> heap top-base))
)
)
(defmethod length game-text-info ((obj game-text-info))
"Get the length (number of strings) in a game-text-info."
(-> obj length)
)
(defmethod asize-of game-text-info ((obj game-text-info))
(the int (+ (-> obj type size) (* 8 (-> obj length))))
)
(defmethod inspect game-text-info ((obj game-text-info))
(format '#t "[~8x] ~A~%" obj (-> obj type))
(format '#t "~Tlength: ~D~%" (-> obj length))
(format '#t "~Tdata[~D]: @ #x~X~%" (-> obj length) (-> obj data))
(let ((i 0))
(while (< i (-> obj length))
(format '#t "~T [~D] #x~X ~A~%" i (-> obj data i id) (-> obj data i text))
(+! i 1)
)
)
obj
)
;; todo method 8
;; todo method 9
;; todo text-is-loading
;; todo load-game-text-info
;; todo load-level-text-files
;; todo draw-debug-text-box
;; todo set-font-color-alpha
;; todo print-game-text-scaled
;; todo print-game-text
;; todo disable-level-text-file-loading
;; todo enable-level-text-file-loading

6
goal_src/goal-lib.gc
View file

@ -33,6 +33,12 @@
)
)
(defmacro build-data ()
`(begin
(asm-data-file game-text "assets/game_text.txt")
)
)
(defmacro blg ()
`(begin
(build-game)

5
goalc/CMakeLists.txt
View file

@ -24,6 +24,8 @@ add_library(compiler
compiler/compilation/Type.cpp
compiler/compilation/Static.cpp
compiler/Util.cpp
data_compiler/game_text.cpp
data_compiler/DataObjectGenerator.cpp
debugger/Debugger.cpp
debugger/DebugInfo.cpp
logger/Logger.cpp
@ -36,12 +38,13 @@ add_library(compiler
compiler/Compiler.cpp)
add_executable(goalc main.cpp)
add_executable(data_compiler data_compiler.cpp)
IF (WIN32)
target_link_libraries(compiler goos type_system mman common_util spdlog cross_os_debug cross_sockets Zydis)
ELSE ()
target_link_libraries(compiler goos type_system common_util spdlog cross_os_debug cross_sockets Zydis)
ENDIF ()
target_link_libraries(goalc goos compiler type_system)
target_link_libraries(data_compiler goos compiler type_system)

5
goalc/compiler/Compiler.cpp
View file

@ -251,6 +251,11 @@ bool Compiler::connect_to_target() {
return true;
}
void Compiler::run_front_end_on_string(const std::string& src) {
auto code = m_goos.reader.read_from_string({src});
compile_object_file("run-on-string", code, true);
}
std::vector<std::string> Compiler::run_test_no_load(const std::string& source_code) {
auto code = m_goos.reader.read_from_file({source_code});
compile_object_file("test-code", code, true);

3
goalc/compiler/Compiler.h
View file

@ -34,6 +34,7 @@ class Compiler {
std::vector<std::string> run_test_from_string(const std::string& src,
const std::string& obj_name = "*listener*");
std::vector<std::string> run_test_no_load(const std::string& source_code);
void run_front_end_on_string(const std::string& src);
void shutdown_target();
void enable_throw_on_redefines() { m_throw_on_define_extern_redefinition = true; }
Debugger& get_debugger() { return m_debugger; }
@ -54,6 +55,7 @@ class Compiler {
Val* compile_pair(const goos::Object& code, Env* env);
Val* compile_integer(const goos::Object& code, Env* env);
Val* compile_integer(s64 value, Env* env);
Val* compile_char(const goos::Object& code, Env* env);
Val* compile_float(const goos::Object& code, Env* env);
Val* compile_float(float value, Env* env, int seg);
Val* compile_symbol(const goos::Object& form, Env* env);
@ -173,6 +175,7 @@ class Compiler {
Val* compile_seval(const goos::Object& form, const goos::Object& rest, Env* env);
Val* compile_exit(const goos::Object& form, const goos::Object& rest, Env* env);
Val* compile_asm_file(const goos::Object& form, const goos::Object& rest, Env* env);
Val* compile_asm_data_file(const goos::Object& form, const goos::Object& rest, Env* env);
Val* compile_listen_to_target(const goos::Object& form, const goos::Object& rest, Env* env);
Val* compile_reset_target(const goos::Object& form, const goos::Object& rest, Env* env);
Val* compile_poke(const goos::Object& form, const goos::Object& rest, Env* env);

8
goalc/compiler/compilation/Atoms.cpp
View file

@ -33,6 +33,7 @@ static const std::unordered_map<
{"gs", &Compiler::compile_gs},
{":exit", &Compiler::compile_exit},
{"asm-file", &Compiler::compile_asm_file},
{"asm-data-file", &Compiler::compile_asm_data_file},
{"listen-to-target", &Compiler::compile_listen_to_target},
{"reset-target", &Compiler::compile_reset_target},
{":status", &Compiler::compile_poke},
@ -139,6 +140,8 @@ Val* Compiler::compile(const goos::Object& code, Env* env) {
return compile_pair(code, env);
case goos::ObjectType::INTEGER:
return compile_integer(code, env);
case goos::ObjectType::CHAR:
return compile_char(code, env);
case goos::ObjectType::SYMBOL:
return compile_symbol(code, env);
case goos::ObjectType::STRING:
@ -194,6 +197,11 @@ Val* Compiler::compile_integer(const goos::Object& code, Env* env) {
return compile_integer(code.integer_obj.value, env);
}
Val* Compiler::compile_char(const goos::Object& code, Env* env) {
assert(code.is_char());
return compile_integer(uint8_t(code.char_obj.value), env);
}
/*!
* Compile an integer constant. Returns an IntegerConstantVal and emits no code.
* These integer constants do not generate static data and are stored directly in the code

19
goalc/compiler/compilation/CompilerControl.cpp
View file

@ -8,6 +8,7 @@
#include "common/util/Timer.h"
#include "common/util/DgoWriter.h"
#include "common/util/FileUtil.h"
#include "goalc/data_compiler/game_text.h"
/*!
* Exit the compiler. Disconnects the listener and tells the target to reset itself.
@ -46,6 +47,22 @@ Val* Compiler::compile_seval(const goos::Object& form, const goos::Object& rest,
return get_none();
}
/*!
* Compile a "data file"
*/
Val* Compiler::compile_asm_data_file(const goos::Object& form, const goos::Object& rest, Env* env) {
(void)env;
auto args = get_va(form, rest);
va_check(form, args, {goos::ObjectType::SYMBOL, goos::ObjectType::STRING}, {});
auto kind = symbol_string(args.unnamed.at(0));
if (kind == "game-text") {
compile_game_text(as_string(args.unnamed.at(1)));
} else {
throw_compile_error(form, "Unknown asm data file mode");
}
return get_none();
}
/*!
* Compile a file, and optionally color, save, or load.
* This should only be used for v3 "code object" files.
@ -143,7 +160,7 @@ Val* Compiler::compile_asm_file(const goos::Object& form, const goos::Object& re
printf("F: %36s ", obj_file_name.c_str());
timing.emplace_back("total", total_timer.getMs());
for (auto& e : timing) {
printf(" %12s %4.2f", e.first.c_str(), e.second / 1000.f);
printf(" %12s %4.0f", e.first.c_str(), e.second);
}
printf("\n");
}

8
goalc/compiler/compilation/Type.cpp
View file

@ -473,7 +473,13 @@ Val* Compiler::compile_deref(const goos::Object& form, const goos::Object& _rest
}
if (result->type().base_type() == "inline-array") {
assert(false);
auto di = m_ts.get_deref_info(result->type());
auto base_type = di.result_type;
assert(di.can_deref);
auto offset = compile_integer(di.stride, env)->to_gpr(env);
// todo, check for integer and avoid runtime multiply
env->emit(std::make_unique<IR_IntegerMath>(IntegerMathKind::IMUL_32, offset, index_value));
result = fe->alloc_val<MemoryOffsetVal>(di.result_type, result, offset);
} else if (result->type().base_type() == "pointer") {
auto di = m_ts.get_deref_info(result->type());
auto base_type = di.result_type;

46
goalc/data_compiler.cpp Normal file
View file

@ -0,0 +1,46 @@
#include <cstdio>
#include "goalc/compiler/Compiler.h"
#include "common/versions.h"
#include "third-party/spdlog/include/spdlog/spdlog.h"
#include "third-party/spdlog/include/spdlog/sinks/basic_file_sink.h"
#include "third-party/spdlog/include/spdlog/sinks/stdout_color_sinks.h"
void setup_logging(bool verbose) {
spdlog::set_level(spdlog::level::debug);
if (verbose) {
auto game_logger = spdlog::stdout_color_mt("GOAL Compiler: Data Mode");
spdlog::set_default_logger(game_logger);
spdlog::flush_on(spdlog::level::info);
spdlog::set_pattern("%v");
spdlog::info("Verbose logging enabled");
} else {
auto game_logger = spdlog::basic_logger_mt("GOAL Compiler", "logs/data_compiler.log");
spdlog::set_default_logger(game_logger);
spdlog::flush_on(spdlog::level::debug);
printf("OpenGOAL Compiler %d.%d: Data Mode\n", versions::GOAL_VERSION_MAJOR,
versions::GOAL_VERSION_MINOR);
}
}
int main(int argc, char** argv) {
(void)argc;
(void)argv;
bool verbose = false;
for (int i = 1; i < argc; i++) {
if (std::string("-v") == argv[i]) {
verbose = true;
break;
}
}
setup_logging(verbose);
spdlog::info("OpenGOAL Compiler {}.{}: Data Mode", versions::GOAL_VERSION_MAJOR,
versions::GOAL_VERSION_MINOR);
Compiler compiler;
compiler.run_front_end_on_string("(build-data)");
printf("Done!\n");
return 0;
}

222
goalc/data_compiler/DataObjectGenerator.cpp Normal file
View file

@ -0,0 +1,222 @@
#include <cstring>
#include <algorithm>
#include <cassert>
#include "DataObjectGenerator.h"
#include "common/link_types.h"
namespace {
template <typename T>
void add_data_to_vector(const T& data, std::vector<u8>* vec) {
auto loc = vec->size();
vec->resize(loc + sizeof(T));
memcpy(vec->data() + loc, &data, sizeof(T));
}
void push_variable_length_integer(u32 value, std::vector<u8>* vec) {
while (value > UINT8_MAX) {
vec->push_back(UINT8_MAX);
value -= UINT8_MAX;
}
if (value == UINT8_MAX) {
vec->push_back(UINT8_MAX);
vec->push_back(0);
} else {
vec->push_back(value);
}
}
void push_better_variable_length_integer(u32 value, std::vector<u8>* vec) {
if (value > 0xffffff) {
vec->push_back((value & 0xff) | 3);
vec->push_back((value >> 8) & 0xff);
vec->push_back((value >> 16) & 0xff);
vec->push_back((value >> 24) & 0xff);
} else if (value > 0xffff) {
vec->push_back((value & 0xff) | 2);
vec->push_back((value >> 8) & 0xff);
vec->push_back((value >> 16) & 0xff);
} else if (value > 0xff) {
vec->push_back((value & 0xff) | 1);
vec->push_back((value >> 8) & 0xff);
} else {
vec->push_back(value & 0xff);
}
}
} // namespace
int DataObjectGenerator::add_word(u32 word) {
auto result = int(m_words.size());
m_words.push_back(word);
return result;
}
void DataObjectGenerator::link_word_to_word(int source, int target, int offset) {
link_word_to_byte(source, target * 4 + offset);
}
void DataObjectGenerator::link_word_to_byte(int source_word, int target_byte) {
PointerLinkRecord rec;
rec.source_word = source_word;
rec.target_byte = target_byte;
m_ptr_links.push_back(rec);
}
int DataObjectGenerator::add_ref_to_string_in_pool(const std::string& str) {
auto result = int(m_words.size());
m_words.push_back(0);
m_string_pool[str].push_back(result);
return result;
}
int DataObjectGenerator::add_type_tag(const std::string& str) {
auto result = int(m_words.size());
m_words.push_back(0);
m_type_links[str].push_back(result);
return result;
}
int DataObjectGenerator::add_symbol_link(const std::string& str) {
auto result = int(m_words.size());
m_words.push_back(0);
m_symbol_links[str].push_back(result);
return result;
}
void DataObjectGenerator::align(int alignment_words) {
while (m_words.size() % alignment_words) {
m_words.push_back(0);
}
}
int DataObjectGenerator::words() const {
return int(m_words.size());
}
std::vector<u8> DataObjectGenerator::generate_v2() {
// add string data at the end.
add_strings();
// Generate the link table.
std::vector<u8> link = generate_link_table();
// add words
// header
LinkHeaderV2 header;
header.type_tag = 0xffffffff;
header.version = 2;
header.length = sizeof(LinkHeaderV2) + link.size();
// build
std::vector<u8> result;
add_data_to_vector(header, &result);
result.insert(result.end(), link.begin(), link.end());
auto start = result.size();
result.resize(result.size() + m_words.size() * 4);
memcpy(result.data() + start, m_words.data(), m_words.size() * 4);
while (result.size() % 16) {
result.push_back(0);
}
return result;
}
std::vector<u8> DataObjectGenerator::generate_link_table() {
std::vector<u8> link;
// pointer links are in source order.
std::sort(m_ptr_links.begin(), m_ptr_links.end(),
[](const PointerLinkRecord& a, const PointerLinkRecord& b) {
return a.source_word < b.source_word;
});
int i = 0;
u32 last_word = 0;
while (i < int(m_ptr_links.size())) {
// seeking
auto& entry = m_ptr_links.at(i);
int diff = int(entry.source_word) - int(last_word);
last_word = entry.source_word + 1;
assert(diff >= 0);
push_variable_length_integer(diff, &link);
m_words.at(entry.source_word) = entry.target_byte;
// count.
int consecutive = 1;
for (;;) {
if (i + 1 < int(m_ptr_links.size()) &&
m_ptr_links.at(i + 1).source_word == m_ptr_links.at(i).source_word + 1) {
m_words.at(m_ptr_links.at(i + 1).source_word) = m_ptr_links.at(i + 1).target_byte;
last_word = m_ptr_links.at(i + 1).source_word + 1;
consecutive++;
i++;
} else {
break;
}
}
push_variable_length_integer(consecutive, &link);
i++;
}
push_variable_length_integer(0, &link);
// todo symbols
assert(m_symbol_links.empty());
// types
for (auto& tl : m_type_links) {
link.push_back(0x80);
for (auto c : tl.first) {
link.push_back(c);
}
link.push_back(0);
std::sort(tl.second.begin(), tl.second.end());
int prev = 0;
for (auto& x : tl.second) {
int diff = x - prev;
assert(diff >= 0);
push_better_variable_length_integer(diff * 4, &link);
m_words.at(x) = 0xffffffff;
prev = x;
}
link.push_back(0);
}
push_variable_length_integer(0, &link);
// align to 16 bytes for data start!
while ((link.size() + sizeof(LinkHeaderV2)) % 64) {
link.push_back(0);
}
return link;
}
void DataObjectGenerator::add_strings() {
for (auto& entry : m_string_pool) {
// add the string
align(4);
add_type_tag("string");
auto target_word = add_word(entry.first.length());
std::vector<u8> string_buff;
for (auto c : entry.first) {
string_buff.push_back(c);
}
string_buff.push_back(0);
while (string_buff.size() & 3) {
string_buff.push_back(0);
}
for (int i = 0; i < int(string_buff.size()) / 4; i++) {
add_word(*(u32*)(string_buff.data() + i * 4));
}
for (auto& source : entry.second) {
link_word_to_word(source, target_word);
}
}
}

36
goalc/data_compiler/DataObjectGenerator.h Normal file
View file

@ -0,0 +1,36 @@
#pragma once
#include "common/common_types.h"
#include <string>
#include <map>
#include <vector>
class DataObjectGenerator {
public:
int add_word(u32 word);
void link_word_to_word(int source, int target, int offset = 0);
void link_word_to_byte(int source_word, int target_byte);
int add_ref_to_string_in_pool(const std::string& str);
int add_type_tag(const std::string& str);
int add_symbol_link(const std::string& str);
std::vector<u8> generate_v2();
void align(int alignment_words);
int words() const;
private:
void add_strings();
std::vector<u8> generate_link_table();
struct PointerLinkRecord {
int source_word;
int target_byte;
};
std::map<std::string, std::vector<int>> m_string_pool;
std::vector<u32> m_words;
std::vector<PointerLinkRecord> m_ptr_links;
// both alphabetical.
// symbols before types.
std::map<std::string, std::vector<int>> m_type_links, m_symbol_links;
};

220
goalc/data_compiler/game_text.cpp Normal file
View file

@ -0,0 +1,220 @@
/*!
* @file game_text.cpp
* Builds the XCOMMON.TXT text files. Each file contains all the strings that appear in the game
* translated into a language.
*
* The decompiler/data/game_text.cpp file extracts text from the game and creates a file that
* can be read with these functions.
*/
#include <algorithm>
#include "game_text.h"
#include "common/goos/Reader.h"
#include "DataObjectGenerator.h"
#include "common/util/FileUtil.h"
#include "third-party/fmt/core.h"
namespace {
template <typename T>
void for_each_in_list(const goos::Object& list, const T& f) {
const goos::Object* iter = &list;
while (iter->is_pair()) {
auto lap = iter->as_pair();
f(lap->car);
iter = &lap->cdr;
}
if (!iter->is_empty_list()) {
throw std::runtime_error("Invalid list");
}
}
int64_t get_int(const goos::Object& obj) {
if (obj.is_int()) {
return obj.integer_obj.value;
}
throw std::runtime_error(obj.print() + " was supposed to be an integer, but isn't");
}
const goos::Object& car(const goos::Object& x) {
if (!x.is_pair()) {
throw std::runtime_error("invalid pair");
}
return x.as_pair()->car;
}
const goos::Object& cdr(const goos::Object& x) {
if (!x.is_pair()) {
throw std::runtime_error("invalid pair");
}
return x.as_pair()->cdr;
}
std::string get_string(const goos::Object& x) {
if (x.is_string()) {
return x.as_string()->data;
}
throw std::runtime_error(x.print() + " was supposed to be an string, but isn't");
}
std::string uppercase(const std::string& in) {
std::string result;
result.reserve(in.size());
for (auto c : in) {
if (c >= 'a' && c <= 'z') {
c -= ('a' - 'A');
}
result.push_back(c);
}
return result;
}
/*!
* Parse a game text file for all languages.
* The result is a vector<map<text_id, string>>
* so result[lang_id][text_id] gets you the text in the given language.
*
* The file should begin with (language-count x) with the given number of languages.
* Each entry should be (text-id "text-in-lang-0" "text-in-lang-1" ... )
* The text id's can be out of order or missing entries.
*/
std::vector<std::unordered_map<int, std::string>> parse(const goos::Object& data,
std::string* group_name) {
std::vector<std::unordered_map<int, std::string>> text;
bool languages_set = false;
bool group_name_set = false;
std::string possible_group_name;
for_each_in_list(data.as_pair()->cdr, [&](const goos::Object& obj) {
if (obj.is_pair()) {
auto head = obj.as_pair()->car;
if (head.is_symbol() && head.as_symbol()->name == "language-count") {
if (languages_set) {
throw std::runtime_error("Languages has been set multiple times.");
}
text.resize(get_int(car(cdr(obj))));
if (!cdr(cdr(obj)).is_empty_list()) {
throw std::runtime_error("language-count has too many arguments");
}
} else if (head.is_symbol() && head.as_symbol()->name == "group-name") {
if (group_name_set) {
throw std::runtime_error("group-name has been set multiple times.");
}
group_name_set = true;
possible_group_name = get_string(car(cdr(obj)));
if (!cdr(cdr(obj)).is_empty_list()) {
throw std::runtime_error("group-name has too many arguments");
}
}
else if (head.is_int()) {
int i = 0;
int id = head.as_int();
for_each_in_list(cdr(obj), [&](const goos::Object& entry) {
if (i >= int(text.size())) {
throw std::runtime_error(
"String has too many entries. There should be one per language");
}
if (entry.is_string()) {
auto& map = text.at(i);
if (map.find(id) != map.end()) {
throw std::runtime_error("Entry appears more than once");
}
map[id] = entry.as_string()->data;
} else {
throw std::runtime_error("Each entry must be a string");
}
i++;
});
if (i != int(text.size())) {
throw std::runtime_error("String did not have an entry for each language");
}
} else {
throw std::runtime_error("Invalid game text file entry: " + head.print());
}
} else {
throw std::runtime_error("Invalid game text file");
}
});
if (!group_name_set) {
throw std::runtime_error("group-name not set.");
}
*group_name = possible_group_name;
return text;
}
/*
(deftype game-text (structure)
((id uint32 :offset-assert 0)
(text basic :offset-assert 4)
)
)
(deftype game-text-info (basic)
((length int32 :offset-assert 4)
(language-id int32 :offset-assert 8)
(group-name basic :offset-assert 12)
(data game-text :dynamic :offset-assert 16)
)
)
*/
/*!
* Write game text data to a file. Uses the V2 object format which is identical between GOAL and
* OpenGOAL, so this should produce exactly identical files to what is found in the game.
*/
void compile(const std::vector<std::unordered_map<int, std::string>>& text,
const std::string& group_name) {
// get all text ID's we know
std::vector<int> add_order;
add_order.reserve(text.front().size());
for (auto& x : text.front()) {
add_order.push_back(x.first);
}
// and sort them to be added in order. This matches the game.
std::sort(add_order.begin(), add_order.end());
for (int lang = 0; lang < int(text.size()); lang++) {
DataObjectGenerator gen;
gen.add_type_tag("game-text-info"); // type
gen.add_word(text.front().size()); // length
gen.add_word(lang); // language-id
// this string is found in the string pool.
gen.add_ref_to_string_in_pool(group_name); // group-name
// now add all the datas:
for (auto id : add_order) {
gen.add_word(id); // id
// these strings must be in the string pool, as sometimes there are duplicate
// strings in a single language, and these strings should be stored once and have multiple
// references to them.
gen.add_ref_to_string_in_pool(text.at(lang).at(id)); // text
}
auto data = gen.generate_v2();
file_util::write_binary_file(
file_util::get_file_path({"out", fmt::format("{}{}.TXT", lang, uppercase(group_name))}),
data.data(), data.size());
}
}
} // namespace
/*!
* Read a game text description file and generate GOAL objects.
*/
void compile_game_text(const std::string& filename) {
goos::Reader reader;
auto code = reader.read_from_file({filename});
printf("[Build Game Text] %s\n", filename.c_str());
std::string group_name;
auto text_map = parse(code, &group_name);
compile(text_map, group_name);
}

4
goalc/data_compiler/game_text.h Normal file
View file

@ -0,0 +1,4 @@
#pragma once
#include <string>
void compile_game_text(const std::string& filename);

2
goalc/emitter/ObjectGenerator.h
View file

@ -15,7 +15,7 @@
#include "Instruction.h"
#include "goalc/debugger/DebugInfo.h"
class FunctionDebugInfo;
struct FunctionDebugInfo;
namespace emitter {

51
offline_test_git_branch.sh Executable file
View file

@ -0,0 +1,51 @@
#!/bin/bash
set -e
echo "======================================="
echo "= Jak Project Offline Test ="
echo "======================================="
echo ""
echo " ================= Cloning..."
ISO_DATA_PATH=${1}
BRANCH_NAME=${2:-master}
if [ -z "$1" ]
then
echo "Must supply path to iso data folder!"
exit 1
fi
echo " Branch: ${BRANCH_NAME}"
mkdir project
cd project
git clone https://github.com/water111/jak-project.git
cd jak-project
git checkout $BRANCH_NAME
git submodule update --init --recursive
# create symlink to the iso_data folder.
rm -r iso_data
ln -s $ISO_DATA_PATH
mkdir build
cd build
echo " =============== Building..."
cmake ..
make -j
echo " ================ Running unit tests..."
../test.sh
echo " ================ Decompiling..."
../decomp.sh
echo " ================ Building assets..."
./goalc/data_compiler
echo " ================ Checking assets..."
../check.sh
echo "Offline test has completed successfully!"

1
out/.gitignore vendored
View file

@ -1,2 +1,3 @@
*
!.gitignore
!hash.md5

7
out/hash.md5 Normal file
View file

@ -0,0 +1,7 @@
b06369c2dd9197cb17aae6286246caf9 0COMMON.TXT
da0d6012181f13803e8b3267a4099b97 1COMMON.TXT
4a9a27beb4ce7e75fa4c70811d2c0013 2COMMON.TXT
abcc25e5d7469dd6a572dc53dbb9671c 3COMMON.TXT
82eabdb7159f2059fbdbd18bb6fc06aa 4COMMON.TXT
5d62de2c78b4cf102b9a78f3aa96c8c9 5COMMON.TXT
9495f80955e6782513fe12f6539fc8e7 6COMMON.TXT

13
test/goalc/source_templates/with_game/test-game-text.gc Normal file
View file

@ -0,0 +1,13 @@
(start-test "game-text")
(let ((text (the game-text-info (load "$JAK-PROJECT/out/0TEST.TXT" *common-text-heap*))))
(format 0 "~I~%" text)
(expect-true (= #x123 (-> text data 0 id)))
(expect-true (= #x456 (-> text data 1 id)))
(expect-true (= #\e (-> text data 1 text data 1)))
(expect-true (= 5 (-> text data 1 text allocated-length)))
(expect-true (= 5 (length (-> text data 1 text))))
)
(finish-test)

151
test/goalc/test_with_game.cpp
View file

@ -27,18 +27,16 @@ struct WithGameParam {
class WithGameTests : public testing::TestWithParam<WithGameParam> {
public:
static void SetUpTestSuite() {
compiler.run_test_no_load("test/goalc/source_templates/with_game/test-build-game.gc");
runtime_thread = std::thread((GoalTest::runtime_with_kernel));
runner.c = &compiler;
compiler.run_test_from_file("test/goalc/source_templates/with_game/test-load-game.gc");
try {
compiler.run_test_from_file("test/goalc/source_templates/with_game/test-build-game.gc");
compiler.run_test_no_load("test/goalc/source_templates/with_game/test-build-game.gc");
} catch (std::exception& e) {
fprintf(stderr, "caught exception %s\n", e.what());
EXPECT_TRUE(false);
}
runtime_thread = std::thread((GoalTest::runtime_with_kernel));
runner.c = &compiler;
compiler.run_test_from_file("test/goalc/source_templates/with_game/test-load-game.gc");
}
static void TearDownTestSuite() {
@ -72,72 +70,199 @@ std::vector<std::string> get_test_pass_string(const std::string& name, int count
}
} // namespace
// TODO - havn't done anything with these really yet
TEST_F(WithGameTests, All) {
TEST_F(WithGameTests, ReturnConstant) {
runner.run_static_test(env, testCategory, "defun-return-constant.static.gc", {"12\n"});
}
TEST_F(WithGameTests, ReturnSymbol) {
runner.run_static_test(env, testCategory, "defun-return-symbol.static.gc", {"42\n"});
}
TEST_F(WithGameTests, MinMax) {
runner.run_static_test(env, testCategory, "test-min-max.gc", {"10\n"});
}
TEST_F(WithGameTests, BoxedFloat) {
runner.run_static_test(env, testCategory, "test-bfloat.gc",
{"data 1.2330 print 1.2330 type bfloat\n0\n"});
}
TEST_F(WithGameTests, BasicTypeCheck) {
runner.run_static_test(env, testCategory, "test-basic-type-check.gc", {"#f#t#t#f#t#f#t#t\n0\n"});
}
TEST_F(WithGameTests, ConditionBoolean) {
runner.run_static_test(env, testCategory, "test-condition-boolean.gc", {"4\n"});
}
TEST_F(WithGameTests, TypeType) {
runner.run_static_test(env, testCategory, "test-type-type.gc", {"#t#f\n0\n"});
}
TEST_F(WithGameTests, AccessInlineArray) {
runner.run_static_test(env, testCategory, "test-access-inline-array.gc", {"1.2345\n0\n"});
}
TEST_F(WithGameTests, FindParentMethod) {
runner.run_static_test(env, testCategory, "test-find-parent-method.gc", {"\"test pass!\"\n0\n"});
}
TEST_F(WithGameTests, Ref) {
runner.run_static_test(env, testCategory, "test-ref.gc", {"83\n"});
}
TEST_F(WithGameTests, PairASzie) {
runner.run_static_test(env, testCategory, "test-pair-asize.gc", {"8\n"});
}
TEST_F(WithGameTests, Last) {
runner.run_static_test(env, testCategory, "test-last.gc", {"d\n0\n"});
}
TEST_F(WithGameTests, Sort) {
runner.run_static_test(
env, testCategory, "test-sort.gc",
{"(24 16 32 56 72 1234 -34 25 654)\n(1234 654 72 56 32 25 24 16 -34)\n0\n"});
}
TEST_F(WithGameTests, Sort2) {
runner.run_static_test(
env, testCategory, "test-sort-2.gc",
{"(24 16 32 56 72 1234 -34 25 654)\n(-34 16 24 25 32 56 72 654 1234)\n0\n"});
}
TEST_F(WithGameTests, Sort3) {
runner.run_static_test(
env, testCategory, "test-sort-3.gc",
{"(24 16 32 56 72 1234 -34 25 654)\n(-34 16 24 25 32 56 72 654 1234)\n0\n"});
}
TEST_F(WithGameTests, PairLength) {
runner.run_static_test(env, testCategory, "test-pair-length.gc", {"6\n"});
}
TEST_F(WithGameTests, Member1) {
runner.run_static_test(env, testCategory, "test-member-1.gc", {"(c d)\n0\n"});
}
TEST_F(WithGameTests, Member2) {
runner.run_static_test(env, testCategory, "test-member-2.gc", {"#f\n0\n"});
}
TEST_F(WithGameTests, Assoc1) {
runner.run_static_test(env, testCategory, "test-assoc-1.gc", {"w\n0\n"});
}
TEST_F(WithGameTests, Assoc2) {
runner.run_static_test(env, testCategory, "test-assoc-2.gc", {"#f\n0\n"});
}
TEST_F(WithGameTests, Assoce1) {
runner.run_static_test(env, testCategory, "test-assoce-1.gc", {"x\n0\n"});
}
TEST_F(WithGameTests, Assoce2) {
runner.run_static_test(env, testCategory, "test-assoce-2.gc", {"x\n0\n"});
}
TEST_F(WithGameTests, Append) {
runner.run_static_test(env, testCategory, "test-append.gc", {"(a b c d e)\n0\n"});
}
TEST_F(WithGameTests, DeleteList) {
runner.run_static_test(env, testCategory, "test-delete-list.gc", {"(a b d e)\n0\n"});
}
TEST_F(WithGameTests, DeleteCar) {
runner.run_static_test(env, testCategory, "test-delete-car.gc", {"((a . b) (e . f))\n#f\n0\n"});
}
TEST_F(WithGameTests, InsertCar) {
runner.run_static_test(env, testCategory, "test-insert-cons.gc",
{"((c . w) (a . b) (e . f))\n0\n"});
}
TEST_F(WithGameTests, InlineArrayClass) {
runner.run_static_test(env, testCategory, "test-new-inline-array-class.gc", {"2824\n"});
}
TEST_F(WithGameTests, Memcpy) {
runner.run_static_test(env, testCategory, "test-memcpy.gc", {"13\n"});
}
TEST_F(WithGameTests, Memset) {
runner.run_static_test(env, testCategory, "test-memset.gc", {"11\n"});
}
TEST_F(WithGameTests, BintegerPrint) {
runner.run_static_test(env, testCategory, "test-binteger-print.gc", {"-17\n0\n"});
}
TEST_F(WithGameTests, TestTests) {
runner.run_static_test(env, testCategory, "test-tests.gc",
{"Test Failed On Test 0: \"unknown\"\nTest Failed On Test 0: "
"\"test\"\nTest \"test-of-test\": 1 Passes\n0\n"});
}
TEST_F(WithGameTests, TypeArrays) {
runner.run_static_test(env, testCategory, "test-type-arrays.gc",
{"Test \"test-type-arrays\": 3 Passes\n0\n"});
}
TEST_F(WithGameTests, NumberComparison) {
runner.run_static_test(env, testCategory, "test-number-comparison.gc",
{"Test \"number-comparison\": 14 Passes\n0\n"});
}
TEST_F(WithGameTests, ApproxPi) {
runner.run_static_test(env, testCategory, "test-approx-pi.gc",
get_test_pass_string("approx-pi", 4));
}
TEST_F(WithGameTests, DynamicType) {
runner.run_static_test(env, testCategory, "test-dynamic-type.gc",
get_test_pass_string("dynamic-type", 4));
}
TEST_F(WithGameTests, StringType) {
runner.run_static_test(env, testCategory, "test-string-type.gc",
get_test_pass_string("string-type", 4));
}
TEST_F(WithGameTests, NewString) {
runner.run_static_test(env, testCategory, "test-new-string.gc",
get_test_pass_string("new-string", 5));
}
TEST_F(WithGameTests, AddrOf) {
runner.run_static_test(env, testCategory, "test-addr-of.gc", get_test_pass_string("addr-of", 2));
}
TEST_F(WithGameTests, SetSelf) {
runner.run_static_test(env, testCategory, "test-set-self.gc", {"#t\n0\n"});
}
TEST_F(WithGameTests, NewArray) {
runner.run_static_test(env, testCategory, "test-new-array.gc",
get_test_pass_string("new-array", 8));
}
TEST_F(WithGameTests, NewStaticStructureIntegers) {
runner.run_static_test(env, testCategory, "test-new-static-structure-integers.gc",
get_test_pass_string("new-static-structure-integers", 7));
}
TEST_F(WithGameTests, NewStaticBasic) {
runner.run_static_test(env, testCategory, "test-new-static-basic.gc",
get_test_pass_string("new-static-basic", 9));
}
TEST_F(WithGameTests, VectorDot) {
runner.run_static_test(env, testCategory, "test-vector-dot.gc",
get_test_pass_string("vector-dot", 1));
}
TEST_F(WithGameTests, DebuggerMemoryMap) {
auto mem_map = compiler.listener().build_memory_map();
// we should have gkernel main segment
@ -147,7 +272,9 @@ TEST_F(WithGameTests, All) {
EXPECT_TRUE(lookup_2.obj_name == "gkernel");
EXPECT_FALSE(lookup_2.empty);
EXPECT_EQ(lookup_2.seg_id, MAIN_SEGMENT);
}
TEST_F(WithGameTests, DebuggerDisassemble) {
auto di = compiler.get_debugger().get_debug_info_for_object("gcommon");
bool fail = false;
auto result = di.disassemble_debug_functions(&fail);
@ -155,6 +282,12 @@ TEST_F(WithGameTests, All) {
EXPECT_FALSE(fail);
}
TEST_F(WithGameTests, GameText) {
compiler.run_test_from_string("(asm-data-file game-text \"test/test_data/test_game_text.txt\")");
runner.run_static_test(env, testCategory, "test-game-text.gc",
get_test_pass_string("game-text", 5));
}
TEST(TypeConsistency, TypeConsistency) {
Compiler compiler;
compiler.enable_throw_on_redefines();

10
test/test_data/test_game_text.txt Normal file
View file

@ -0,0 +1,10 @@
(language-count 3)
(group-name "test")
(#x123 "language 0"
"language 1"
"language 2")
(#x456 "hello"
"goodbye"
"aaaaaaaa")

0
test/test_goos_file0.gs → test/test_data/test_goos_file0.gs
View file

0
test/test_reader_file0.gc → test/test_data/test_reader_file0.gc
View file

4
test/test_goos.cpp
View file

@ -48,7 +48,7 @@ TEST(GoosBuiltins, Read) {
TEST(GoosBuiltins, ReadFile) {
Interpreter i;
// check that we can read a file.
EXPECT_EQ(e(i, "(read-file \"test/test_reader_file0.gc\")"), "(top-level (1 2 3 4))");
EXPECT_EQ(e(i, "(read-file \"test/test_data/test_reader_file0.gc\")"), "(top-level (1 2 3 4))");
i.disable_printfs();
for (auto x : {"(read-file 1)", "(read-file)", "(read-file \"goal/test/not_a_real_file.gc\")"}) {
@ -59,7 +59,7 @@ TEST(GoosBuiltins, ReadFile) {
TEST(GoosBuiltins, LoadFile) {
Interpreter i;
// check that we can read and execute a file.
e(i, "(load-file \"test/test_goos_file0.gs\")");
e(i, "(load-file \"test/test_data/test_goos_file0.gs\")");
EXPECT_EQ(e(i, "x"), "23");
i.disable_printfs();
for (auto x : {"(load-file 1)", "(load-file)", "(load-file \"goal/test/not_a_real_file.gc\")"}) {

11
test/test_reader.cpp
View file

@ -338,16 +338,19 @@ TEST(GoosReader, TopLevel) {
TEST(GoosReader, FromFile) {
Reader reader;
auto result = reader.read_from_file({"test", "test_reader_file0.gc"}).print();
auto result = reader.read_from_file({"test", "test_data", "test_reader_file0.gc"}).print();
EXPECT_TRUE(result == "(top-level (1 2 3 4))");
}
TEST(GoosReader, TextDb) {
// very specific to this particular test file, but whatever.
Reader reader;
auto result =
reader.read_from_file({"test", "test_reader_file0.gc"}).as_pair()->cdr.as_pair()->car;
std::string expected = "text from " + file_util::get_file_path({"test", "test_reader_file0.gc"}) +
auto result = reader.read_from_file({"test", "test_data", "test_reader_file0.gc"})
.as_pair()
->cdr.as_pair()
->car;
std::string expected = "text from " +
file_util::get_file_path({"test", "test_data", "test_reader_file0.gc"}) +
", line: 5\n(1 2 3 4)\n";
EXPECT_EQ(expected, reader.db.get_info_for(result));
}