Archive/jak-project
1
0
Fork 0
mirror of https://github.com/open-goal/jak-project.git synced 2024-10-20 00:57:44 -04:00
Code Issues Actions 3 Packages Projects Releases Wiki Activity

offline-test: Partition by DGO and colorize/condense output (#2045)

Browse source 
This solves two main problems:
- the looming threat of running out of memory since every thread would
consume duplicate (and probably not needed) resources
- though I will point out, jak 2's offline tests seem to hardly use any
memory even with 400+ files, duplicated across many threads. Where as
jak 1 does indeed use tons more memory. So I think there is something
going on besides just the source files
- condense the output so it's much easier to see what is happening / how
close the test is to completing.
- one annoying thing about the multiple thread change was errors were
typically buried far in the middle of the output, this fixes that
- refactors the offline test code in general to be a lot more modular

The pretty printing is not enabled by default, run with `-p` or
`--pretty-print` if you want to use it


https://user-images.githubusercontent.com/13153231/205513212-a65c20d4-ce36-44f6-826a-cd475505dbf9.mp4
This commit is contained in:
Tyler Wilding 2022-12-22 13:41:33 -05:00 committed by GitHub
parent 74d1074eef
commit 9c631e11fe
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
39 changed files with 8619 additions and 7375 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
Taskfile.yml
View file

@ -125,7 +125,7 @@ tasks:
- '{{.OFFLINETEST_BIN_RELEASE_DIR}}/offline-test --iso_data_path "./iso_data/{{.GAME}}" --game {{.GAME}}'
offline-tests-fast:
cmds:
- '{{.OFFLINETEST_BIN_RELEASE_DIR}}/offline-test --iso_data_path "./iso_data/{{.GAME}}" --game {{.GAME}} --num_threads 32 --dump_current_output --fail-on-cmp'
- '{{.OFFLINETEST_BIN_RELEASE_DIR}}/offline-test --iso_data_path "./iso_data/{{.GAME}}" --game {{.GAME}} -p --num_threads 32 --dump_current_output --fail-on-cmp'
offline-test-file:
cmds:
- '{{.OFFLINETEST_BIN_RELEASE_DIR}}/offline-test --iso_data_path "./iso_data/{{.GAME}}" --game {{.GAME}} --file {{.FILE}}'

9
common/dma/gs.cpp
View file

@ -3,6 +3,7 @@
#include "common/util/Assert.h"
#include "third-party/fmt/core.h"
#include "third-party/fmt/format.h"
std::string reg_descriptor_name(GifTag::RegisterDescriptor reg) {
switch (reg) {
@ -352,15 +353,17 @@ std::string GsTexa::print() const {
std::string GsTex0::print() const {
return fmt::format(
"tbp0: {} tbw: {} psm: {} tw: {} th: {} tcc: {} tfx: {} cbp: {} cpsm: {} csm: {}\n", tbp0(),
tbw(), psm(), tw(), th(), tcc(), tfx(), cbp(), cpsm(), csm());
tbw(), fmt::underlying(psm()), tw(), th(), tcc(), fmt::underlying(tfx()), cbp(), cpsm(),
csm());
}
std::string GsPrim::print() const {
return fmt::format("0x{:x}, kind {}\n", data, kind());
return fmt::format("0x{:x}, kind {}\n", data, fmt::underlying(kind()));
}
std::string GsFrame::print() const {
return fmt::format("fbp: {} fbw: {} psm: {} fbmsk: {:x}\n", fbp(), fbw(), psm(), fbmsk());
return fmt::format("fbp: {} fbw: {} psm: {} fbmsk: {:x}\n", fbp(), fbw(), fmt::underlying(psm()),
fbmsk());
}
std::string GsXYOffset::print() const {

3
common/goos/Interpreter.cpp
View file

@ -11,6 +11,7 @@
#include "common/util/FileUtil.h"
#include "common/util/unicode_util.h"
#include <common/log/log.h>
#include "third-party/fmt/core.h"
@ -1590,7 +1591,7 @@ Object Interpreter::eval_format(const Object& form,
fmt::format_args(args2.data(), static_cast<unsigned>(args2.size())));
if (truthy(dest)) {
printf("%s", formatted.c_str());
lg::print(formatted.c_str());
}
return StringObject::make_new(formatted);

3
common/goos/PrettyPrinter2.cpp
View file

@ -4,6 +4,7 @@
#include "common/util/Assert.h"
#include "third-party/fmt/core.h"
#include "third-party/fmt/format.h"
namespace pretty_print {
@ -100,7 +101,7 @@ inline const std::string quote_symbol(Node::QuoteKind kind) {
case Node::QuoteKind::UNQUOTE_SPLICING:
return ",@";
default:
ASSERT_MSG(false, fmt::format("invalid quote kind {}", kind));
ASSERT_MSG(false, fmt::format("invalid quote kind {}", fmt::underlying(kind)));
return "[invalid]";
}
}

1
common/type_system/TypeSystem.cpp
View file

@ -7,6 +7,7 @@
#include "TypeSystem.h"
#include <algorithm>
#include <stdexcept>
#include "common/log/log.h"

3
common/util/FontUtils.cpp
View file

@ -15,6 +15,7 @@
#include "common/util/Assert.h"
#include "third-party/fmt/core.h"
#include "third-party/fmt/format.h"
namespace {
@ -38,7 +39,7 @@ const std::string& get_text_version_name(GameTextVersion version) {
return name;
}
}
throw std::runtime_error(fmt::format("invalid text version {}", version));
throw std::runtime_error(fmt::format("invalid text version {}", fmt::underlying(version)));
}
GameTextFontBank::GameTextFontBank(GameTextVersion version,

10
common/util/StringUtil.cpp
View file

@ -21,4 +21,14 @@ std::string rtrim(const std::string& s) {
std::string trim(const std::string& s) {
return rtrim(ltrim(s));
}
int line_count(const std::string& str) {
int result = 0;
for (auto& c : str) {
if (c == '\n') {
result++;
}
}
return result;
}
} // namespace str_util

1
common/util/StringUtil.h
View file

@ -5,4 +5,5 @@ bool starts_with(const std::string& s, const std::string& prefix);
std::string ltrim(const std::string& s);
std::string rtrim(const std::string& s);
std::string trim(const std::string& s);
int line_count(const std::string& str);
} // namespace str_util

3
common/versions.cpp
View file

@ -3,6 +3,7 @@
#include "common/util/Assert.h"
#include "third-party/fmt/core.h"
#include "third-party/fmt/format.h"
GameVersion game_name_to_version(const std::string& name) {
if (name == "jak1") {
@ -25,6 +26,6 @@ std::string version_to_game_name(GameVersion v) {
case GameVersion::Jak2:
return "jak2";
default:
ASSERT_MSG(false, fmt::format("no game_name for version: {} found", v));
ASSERT_MSG(false, fmt::format("no game_name for version: {} found", fmt::underlying(v)));
}
}

2
decompiler/ObjectFile/ObjectFileDB.h
View file

@ -185,6 +185,8 @@ class ObjectFileDB {
void analyze_functions_ir2(
const fs::path& output_dir,
const Config& config,
const std::optional<std::function<void(std::string)>> prefile_callback,
const std::optional<std::function<void()>> postfile_callback,
const std::unordered_set<std::string>& skip_functions,
const std::unordered_map<std::string, std::unordered_set<std::string>>& skip_states = {});
void ir2_top_level_pass(const Config& config);

8
decompiler/ObjectFile/ObjectFileDB_IR2.cpp
View file

@ -125,6 +125,8 @@ void ObjectFileDB::process_object_file_data(
void ObjectFileDB::analyze_functions_ir2(
const fs::path& output_dir,
const Config& config,
const std::optional<std::function<void(std::string)>> prefile_callback,
const std::optional<std::function<void()>> postfile_callback,
const std::unordered_set<std::string>& skip_functions,
const std::unordered_map<std::string, std::unordered_set<std::string>>& skip_states) {
int total_file_count = 0;
@ -133,8 +135,14 @@ void ObjectFileDB::analyze_functions_ir2(
}
int file_idx = 1;
for_each_obj([&](ObjectFileData& data) {
if (prefile_callback) {
prefile_callback.value()(data.to_unique_name());
}
lg::info("[{:3d}/{}]------ {}", file_idx++, total_file_count, data.to_unique_name());
process_object_file_data(data, output_dir, config, skip_functions, skip_states);
if (postfile_callback) {
postfile_callback.value()();
}
});
lg::info("{}", stats.let.print());

2
decompiler/config/jak2/hacks.jsonc
View file

@ -458,7 +458,7 @@
115, // goto L55
121,
131
]
]
},
// Sometimes the game might use format strings that are fetched dynamically,

2
decompiler/main.cpp
View file

@ -159,7 +159,7 @@ int main(int argc, char** argv) {
// main decompile.
if (config.decompile_code) {
db.analyze_functions_ir2(out_folder, config, {});
db.analyze_functions_ir2(out_folder, config, {}, {}, {});
}
if (config.generate_all_types) {

1
decompiler/util/DataParser.cpp
View file

@ -1,6 +1,7 @@
#include "DataParser.h"
#include <stdexcept>
#include <unordered_map>
#include "common/util/Assert.h"

12
game/graphics/gfx.cpp
View file

@ -238,12 +238,12 @@ void LoadSettings() {
const GfxRendererModule* GetRenderer(GfxPipeline pipeline) {
switch (pipeline) {
case GfxPipeline::Invalid:
lg::error("Requested invalid renderer", pipeline);
lg::error("Requested invalid renderer", fmt::underlying(pipeline));
return NULL;
case GfxPipeline::OpenGL:
return &gRendererOpenGL;
default:
lg::error("Requested unknown renderer {}", (u64)pipeline);
lg::error("Requested unknown renderer {}", fmt::underlying(pipeline));
return NULL;
}
}
@ -509,7 +509,7 @@ void SetLod(RendererTreeType tree, int lod) {
g_global_settings.lod_tie = lod;
break;
default:
lg::error("Invalid tree {} specified for SetLod ({})", tree, lod);
lg::error("Invalid tree {} specified for SetLod ({})", fmt::underlying(tree), lod);
break;
}
}
@ -529,7 +529,7 @@ bool CollisionRendererGetMask(GfxGlobalSettings::CollisionRendererMode mode, int
ASSERT(arr_idx == 0);
return (g_global_settings.collision_skip_mask >> arr_ofs) & 1;
default:
lg::error("{} invalid params {} {}", __PRETTY_FUNCTION__, mode, mask_id);
lg::error("{} invalid params {} {}", __PRETTY_FUNCTION__, fmt::underlying(mode), mask_id);
return false;
}
}
@ -553,7 +553,7 @@ void CollisionRendererSetMask(GfxGlobalSettings::CollisionRendererMode mode, int
g_global_settings.collision_skip_mask |= 1 << arr_ofs;
break;
default:
lg::error("{} invalid params {} {}", __PRETTY_FUNCTION__, mode, mask_id);
lg::error("{} invalid params {} {}", __PRETTY_FUNCTION__, fmt::underlying(mode), mask_id);
break;
}
}
@ -577,7 +577,7 @@ void CollisionRendererClearMask(GfxGlobalSettings::CollisionRendererMode mode, i
g_global_settings.collision_skip_mask &= ~(1 << arr_ofs);
break;
default:
lg::error("{} invalid params {} {}", __PRETTY_FUNCTION__, mode, mask_id);
lg::error("{} invalid params {} {}", __PRETTY_FUNCTION__, fmt::underlying(mode), mask_id);
break;
}
}

2
game/runtime.cpp
View file

@ -377,7 +377,7 @@ RuntimeExitStatus exec_runtime(int argc, char** argv) {
if (enable_display) {
Gfx::Exit();
}
lg::info("GOAL Runtime Shutdown (code {})", MasterExit);
lg::info("GOAL Runtime Shutdown (code {})", fmt::underlying(MasterExit));
munmap(g_ee_main_mem, EE_MAIN_MEM_SIZE);
return MasterExit;
}

2
game/sound/989snd/sfxgrain.h
View file

@ -521,7 +521,7 @@ std::unique_ptr<Grain> new_grain(grain_type id, Args&&... args) {
case grain_type::COPY_REGISTER:
return std::make_unique<SFXGrain_CopyRegister>(std::forward<Args>(args)...);
default:
throw std::runtime_error(fmt::format("Unknown grain type {}", id));
throw std::runtime_error(fmt::format("Unknown grain type {}", fmt::underlying(id)));
}
return nullptr;
}

2
game/sound/sdshim.h
View file

@ -1,4 +1,6 @@
#pragma once
#include <memory>
#include "common/common_types.h"
#include "game/sound/common/voice.h"

2
game/system/vm/vm.cpp
View file

@ -60,7 +60,7 @@ void vm_prepare() {
void vm_init() {
if (status != Status::Uninited) {
lg::warn("[VM] unexpected status {}", status);
lg::warn("[VM] unexpected status {}", fmt::underlying(status));
}
lg::debug("[VM] Inited");

1
goalc/build_level/collide_drawable.cpp
View file

@ -1,5 +1,6 @@
#include "collide_drawable.h"
#include <unordered_map>
#include <unordered_set>
#include "common/util/Assert.h"

1
goalc/build_level/color_quantization.cpp
View file

@ -1,6 +1,7 @@
#include "color_quantization.h"
#include <algorithm>
#include <unordered_map>
#include "common/log/log.h"
#include "common/util/Assert.h"

5
goalc/compiler/IR.cpp
View file

@ -7,6 +7,7 @@
#include "goalc/emitter/IGen.h"
#include "third-party/fmt/core.h"
#include "third-party/fmt/format.h"
using namespace emitter;
namespace {
@ -1006,7 +1007,7 @@ void IR_StoreConstOffset::do_codegen(emitter::ObjectGenerator* gen,
} else {
throw std::runtime_error(
fmt::format("IR_StoreConstOffset::do_codegen can't handle this (c {} sz {})",
m_value->ireg().reg_class, m_size));
fmt::underlying(m_value->ireg().reg_class), m_size));
}
}
@ -1934,7 +1935,7 @@ IR_SplatVF::IR_SplatVF(bool use_color,
std::string IR_SplatVF::print() {
return fmt::format(".splat.vf{} {}, {}, {}", get_color_suffix_string(), m_dst->print(),
m_src->print(), m_element);
m_src->print(), fmt::underlying(m_element));
}
RegAllocInstr IR_SplatVF::to_rai() {

4
test/offline/CMakeLists.txt
View file

@ -1,5 +1,9 @@
add_executable(offline-test
${CMAKE_CURRENT_LIST_DIR}/config/config.cpp
${CMAKE_CURRENT_LIST_DIR}/framework/execution.cpp
${CMAKE_CURRENT_LIST_DIR}/framework/orchestration.cpp
${CMAKE_CURRENT_LIST_DIR}/framework/file_management.cpp
${CMAKE_CURRENT_LIST_DIR}/offline_test_main.cpp)
target_link_libraries(offline-test common gtest decomp compiler)

36
test/offline/config/config.cpp Normal file
View file

@ -0,0 +1,36 @@
#include "config.h"
#include "common/log/log.h"
#include "common/util/FileUtil.h"
#include "common/util/json_util.h"
OfflineTestConfig::OfflineTestConfig(const std::string_view& _game_name,
const std::string& _iso_data_path,
const u32 _num_threads,
const bool _dump_mode,
const bool _fail_on_cmp,
const bool _fail_on_compile,
const bool _pretty_print)
: game_name(_game_name),
iso_data_path(_iso_data_path),
num_threads(_num_threads),
dump_mode(_dump_mode),
fail_on_cmp(_fail_on_cmp),
fail_on_compile(_fail_on_compile),
pretty_print(_pretty_print) {
lg::info("Reading Configuration...");
auto json_file_path =
file_util::get_jak_project_dir() / "test" / "offline" / "config" / game_name / "config.jsonc";
if (!fs::exists(json_file_path)) {
lg::error("Couldn't load configuration, '{}' doesn't exist", json_file_path.string());
throw std::exception();
}
auto json = parse_commented_json(file_util::read_text_file(json_file_path.string()),
json_file_path.string());
dgos = json["dgos"].get<std::vector<std::string>>();
skip_compile_files = json["skip_compile_files"].get<std::unordered_set<std::string>>();
skip_compile_functions = json["skip_compile_functions"].get<std::unordered_set<std::string>>();
skip_compile_states =
json["skip_compile_states"]
.get<std::unordered_map<std::string, std::unordered_set<std::string>>>();
}

32
test/offline/config/config.h Normal file
View file

@ -0,0 +1,32 @@
#pragma once
#include <string>
#include <string_view>
#include <unordered_map>
#include <unordered_set>
#include <vector>
#include "common/common_types.h"
class OfflineTestConfig {
public:
OfflineTestConfig(const std::string_view& _game_name,
const std::string& _iso_data_path,
const u32 _num_threads,
const bool _dump_mode,
const bool _fail_on_cmp,
const bool _fail_on_compile,
const bool _pretty_print);
std::string game_name;
std::string iso_data_path;
u32 num_threads;
bool dump_mode;
bool fail_on_cmp = false;
bool fail_on_compile = false;
bool pretty_print = false;
std::vector<std::string> dgos;
std::unordered_set<std::string> skip_compile_files;
std::unordered_set<std::string> skip_compile_functions;
std::unordered_map<std::string, std::unordered_set<std::string>> skip_compile_states;
};

143
test/offline/framework/execution.cpp Normal file
View file

@ -0,0 +1,143 @@
#include "execution.h"
#include "common/util/StringUtil.h"
#include "common/util/diff.h"
#include "goalc/compiler/Compiler.h"
#include "test/offline/config/config.h"
#include "third-party/fmt/ranges.h"
// TODO - i think these should be partitioned by game name instead of it being in the filename
// (and the names not being consistent)
std::unordered_map<std::string, std::string> game_name_to_all_types1 = {
{"jak1", "all-types.gc"},
{"jak2", "jak2/all-types.gc"}};
void disassemble(OfflineTestDecompiler& dc) {
dc.db->process_link_data(*dc.config);
dc.db->find_code(*dc.config);
dc.db->process_labels();
}
void decompile(OfflineTestDecompiler& dc,
const OfflineTestConfig& config,
const std::shared_ptr<OfflineTestThreadStatus> status) {
dc.db->extract_art_info();
dc.db->ir2_top_level_pass(*dc.config);
dc.db->analyze_functions_ir2(
{}, *dc.config,
[status](std::string file_name) mutable { status->update_curr_file(file_name); },
[status]() mutable { status->complete_step(); }, config.skip_compile_functions,
config.skip_compile_states);
}
/// @brief Removes trailing new-lines and comment lines
std::string clean_decompilation_code(const std::string& in, const bool leave_comments = false) {
std::string out = in;
if (!leave_comments) {
std::vector<std::string> lines = split_string(in);
// Remove all lines that are comments
// comments are added only by us, meaning this _should_ be consistent
std::vector<std::string>::iterator line_itr = lines.begin();
while (line_itr != lines.end()) {
if (line_itr->rfind(";", 0) == 0) {
// remove comment line
line_itr = lines.erase(line_itr);
} else {
// iterate
++line_itr;
}
}
out = fmt::format("{}", fmt::join(lines, "\n"));
}
while (!out.empty() && out.back() == '\n') {
out.pop_back();
}
return out;
}
decompiler::ObjectFileData& get_data(OfflineTestDecompiler& dc,
const std::string& unique_name,
const std::string& name_in_dgo) {
auto& files = dc.db->obj_files_by_name.at(name_in_dgo);
auto it = std::find_if(files.begin(), files.end(), [&](const decompiler::ObjectFileData& data) {
return data.to_unique_name() == unique_name;
});
ASSERT(it != files.end());
return *it;
}
OfflineTestCompareResult compare(OfflineTestDecompiler& dc,
const OfflineTestWorkGroup& work_group,
const OfflineTestConfig& config) {
OfflineTestCompareResult compare_result;
for (const auto& collection : work_group.work_collections) {
for (const auto& file : collection.source_files) {
work_group.status->update_curr_file(file.name_in_dgo);
auto& data = get_data(dc, file.unique_name, file.name_in_dgo);
std::string result = clean_decompilation_code(data.full_output);
std::string ref = clean_decompilation_code(file_util::read_text_file(file.path.string()));
compare_result.total_files++;
compare_result.total_lines += str_util::line_count(result);
if (result != ref) {
compare_result.failing_files.push_back({file.unique_name, diff_strings(ref, result)});
compare_result.total_pass = false;
if (config.dump_mode) {
auto failure_dir = file_util::get_jak_project_dir() / "failures";
file_util::create_dir_if_needed(failure_dir);
file_util::write_text_file(failure_dir / fmt::format("{}_REF.gc", file.unique_name),
clean_decompilation_code(data.full_output, true));
}
} else {
compare_result.ok_files++;
}
work_group.status->complete_step();
}
}
return compare_result;
}
OfflineTestCompileResult compile(OfflineTestDecompiler& dc,
const OfflineTestWorkGroup& work_group,
const OfflineTestConfig& config) {
OfflineTestCompileResult result;
Compiler compiler(game_name_to_version(config.game_name));
compiler.run_front_end_on_file(
{"decompiler", "config", game_name_to_all_types1[config.game_name]});
compiler.run_front_end_on_file(
{"test", "decompiler", "reference", config.game_name, "decompiler-macros.gc"});
int total_lines = 0;
for (const auto& coll : work_group.work_collections) {
for (const auto& file : coll.source_files) {
work_group.status->update_curr_file(file.name_in_dgo);
if (config.skip_compile_files.count(file.name_in_dgo)) {
lg::warn("Skipping {}", file.name_in_dgo);
continue;
}
lg::info("Compiling {}...", file.unique_name);
auto& data = get_data(dc, file.unique_name, file.name_in_dgo);
try {
const auto& src = data.output_with_skips;
total_lines += str_util::line_count(src);
compiler.run_full_compiler_on_string_no_save(src, file.name_in_dgo);
} catch (const std::exception& e) {
result.ok = false;
result.failing_files.push_back({file.name_in_dgo, e.what()});
}
work_group.status->complete_step();
}
}
result.num_lines = total_lines;
return result;
}

22
test/offline/framework/execution.h Normal file
View file

@ -0,0 +1,22 @@
#pragma once
#include "orchestration.h"
#include "decompiler/ObjectFile/ObjectFileDB.h"
#include "test/offline/config/config.h"
struct OfflineTestDecompiler {
std::unique_ptr<decompiler::ObjectFileDB> db;
std::unique_ptr<decompiler::Config> config;
};
void disassemble(OfflineTestDecompiler& dc);
void decompile(OfflineTestDecompiler& dc,
const OfflineTestConfig& config,
const std::shared_ptr<OfflineTestThreadStatus> status);
OfflineTestCompareResult compare(OfflineTestDecompiler& dc,
const OfflineTestWorkGroup& work_group,
const OfflineTestConfig& config);
OfflineTestCompileResult compile(OfflineTestDecompiler& dc,
const OfflineTestWorkGroup& work_group,
const OfflineTestConfig& config);

145
test/offline/framework/file_management.cpp Normal file
View file

@ -0,0 +1,145 @@
#include "file_management.h"
#include <unordered_set>
#include <vector>
#include "common/log/log.h"
#include "common/util/json_util.h"
#include "third-party/fmt/core.h"
std::vector<OfflineTestSourceFile> find_source_files(const std::string& game_name,
const std::vector<std::string>& dgos,
const std::string& single_file) {
std::vector<OfflineTestSourceFile> result;
auto base_dir =
file_util::get_jak_project_dir() / "test" / "decompiler" / "reference" / game_name;
auto ref_file_paths = file_util::find_files_recursively(base_dir, std::regex(".*_REF\\..*"));
std::unordered_map<std::string, fs::path> ref_file_names = {};
for (const auto& path : ref_file_paths) {
auto ref_name = path.filename().replace_extension().string();
ref_name.erase(ref_name.begin() + ref_name.find("_REF"), ref_name.end());
if (single_file.empty() || ref_name == single_file) {
ref_file_names[ref_name] = path;
}
}
lg::info("Found {} reference files", ref_file_paths.size());
// use the all_objs.json file to place them in the correct build order
auto obj_json = parse_commented_json(
file_util::read_text_file(
(file_util::get_jak_project_dir() / "goal_src" / game_name / "build" / "all_objs.json")
.string()),
"all_objs.json");
std::unordered_set<std::string> matched_files;
for (auto& x : obj_json) {
auto unique_name = x[0].get<std::string>();
std::vector<std::string> dgoList = x[3].get<std::vector<std::string>>();
auto it = ref_file_names.find(unique_name);
if (it != ref_file_names.end()) {
// Check to see if we've included atleast one of the DGO/CGOs in our hardcoded list
// If not BLOW UP
std::optional<std::string> containing_dgo = {};
for (int i = 0; i < (int)dgoList.size(); i++) {
std::string& dgo = dgoList.at(i);
// can either be in the DGO or CGO folder, and can either end with .CGO or .DGO
if (std::find(dgos.begin(), dgos.end(), fmt::format("DGO/{}.DGO", dgo)) != dgos.end() ||
std::find(dgos.begin(), dgos.end(), fmt::format("DGO/{}.CGO", dgo)) != dgos.end() ||
std::find(dgos.begin(), dgos.end(), fmt::format("CGO/{}.DGO", dgo)) != dgos.end() ||
std::find(dgos.begin(), dgos.end(), fmt::format("CGO/{}.CGO", dgo)) != dgos.end()) {
containing_dgo = dgo;
break;
}
}
if (!containing_dgo) {
lg::error(
"File [{}] is in the following DGOs [{}], and not one of these is in our list! Add "
"it!",
unique_name, fmt::join(dgoList, ", "));
exit(1);
}
OfflineTestSourceFile file(it->second, containing_dgo.value(), x[1], it->first);
result.push_back(file);
matched_files.insert(unique_name);
}
}
if (matched_files.size() != ref_file_names.size()) {
lg::error("Some REF files were not matched to files in all_objs.json:");
for (const auto& [path, flag] : ref_file_names) {
if (matched_files.count(path) == 0) {
lg::error("- '{}'", path);
}
}
exit(1);
}
return result;
}
std::vector<OfflineTestArtFile> find_art_files(const std::string& game_name,
const std::vector<std::string>& dgos) {
// TODO - Jak 2
if (game_name != "jak1") {
return {};
}
std::vector<OfflineTestArtFile> result;
// use the all_objs.json file to place them in the correct build order
auto obj_json = parse_commented_json(
file_util::read_text_file(
(file_util::get_jak_project_dir() / "goal_src" / game_name / "build" / "all_objs.json")
.string()),
"all_objs.json");
for (const auto& x : obj_json) {
auto unique_name = x[0].get<std::string>();
auto version = x[2].get<int>();
std::vector<std::string> dgoList = x[3].get<std::vector<std::string>>();
if (version == 4) {
bool skip_file = false;
// Check to see if we've included atleast one of the DGO/CGOs in our hardcoded list
// If not BLOW UP
std::optional<std::string> containing_dgo = {};
for (int i = 0; i < (int)dgoList.size(); i++) {
std::string& dgo = dgoList.at(i);
if (dgo == "NO-XGO") {
skip_file = true;
break;
}
// can either be in the DGO or CGO folder, and can either end with .CGO or .DGO
// TODO - Jak 2 folder may structure will be different!
if (std::find(dgos.begin(), dgos.end(), fmt::format("DGO/{}.DGO", dgo)) != dgos.end() ||
std::find(dgos.begin(), dgos.end(), fmt::format("DGO/{}.CGO", dgo)) != dgos.end() ||
std::find(dgos.begin(), dgos.end(), fmt::format("CGO/{}.DGO", dgo)) != dgos.end() ||
std::find(dgos.begin(), dgos.end(), fmt::format("CGO/{}.CGO", dgo)) != dgos.end()) {
containing_dgo = dgo;
break;
}
}
if (skip_file) {
continue;
}
if (!containing_dgo) {
lg::error(
"File [{}] is in the following DGOs [{}], and not one of these is in our list! Add "
"it!",
unique_name, fmt::join(dgoList, ", "));
exit(1);
}
OfflineTestArtFile file(containing_dgo.value(), x[1], unique_name);
result.push_back(file);
}
}
return result;
}

36
test/offline/framework/file_management.h Normal file
View file

@ -0,0 +1,36 @@
#pragma once
#include <string>
#include "common/util/FileUtil.h"
struct OfflineTestSourceFile {
OfflineTestSourceFile(fs::path _path,
std::string _containing_dgo,
std::string _name_in_dgo,
std::string _unique_name)
: path(_path),
containing_dgo(_containing_dgo),
name_in_dgo(_name_in_dgo),
unique_name(_unique_name){};
fs::path path;
std::string containing_dgo;
std::string name_in_dgo;
std::string unique_name;
};
struct OfflineTestArtFile {
OfflineTestArtFile(std::string _containing_dgo,
std::string _name_in_dgo,
std::string _unique_name)
: containing_dgo(_containing_dgo), name_in_dgo(_name_in_dgo), unique_name(_unique_name){};
std::string containing_dgo;
std::string name_in_dgo;
std::string unique_name;
};
std::vector<OfflineTestSourceFile> find_source_files(const std::string& game_name,
const std::vector<std::string>& dgos,
const std::string& single_file);
std::vector<OfflineTestArtFile> find_art_files(const std::string& game_name,
const std::vector<std::string>& dgos);

289
test/offline/framework/orchestration.cpp Normal file
View file

@ -0,0 +1,289 @@
#include "orchestration.h"
#include "execution.h"
#include "file_management.h"
#include "common/log/log.h"
#include "common/util/FileUtil.h"
#include "common/util/StringUtil.h"
#include "common/util/diff.h"
#include "decompiler/ObjectFile/ObjectFileDB.h"
#include "test/offline/config/config.h"
#include "third-party/fmt/color.h"
#include "third-party/fmt/core.h"
#include "third-party/fmt/ranges.h"
// TODO - this should probably go somewhere common when it's needed eventually
std::unordered_map<std::string, std::string> game_name_to_config = {
{"jak1", "jak1_ntsc_black_label.jsonc"},
{"jak2", "jak2_ntsc_v1.jsonc"}};
OfflineTestThreadManager g_offline_test_thread_manager;
OfflineTestDecompiler setup_decompiler(const OfflineTestWorkGroup& work,
const fs::path& iso_data_path,
const OfflineTestConfig& offline_config) {
// TODO - pull out extractor logic to determine release into common and use here
OfflineTestDecompiler dc;
dc.config = std::make_unique<decompiler::Config>(
decompiler::read_config_file((file_util::get_jak_project_dir() / "decompiler" / "config" /
game_name_to_config[offline_config.game_name])
.string()));
// TODO - do I need to limit the `inputs.jsonc` as well, or is the decompiler smart enough
// to lazily load the DGOs as needed based on the allowed objects?
// modify the config
std::unordered_set<std::string> object_files;
for (const auto& coll : work.work_collections) {
for (auto& file : coll.source_files) {
object_files.insert(file.name_in_dgo); // todo, make this work with unique_name
}
for (auto& file : coll.art_files) {
object_files.insert(file.unique_name);
}
}
dc.config->allowed_objects = object_files;
// don't try to do this because we can't write the file
dc.config->generate_symbol_definition_map = false;
std::vector<fs::path> dgo_paths;
for (auto& x : offline_config.dgos) {
dgo_paths.push_back(iso_data_path / x);
}
dc.db = std::make_unique<decompiler::ObjectFileDB>(dgo_paths, dc.config->obj_file_name_map_file,
std::vector<fs::path>{},
std::vector<fs::path>{}, *dc.config);
std::unordered_set<std::string> db_files;
for (auto& files_by_name : dc.db->obj_files_by_name) {
for (auto& f : files_by_name.second) {
db_files.insert(f.to_unique_name());
}
}
if (db_files.size() != object_files.size()) {
lg::error("DB file error: has {} entries, but expected {", db_files.size(),
object_files.size());
for (const auto& coll : work.work_collections) {
for (auto& file : coll.source_files) {
if (!db_files.count(file.unique_name)) {
lg::error(
"didn't find {}, make sure it's part of the DGO inputs and not in the banned objects "
"list\n",
file.unique_name);
}
}
for (auto& file : coll.art_files) {
if (!db_files.count(file.unique_name)) {
lg::error("didn't find {}\n", file.unique_name);
}
}
}
exit(1);
}
return dc;
}
std::vector<std::future<OfflineTestThreadResult>> distribute_work(
const OfflineTestConfig& offline_config,
const std::vector<OfflineTestSourceFile>& files,
const std::vector<OfflineTestArtFile>& art_files) {
// First, group files by their DGO so they can be partitioned
std::unordered_map<std::string, OfflineTestWorkCollection> work_colls = {};
for (const auto& file : files) {
if (work_colls.count(file.containing_dgo) == 0) {
work_colls[file.containing_dgo] = OfflineTestWorkCollection();
}
work_colls[file.containing_dgo].source_files.push_back(file);
}
for (const auto& file : art_files) {
if (work_colls.count(file.containing_dgo) == 0) {
work_colls[file.containing_dgo] = OfflineTestWorkCollection();
}
work_colls[file.containing_dgo].art_files.push_back(file);
}
// Now partition by DGO so that threads do not consume unnecessary or duplicate resources
// this is a half-decent approximation of a greedy-knapsack approach (where the knapsack can be
// overstuffed) Repeatedly just add to the thread with the current least amount of work
//
// TODO - if it ends up being that it would be advantageous to split up a massive dgo into
// multiple threads (ie. lots in engine) then this should be improved to accomodate that.
//
// TODO - additionally, if we have more threads than we can actually utilize we should not
// reserve them and dynamically adjust the used thread count
std::vector<OfflineTestWorkGroup> work_groups = {};
for (int i = 0; i < offline_config.num_threads; i++) {
auto new_group = OfflineTestWorkGroup();
new_group.status = std::make_shared<OfflineTestThreadStatus>(offline_config);
work_groups.push_back(new_group);
g_offline_test_thread_manager.statuses.push_back(new_group.status);
}
g_offline_test_thread_manager.print_current_test_status(offline_config);
for (const auto& [dgo, work] : work_colls) {
// Find the smallest group
u32 smallest_group_idx = 0;
for (int i = 0; i < work_groups.size(); i++) {
if (work_groups[i].work_size() < work_groups[smallest_group_idx].work_size()) {
smallest_group_idx = i;
}
}
// Add the DGO and the files to it
work_groups[smallest_group_idx].dgos.push_back(dgo);
work_groups[smallest_group_idx].work_collections.push_back(work);
work_groups[smallest_group_idx].status->dgos.push_back(dgo);
work_groups[smallest_group_idx].status->total_steps =
work_groups[smallest_group_idx].work_size() * 3; // decomp, compare, compile
}
// Now we can finally create the futures
std::vector<std::future<OfflineTestThreadResult>> threads;
for (auto& work_group : work_groups) {
threads.push_back(std::async(std::launch::async, [&, work_group]() mutable {
OfflineTestThreadResult result;
if (work_group.work_size() == 0) {
return result;
}
Timer total_timer;
Timer decompiler_timer;
work_group.status->update_stage(OfflineTestThreadStatus::Stage::PREPARING);
auto decompiler =
setup_decompiler(work_group, fs::path(offline_config.iso_data_path), offline_config);
disassemble(decompiler);
work_group.status->update_stage(OfflineTestThreadStatus::Stage::DECOMPILING);
decompile(decompiler, offline_config, work_group.status);
result.time_spent_decompiling = decompiler_timer.getSeconds();
work_group.status->update_stage(OfflineTestThreadStatus::Stage::COMPARING);
result.compare = compare(decompiler, work_group, offline_config);
if (!result.compare.total_pass) {
result.exit_code = 1;
if (offline_config.fail_on_cmp) {
work_group.status->update_stage(OfflineTestThreadStatus::Stage::FAILED);
return result;
}
}
// TODO - if anything has failed, fail fast and skip compiling
Timer compile_timer;
work_group.status->update_stage(OfflineTestThreadStatus::Stage::COMPILING);
result.compile = compile(decompiler, work_group, offline_config);
result.time_spent_compiling = compile_timer.getSeconds();
if (!result.compile.ok) {
work_group.status->update_stage(OfflineTestThreadStatus::Stage::FAILED);
result.exit_code = 1;
} else {
work_group.status->update_stage(OfflineTestThreadStatus::Stage::FINISHED);
}
result.total_time = total_timer.getSeconds();
return result;
}));
}
return threads;
}
void OfflineTestThreadStatus::update_stage(Stage new_stage) {
stage = new_stage;
g_offline_test_thread_manager.print_current_test_status(config);
}
void OfflineTestThreadStatus::update_curr_file(const std::string& _curr_file) {
curr_file = _curr_file;
g_offline_test_thread_manager.print_current_test_status(config);
}
void OfflineTestThreadStatus::complete_step() {
curr_step++;
g_offline_test_thread_manager.print_current_test_status(config);
}
std::tuple<fmt::color, std::string> thread_stage_to_str(OfflineTestThreadStatus::Stage stage) {
switch (stage) {
case OfflineTestThreadStatus::Stage::IDLE:
return {fmt::color::gray, "IDLE"};
case OfflineTestThreadStatus::Stage::PREPARING:
return {fmt::color::light_gray, "PREPARING"};
case OfflineTestThreadStatus::Stage::DECOMPILING:
return {fmt::color::orange, "DECOMPILING"};
case OfflineTestThreadStatus::Stage::COMPARING:
return {fmt::color::dark_orange, "COMPARING"};
case OfflineTestThreadStatus::Stage::COMPILING:
return {fmt::color::cyan, "COMPILING"};
case OfflineTestThreadStatus::Stage::FINISHED:
return {fmt::color::light_green, "FINISHED"};
case OfflineTestThreadStatus::Stage::FAILED:
return {fmt::color::red, "FAILED"};
default:
return {fmt::color::red, "UNKNOWN"};
}
}
std::string thread_dgos_to_str(std::vector<std::string> dgos) {
std::vector<std::string> ones_to_print = {};
for (const auto& dgo : dgos) {
ones_to_print.push_back(dgo);
if (ones_to_print.size() >= 3) {
break;
}
}
if (ones_to_print.size() < dgos.size()) {
return fmt::format("{}, +{} more", fmt::join(ones_to_print, ","),
dgos.size() - ones_to_print.size());
}
return fmt::format("{}", fmt::join(ones_to_print, ","));
}
std::string thread_progress_bar(u32 curr_step, u32 total_steps) {
const u32 completion =
std::floor(static_cast<double>(curr_step) / static_cast<double>(total_steps) * 100.0);
const u32 completed_segments = completion / 10;
std::string progress_bar = "";
int added_segments = 0;
for (int i = 0; i < completed_segments; i++) {
progress_bar += "■";
added_segments++;
}
while (added_segments < 10) {
progress_bar += "□";
added_segments++;
}
return progress_bar;
}
void OfflineTestThreadManager::print_current_test_status(const OfflineTestConfig& config) {
if (!config.pretty_print) {
return;
}
// [DECOMP] ▰▰▰▰▰▰▱▱▱▱ (PRI, RUI, FOR, +3 more)
// [1/30] - target-turret-shot // MUTED TEXT
std::lock_guard<std::mutex> guard(print_lock);
fmt::print("\x1b[{}A", g_offline_test_thread_manager.statuses.size() * 2); // move n lines up
for (const auto& status : g_offline_test_thread_manager.statuses) {
// first line
const auto [color, stage_text] = thread_stage_to_str(status->stage);
fmt::print(
"\33[2K\r[{:>12}] {} ({})\n", fmt::styled(stage_text, fmt::fg(color)),
fmt::styled(thread_progress_bar(status->curr_step, status->total_steps), fmt::fg(color)),
thread_dgos_to_str(status->dgos));
// second line
fmt::print(fmt::fg(fmt::color::gray), "\33[2K\r{:>14} - {}\n",
fmt::format("[{}/{}]", status->curr_step, status->total_steps), status->curr_file);
}
}

131
test/offline/framework/orchestration.h Normal file
View file

@ -0,0 +1,131 @@
#pragma once
#include <future>
#include <mutex>
#include <string>
#include <vector>
#include "file_management.h"
#include "common/util/Timer.h"
#include "test/offline/config/config.h"
struct OfflineTestCompareResult {
struct Fail {
std::string filename;
std::string diff;
};
std::vector<Fail> failing_files;
int total_files = 0;
int ok_files = 0;
int total_lines = 0;
bool total_pass = true;
void add(const OfflineTestCompareResult& other) {
failing_files.insert(failing_files.end(), other.failing_files.begin(),
other.failing_files.end());
total_files += other.total_files;
ok_files += other.ok_files;
total_lines += other.total_lines;
if (!other.total_pass) {
total_pass = false;
}
}
};
struct OfflineTestCompileResult {
bool ok = true;
struct Fail {
std::string filename;
std::string error;
};
std::vector<Fail> failing_files;
int num_lines = 0;
void add(const OfflineTestCompileResult& other) {
failing_files.insert(failing_files.end(), other.failing_files.begin(),
other.failing_files.end());
num_lines += other.num_lines;
if (!other.ok) {
ok = false;
}
}
};
/// @brief A simple struct to contain the reason for failure from a thread
struct OfflineTestThreadResult {
int exit_code = 0;
std::string reason;
float time_spent_compiling = 0;
float time_spent_decompiling = 0;
float total_time = 0;
OfflineTestCompareResult compare;
OfflineTestCompileResult compile;
void add(const OfflineTestThreadResult& other) {
if (other.exit_code) {
exit_code = other.exit_code;
}
time_spent_compiling += other.time_spent_compiling;
time_spent_decompiling += other.time_spent_decompiling;
total_time += other.total_time;
compare.add(other.compare);
compile.add(other.compile);
}
};
class OfflineTestThreadStatus {
public:
enum class Stage { IDLE, PREPARING, DECOMPILING, COMPARING, COMPILING, FAILED, FINISHED };
OfflineTestThreadStatus(const OfflineTestConfig& _config) : config(_config){};
Stage stage = Stage::IDLE;
uint32_t total_steps = 0;
uint32_t curr_step = 0;
std::vector<std::string> dgos = {};
std::string curr_file;
OfflineTestConfig config;
void update_stage(Stage new_stage);
void update_curr_file(const std::string& _curr_file);
void complete_step();
};
struct OfflineTestWorkCollection {
std::vector<OfflineTestSourceFile> source_files;
std::vector<OfflineTestArtFile> art_files;
};
struct OfflineTestWorkGroup {
std::vector<std::string> dgos;
std::vector<OfflineTestWorkCollection> work_collections;
std::shared_ptr<OfflineTestThreadStatus> status;
int work_size() const {
int i = 0;
for (const auto& coll : work_collections) {
i += coll.source_files.size() + coll.art_files.size();
}
return i;
}
};
class OfflineTestThreadManager {
public:
void print_current_test_status(const OfflineTestConfig& config);
std::vector<std::shared_ptr<OfflineTestThreadStatus>> statuses = {};
private:
std::mutex print_lock;
};
extern OfflineTestThreadManager g_offline_test_thread_manager;
std::vector<std::future<OfflineTestThreadResult>> distribute_work(
const OfflineTestConfig& offline_config,
const std::vector<OfflineTestSourceFile>& files,
const std::vector<OfflineTestArtFile>& art_files);

554
test/offline/offline_test_main.cpp
View file

@ -1,472 +1,31 @@
#include <future>
#include <memory>
#include <string>
#include <unordered_set>
#include <vector>
#include "common/common_types.h"
#include "common/log/log.h"
#include "common/util/FileUtil.h"
#include "common/util/Timer.h"
#include "common/util/diff.h"
#include "common/util/json_util.h"
#include <common/util/unicode_util.h>
#include "common/util/unicode_util.h"
#include "config/config.h"
#include "decompiler/ObjectFile/ObjectFileDB.h"
#include "goalc/compiler/Compiler.h"
#include "framework/file_management.h"
#include "framework/orchestration.h"
#include "third-party/CLI11.hpp"
#include "third-party/fmt/format.h"
// json config file data (previously was in source of offline_test_main.cpp)
struct OfflineTestConfig {
std::vector<std::string> dgos;
std::unordered_set<std::string> skip_compile_files;
std::unordered_set<std::string> skip_compile_functions;
std::unordered_map<std::string, std::unordered_set<std::string>> skip_compile_states;
};
struct DecompilerFile {
fs::path path;
std::string name_in_dgo;
std::string unique_name;
std::string reference;
};
struct DecompilerArtFile {
std::string name_in_dgo;
std::string unique_name;
};
struct Decompiler {
std::unique_ptr<decompiler::ObjectFileDB> db;
std::unique_ptr<decompiler::Config> config;
};
// TODO - this should probably go somewhere common when it's needed eventually
std::unordered_map<std::string, std::string> game_name_to_config = {
{"jak1", "jak1_ntsc_black_label.jsonc"},
{"jak2", "jak2_ntsc_v1.jsonc"}};
// TODO - i think these should be partitioned by game name instead of it being in the filename
// (and the names not being consistent)
std::unordered_map<std::string, std::string> game_name_to_all_types = {
{"jak1", "all-types.gc"},
{"jak2", "jak2/all-types.gc"}};
Decompiler setup_decompiler(const std::vector<DecompilerFile>& files,
const std::vector<DecompilerArtFile>& art_files,
const fs::path& iso_data_path,
const OfflineTestConfig& offline_config,
const std::string& game_name) {
// TODO - pull out extractor logic to determine release into common and use here
Decompiler dc;
decompiler::init_opcode_info();
dc.config = std::make_unique<decompiler::Config>(decompiler::read_config_file(
(file_util::get_jak_project_dir() / "decompiler" / "config" / game_name_to_config[game_name])
.string()));
// modify the config
std::unordered_set<std::string> object_files;
for (auto& file : files) {
object_files.insert(file.name_in_dgo); // todo, make this work with unique_name
}
for (auto& file : art_files) {
object_files.insert(file.unique_name);
}
dc.config->allowed_objects = object_files;
// don't try to do this because we can't write the file
dc.config->generate_symbol_definition_map = false;
std::vector<fs::path> dgo_paths;
for (auto& x : offline_config.dgos) {
dgo_paths.push_back(iso_data_path / x);
}
dc.db = std::make_unique<decompiler::ObjectFileDB>(dgo_paths, dc.config->obj_file_name_map_file,
std::vector<fs::path>{},
std::vector<fs::path>{}, *dc.config);
std::unordered_set<std::string> db_files;
for (auto& files_by_name : dc.db->obj_files_by_name) {
for (auto& f : files_by_name.second) {
db_files.insert(f.to_unique_name());
}
}
if (db_files.size() != files.size() + art_files.size()) {
lg::error("DB file error: {} {} {}", db_files.size(), files.size(), art_files.size());
for (auto& f : files) {
if (!db_files.count(f.unique_name)) {
lg::error(
"didn't find {}, make sure it's part of the DGO inputs and not in the banned objects "
"list\n",
f.unique_name);
}
}
for (auto& f : art_files) {
if (!db_files.count(f.unique_name)) {
lg::error("didn't find {}\n", f.unique_name);
}
}
exit(1);
}
return dc;
// TODO - move this into a common lib eventually
void clear_terminal() {
#if defined _WIN32
system("cls");
#elif defined(__LINUX__) || defined(__gnu_linux__) || defined(__linux__)
system("clear");
#elif defined(__APPLE__)
system("clear");
#endif
}
void disassemble(Decompiler& dc) {
dc.db->process_link_data(*dc.config);
dc.db->find_code(*dc.config);
dc.db->process_labels();
}
void decompile(Decompiler& dc, const OfflineTestConfig& config) {
dc.db->extract_art_info();
dc.db->ir2_top_level_pass(*dc.config);
dc.db->analyze_functions_ir2({}, *dc.config, config.skip_compile_functions,
config.skip_compile_states);
}
/// @brief Removes trailing new-lines and comment lines
std::string clean_decompilation_code(const std::string& in, const bool leave_comments = false) {
std::string out = in;
if (!leave_comments) {
std::vector<std::string> lines = split_string(in);
// Remove all lines that are comments
// comments are added only by us, meaning this _should_ be consistent
std::vector<std::string>::iterator line_itr = lines.begin();
while (line_itr != lines.end()) {
if (line_itr->rfind(";", 0) == 0) {
// remove comment line
line_itr = lines.erase(line_itr);
} else {
// iterate
line_itr++;
}
}
out = fmt::format("{}", fmt::join(lines, "\n"));
}
while (!out.empty() && out.back() == '\n') {
out.pop_back();
}
return out;
}
decompiler::ObjectFileData& get_data(Decompiler& dc,
const std::string& unique_name,
const std::string& name_in_dgo) {
auto& files = dc.db->obj_files_by_name.at(name_in_dgo);
auto it = std::find_if(files.begin(), files.end(), [&](const decompiler::ObjectFileData& data) {
return data.to_unique_name() == unique_name;
});
ASSERT(it != files.end());
return *it;
}
int line_count(const std::string& str) {
int result = 0;
for (auto& c : str) {
if (c == '\n') {
result++;
}
}
return result;
}
struct CompareResult {
struct Fail {
std::string filename;
std::string diff;
};
std::vector<Fail> failing_files;
int total_files = 0;
int ok_files = 0;
int total_lines = 0;
bool total_pass = true;
void add(const CompareResult& other) {
failing_files.insert(failing_files.end(), other.failing_files.begin(),
other.failing_files.end());
total_files += other.total_files;
ok_files += other.ok_files;
total_lines += other.total_lines;
if (!other.total_pass) {
total_pass = false;
}
}
};
CompareResult compare(Decompiler& dc, const std::vector<DecompilerFile>& refs, bool dump_mode) {
CompareResult compare_result;
for (const auto& file : refs) {
auto& data = get_data(dc, file.unique_name, file.name_in_dgo);
std::string result = clean_decompilation_code(data.full_output);
std::string ref = clean_decompilation_code(file_util::read_text_file(file.path.string()));
compare_result.total_files++;
compare_result.total_lines += line_count(result);
if (result != ref) {
compare_result.failing_files.push_back({file.unique_name, diff_strings(ref, result)});
compare_result.total_pass = false;
if (dump_mode) {
auto failure_dir = file_util::get_jak_project_dir() / "failures";
file_util::create_dir_if_needed(failure_dir);
file_util::write_text_file(failure_dir / fmt::format("{}_REF.gc", file.unique_name),
clean_decompilation_code(data.full_output, true));
}
} else {
compare_result.ok_files++;
}
}
return compare_result;
}
struct CompileResult {
bool ok = true;
struct Fail {
std::string filename;
std::string error;
};
std::vector<Fail> failing_files;
int num_lines = 0;
void add(const CompileResult& other) {
failing_files.insert(failing_files.end(), other.failing_files.begin(),
other.failing_files.end());
num_lines += other.num_lines;
if (!other.ok) {
ok = false;
}
}
};
CompileResult compile(Decompiler& dc,
const std::vector<DecompilerFile>& refs,
const OfflineTestConfig& config,
const std::string& game_name) {
CompileResult result;
Compiler compiler(game_name_to_version(game_name));
compiler.run_front_end_on_file({"decompiler", "config", game_name_to_all_types[game_name]});
compiler.run_front_end_on_file(
{"test", "decompiler", "reference", game_name, "decompiler-macros.gc"});
int total_lines = 0;
for (const auto& file : refs) {
if (config.skip_compile_files.count(file.name_in_dgo)) {
fmt::print("Skipping {}\n", file.name_in_dgo);
continue;
}
fmt::print("Compiling {}...\n", file.unique_name);
auto& data = get_data(dc, file.unique_name, file.name_in_dgo);
try {
const auto& src = data.output_with_skips;
total_lines += line_count(src);
compiler.run_full_compiler_on_string_no_save(src, file.name_in_dgo);
} catch (const std::exception& e) {
result.ok = false;
result.failing_files.push_back({file.name_in_dgo, e.what()});
}
}
result.num_lines = total_lines;
return result;
}
std::vector<DecompilerArtFile> find_art_files(const std::string& game_name,
const std::vector<std::string>& dgos) {
std::vector<DecompilerArtFile> result;
// use the all_objs.json file to place them in the correct build order
auto obj_json = parse_commented_json(
file_util::read_text_file(
(file_util::get_jak_project_dir() / "goal_src" / game_name / "build" / "all_objs.json")
.string()),
"all_objs.json");
for (const auto& x : obj_json) {
auto unique_name = x[0].get<std::string>();
auto version = x[2].get<int>();
std::vector<std::string> dgoList = x[3].get<std::vector<std::string>>();
if (version == 4) {
bool skip_this = false;
// Check to see if we've included atleast one of the DGO/CGOs in our hardcoded list
// If not BLOW UP
bool dgoValidated = false;
for (int i = 0; i < (int)dgoList.size(); i++) {
std::string& dgo = dgoList.at(i);
if (dgo == "NO-XGO") {
skip_this = true;
break;
}
// can either be in the DGO or CGO folder, and can either end with .CGO or .DGO
// TODO - Jak 2 Folder structure will be different!
if (std::find(dgos.begin(), dgos.end(), fmt::format("DGO/{}.DGO", dgo)) != dgos.end() ||
std::find(dgos.begin(), dgos.end(), fmt::format("DGO/{}.CGO", dgo)) != dgos.end() ||
std::find(dgos.begin(), dgos.end(), fmt::format("CGO/{}.DGO", dgo)) != dgos.end() ||
std::find(dgos.begin(), dgos.end(), fmt::format("CGO/{}.CGO", dgo)) != dgos.end()) {
dgoValidated = true;
}
}
if (skip_this) {
continue;
}
if (!dgoValidated) {
lg::error(
"File [{}] is in the following DGOs [{}], and not one of these is in our list! Add "
"it!",
unique_name, fmt::join(dgoList, ", "));
exit(1);
}
DecompilerArtFile file;
file.unique_name = unique_name;
file.name_in_dgo = x[1];
result.push_back(file);
}
}
return result;
}
std::vector<DecompilerFile> find_files(const std::string& game_name,
const std::vector<std::string>& dgos,
const std::string& single_file) {
std::vector<DecompilerFile> result;
auto base_dir =
file_util::get_jak_project_dir() / "test" / "decompiler" / "reference" / game_name;
auto ref_file_paths = file_util::find_files_recursively(base_dir, std::regex(".*_REF\\..*"));
std::unordered_map<std::string, fs::path> ref_file_names = {};
for (const auto& path : ref_file_paths) {
auto ref_name = path.filename().replace_extension().string();
ref_name.erase(ref_name.begin() + ref_name.find("_REF"), ref_name.end());
if (single_file.empty() || ref_name == single_file) {
ref_file_names[ref_name] = path;
}
}
lg::info("Found {} reference files", ref_file_paths.size());
// use the all_objs.json file to place them in the correct build order
auto obj_json = parse_commented_json(
file_util::read_text_file(
(file_util::get_jak_project_dir() / "goal_src" / game_name / "build" / "all_objs.json")
.string()),
"all_objs.json");
std::unordered_set<std::string> matched_files;
for (auto& x : obj_json) {
auto unique_name = x[0].get<std::string>();
std::vector<std::string> dgoList = x[3].get<std::vector<std::string>>();
auto it = ref_file_names.find(unique_name);
if (it != ref_file_names.end()) {
// Check to see if we've included atleast one of the DGO/CGOs in our hardcoded list
// If not BLOW UP
bool dgoValidated = false;
for (int i = 0; i < (int)dgoList.size(); i++) {
std::string& dgo = dgoList.at(i);
// can either be in the DGO or CGO folder, and can either end with .CGO or .DGO
// TODO - Jak 2 Folder structure will be different!
if (std::find(dgos.begin(), dgos.end(), fmt::format("DGO/{}.DGO", dgo)) != dgos.end() ||
std::find(dgos.begin(), dgos.end(), fmt::format("DGO/{}.CGO", dgo)) != dgos.end() ||
std::find(dgos.begin(), dgos.end(), fmt::format("CGO/{}.DGO", dgo)) != dgos.end() ||
std::find(dgos.begin(), dgos.end(), fmt::format("CGO/{}.CGO", dgo)) != dgos.end()) {
dgoValidated = true;
}
}
if (!dgoValidated) {
lg::error(
"File [{}] is in the following DGOs [{}], and not one of these is in our list! Add "
"it!",
unique_name, fmt::join(dgoList, ", "));
exit(1);
}
DecompilerFile file;
file.path = it->second;
file.unique_name = it->first;
file.name_in_dgo = x[1];
result.push_back(file);
matched_files.insert(unique_name);
}
}
if (matched_files.size() != ref_file_names.size()) {
lg::error("Some REF files were not matched to files in all_objs.json:");
for (const auto& [path, flag] : ref_file_names) {
if (matched_files.count(path) == 0) {
lg::error("- '{}'", path);
}
}
exit(1);
}
return result;
}
/*!
* Read and parse the json config file, config.json, located in test/offline
*/
std::optional<OfflineTestConfig> parse_config(const std::string_view& game_name) {
lg::info("Reading Configuration...");
auto json_file_path =
file_util::get_jak_project_dir() / "test" / "offline" / "config" / game_name / "config.jsonc";
if (!fs::exists(json_file_path)) {
lg::error("Couldn't load configuration, '{}' doesn't exist", json_file_path.string());
return {};
}
auto json = parse_commented_json(file_util::read_text_file(json_file_path.string()),
json_file_path.string());
OfflineTestConfig result;
result.dgos = json["dgos"].get<std::vector<std::string>>();
result.skip_compile_files = json["skip_compile_files"].get<std::unordered_set<std::string>>();
result.skip_compile_functions =
json["skip_compile_functions"].get<std::unordered_set<std::string>>();
result.skip_compile_states =
json["skip_compile_states"]
.get<std::unordered_map<std::string, std::unordered_set<std::string>>>();
return std::make_optional(result);
}
/// @brief A simple struct to contain the reason for failure from a thread
struct OfflineTestResult {
int exit_code = 0;
std::string reason;
float time_spent_compiling = 0;
float time_spent_decompiling = 0;
float total_time = 0;
CompareResult compare;
CompileResult compile;
void add(const OfflineTestResult& other) {
if (other.exit_code) {
exit_code = other.exit_code;
}
time_spent_compiling += other.time_spent_compiling;
time_spent_decompiling += other.time_spent_decompiling;
total_time += other.total_time;
compare.add(other.compare);
compile.add(other.compile);
}
};
int main(int argc, char* argv[]) {
ArgumentGuard u8_guard(argc, argv);
lg::initialize();
bool dump_current_output = false;
std::string iso_data_path;
std::string game_name;
@ -475,6 +34,7 @@ int main(int argc, char* argv[]) {
std::string single_file = "";
uint32_t num_threads = 1;
bool fail_on_cmp = false;
bool pretty_print = false;
CLI::App app{"OpenGOAL - Offline Reference Test Runner"};
app.add_option("--iso_data_path", iso_data_path, "The path to the folder with the ISO data files")
@ -492,92 +52,48 @@ int main(int argc, char* argv[]) {
"Limit the offline test routine to a single file to decompile/compile -- useful "
"when you are just iterating on a single file");
app.add_flag("--fail-on-cmp", fail_on_cmp, "Fail the tests immediately if the comparison fails");
app.add_flag("-p,--pretty-print", pretty_print,
"Use the condensed and progress-indicating printing format");
app.validate_positionals();
CLI11_PARSE(app, argc, argv);
if (pretty_print) {
lg::set_stdout_level(lg::level::off);
clear_terminal();
}
lg::initialize();
if (!file_util::setup_project_path(std::nullopt)) {
lg::error("Couldn't setup project path, tool is supposed to be ran in the jak-project repo!");
return 1;
}
auto config = parse_config(game_name);
if (!config.has_value()) {
return 1;
}
// Setup environment, fetch files
auto config = OfflineTestConfig(game_name, iso_data_path, num_threads, dump_current_output,
fail_on_cmp, false, pretty_print);
lg::info("Finding files...");
auto files = find_files(game_name, config->dgos, single_file);
if (max_files > 0 && max_files < (int)files.size()) {
files.erase(files.begin() + max_files, files.end());
auto source_files = find_source_files(game_name, config.dgos, single_file);
if (max_files > 0 && max_files < (int)source_files.size()) {
source_files.erase(source_files.begin() + max_files, source_files.end());
}
auto art_files = find_art_files(game_name, config.dgos);
std::vector<DecompilerArtFile> art_files;
if (game_name == "jak1") {
art_files = find_art_files(game_name, config->dgos);
}
// Create a bunch of threads to disassemble/decompile/compile the files
// Figure out the number of threads, prepare their statuses and start printing it
if (num_threads < 1) {
num_threads = 1;
} else if (num_threads > 1) {
num_threads = std::min(num_threads, std::thread::hardware_concurrency());
}
// First, prepare our batches of files to be processed
std::vector<std::vector<DecompilerFile>> work_groups = {};
for (size_t i = 0; i < num_threads; i++) {
work_groups.push_back({});
}
int total_added = 0;
for (auto& file : files) {
work_groups.at(total_added % num_threads).push_back(file);
total_added++;
}
// TODO - nicer printing, very messy with dozens of threads processing the job
// Now we create a thread to process each group of work, and then await them
std::vector<std::future<OfflineTestResult>> threads;
// Distribute the work amongst the threads, partitioning by DGO
decompiler::init_opcode_info();
for (const auto& work_group : work_groups) {
threads.push_back(std::async(std::launch::async, [&]() {
OfflineTestResult result;
Timer total_timer;
Timer decompiler_timer;
auto decompiler = setup_decompiler(work_group, art_files, fs::path(iso_data_path),
config.value(), game_name);
disassemble(decompiler);
decompile(decompiler, config.value());
// It's about 100ms per file to decompile on average
// meaning that when we have all 900 files, a full offline test will take 1.5 minutes
result.time_spent_decompiling = decompiler_timer.getSeconds();
result.compare = compare(decompiler, work_group, dump_current_output);
if (!result.compare.total_pass) {
result.exit_code = 1;
if (fail_on_cmp) {
return result;
}
}
// TODO - if anything has failed, skip compiling
Timer compile_timer;
result.compile = compile(decompiler, work_group, config.value(), game_name);
result.time_spent_compiling = compile_timer.getSeconds();
if (!result.compile.ok) {
result.exit_code = 1;
}
result.total_time = total_timer.getSeconds();
return result;
}));
}
auto workers = distribute_work(config, source_files, art_files);
// summarize results:
OfflineTestResult total;
for (auto& thread : threads) {
auto ret = thread.get();
OfflineTestThreadResult total;
for (auto& worker : workers) {
auto ret = worker.get();
total.add(ret);
}

278
third-party/fmt/color.h generated vendored
View file

@ -11,6 +11,7 @@
#include "format.h"
FMT_BEGIN_NAMESPACE
FMT_MODULE_EXPORT_BEGIN
enum class color : uint32_t {
alice_blue = 0xF0F8FF, // rgb(240,248,255)
@ -177,9 +178,13 @@ enum class terminal_color : uint8_t {
enum class emphasis : uint8_t {
bold = 1,
italic = 1 << 1,
underline = 1 << 2,
strikethrough = 1 << 3
faint = 1 << 1,
italic = 1 << 2,
underline = 1 << 3,
blink = 1 << 4,
reverse = 1 << 5,
conceal = 1 << 6,
strikethrough = 1 << 7,
};
// rgb is a struct for red, green and blue colors.
@ -198,21 +203,20 @@ struct rgb {
uint8_t b;
};
namespace detail {
FMT_BEGIN_DETAIL_NAMESPACE
// color is a struct of either a rgb color or a terminal color.
struct color_type {
FMT_CONSTEXPR color_type() FMT_NOEXCEPT : is_rgb(), value{} {}
FMT_CONSTEXPR color_type(color rgb_color) FMT_NOEXCEPT : is_rgb(true),
value{} {
FMT_CONSTEXPR color_type() noexcept : is_rgb(), value{} {}
FMT_CONSTEXPR color_type(color rgb_color) noexcept : is_rgb(true), value{} {
value.rgb_color = static_cast<uint32_t>(rgb_color);
}
FMT_CONSTEXPR color_type(rgb rgb_color) FMT_NOEXCEPT : is_rgb(true), value{} {
FMT_CONSTEXPR color_type(rgb rgb_color) noexcept : is_rgb(true), value{} {
value.rgb_color = (static_cast<uint32_t>(rgb_color.r) << 16) |
(static_cast<uint32_t>(rgb_color.g) << 8) | rgb_color.b;
}
FMT_CONSTEXPR color_type(terminal_color term_color) FMT_NOEXCEPT : is_rgb(),
value{} {
FMT_CONSTEXPR color_type(terminal_color term_color) noexcept
: is_rgb(), value{} {
value.term_color = static_cast<uint8_t>(term_color);
}
bool is_rgb;
@ -221,15 +225,14 @@ struct color_type {
uint32_t rgb_color;
} value;
};
} // namespace detail
// Experimental text formatting support.
FMT_END_DETAIL_NAMESPACE
/** A text style consisting of foreground and background colors and emphasis. */
class text_style {
public:
FMT_CONSTEXPR text_style(emphasis em = emphasis()) FMT_NOEXCEPT
: set_foreground_color(),
set_background_color(),
ems(em) {}
FMT_CONSTEXPR text_style(emphasis em = emphasis()) noexcept
: set_foreground_color(), set_background_color(), ems(em) {}
FMT_CONSTEXPR text_style& operator|=(const text_style& rhs) {
if (!set_foreground_color) {
@ -260,63 +263,32 @@ class text_style {
return lhs |= rhs;
}
FMT_CONSTEXPR text_style& operator&=(const text_style& rhs) {
if (!set_foreground_color) {
set_foreground_color = rhs.set_foreground_color;
foreground_color = rhs.foreground_color;
} else if (rhs.set_foreground_color) {
if (!foreground_color.is_rgb || !rhs.foreground_color.is_rgb)
FMT_THROW(format_error("can't AND a terminal color"));
foreground_color.value.rgb_color &= rhs.foreground_color.value.rgb_color;
}
if (!set_background_color) {
set_background_color = rhs.set_background_color;
background_color = rhs.background_color;
} else if (rhs.set_background_color) {
if (!background_color.is_rgb || !rhs.background_color.is_rgb)
FMT_THROW(format_error("can't AND a terminal color"));
background_color.value.rgb_color &= rhs.background_color.value.rgb_color;
}
ems = static_cast<emphasis>(static_cast<uint8_t>(ems) &
static_cast<uint8_t>(rhs.ems));
return *this;
}
friend FMT_CONSTEXPR text_style operator&(text_style lhs,
const text_style& rhs) {
return lhs &= rhs;
}
FMT_CONSTEXPR bool has_foreground() const FMT_NOEXCEPT {
FMT_CONSTEXPR bool has_foreground() const noexcept {
return set_foreground_color;
}
FMT_CONSTEXPR bool has_background() const FMT_NOEXCEPT {
FMT_CONSTEXPR bool has_background() const noexcept {
return set_background_color;
}
FMT_CONSTEXPR bool has_emphasis() const FMT_NOEXCEPT {
FMT_CONSTEXPR bool has_emphasis() const noexcept {
return static_cast<uint8_t>(ems) != 0;
}
FMT_CONSTEXPR detail::color_type get_foreground() const FMT_NOEXCEPT {
FMT_CONSTEXPR detail::color_type get_foreground() const noexcept {
FMT_ASSERT(has_foreground(), "no foreground specified for this style");
return foreground_color;
}
FMT_CONSTEXPR detail::color_type get_background() const FMT_NOEXCEPT {
FMT_CONSTEXPR detail::color_type get_background() const noexcept {
FMT_ASSERT(has_background(), "no background specified for this style");
return background_color;
}
FMT_CONSTEXPR emphasis get_emphasis() const FMT_NOEXCEPT {
FMT_CONSTEXPR emphasis get_emphasis() const noexcept {
FMT_ASSERT(has_emphasis(), "no emphasis specified for this style");
return ems;
}
private:
FMT_CONSTEXPR text_style(bool is_foreground,
detail::color_type text_color) FMT_NOEXCEPT
: set_foreground_color(),
set_background_color(),
ems() {
detail::color_type text_color) noexcept
: set_foreground_color(), set_background_color(), ems() {
if (is_foreground) {
foreground_color = text_color;
set_foreground_color = true;
@ -326,10 +298,9 @@ class text_style {
}
}
friend FMT_CONSTEXPR_DECL text_style fg(detail::color_type foreground)
FMT_NOEXCEPT;
friend FMT_CONSTEXPR_DECL text_style bg(detail::color_type background)
FMT_NOEXCEPT;
friend FMT_CONSTEXPR text_style fg(detail::color_type foreground) noexcept;
friend FMT_CONSTEXPR text_style bg(detail::color_type background) noexcept;
detail::color_type foreground_color;
detail::color_type background_color;
@ -338,27 +309,29 @@ class text_style {
emphasis ems;
};
FMT_CONSTEXPR text_style fg(detail::color_type foreground) FMT_NOEXCEPT {
return text_style(/*is_foreground=*/true, foreground);
/** Creates a text style from the foreground (text) color. */
FMT_CONSTEXPR inline text_style fg(detail::color_type foreground) noexcept {
return text_style(true, foreground);
}
FMT_CONSTEXPR text_style bg(detail::color_type background) FMT_NOEXCEPT {
return text_style(/*is_foreground=*/false, background);
/** Creates a text style from the background color. */
FMT_CONSTEXPR inline text_style bg(detail::color_type background) noexcept {
return text_style(false, background);
}
FMT_CONSTEXPR text_style operator|(emphasis lhs, emphasis rhs) FMT_NOEXCEPT {
FMT_CONSTEXPR inline text_style operator|(emphasis lhs, emphasis rhs) noexcept {
return text_style(lhs) | rhs;
}
namespace detail {
FMT_BEGIN_DETAIL_NAMESPACE
template <typename Char> struct ansi_color_escape {
FMT_CONSTEXPR ansi_color_escape(detail::color_type text_color,
const char* esc) FMT_NOEXCEPT {
const char* esc) noexcept {
// If we have a terminal color, we need to output another escape code
// sequence.
if (!text_color.is_rgb) {
bool is_background = esc == detail::data::background_color;
bool is_background = esc == string_view("\x1b[48;2;");
uint32_t value = text_color.value.term_color;
// Background ASCII codes are the same as the foreground ones but with
// 10 more.
@ -389,17 +362,19 @@ template <typename Char> struct ansi_color_escape {
to_esc(color.b, buffer + 15, 'm');
buffer[19] = static_cast<Char>(0);
}
FMT_CONSTEXPR ansi_color_escape(emphasis em) FMT_NOEXCEPT {
uint8_t em_codes[4] = {};
uint8_t em_bits = static_cast<uint8_t>(em);
if (em_bits & static_cast<uint8_t>(emphasis::bold)) em_codes[0] = 1;
if (em_bits & static_cast<uint8_t>(emphasis::italic)) em_codes[1] = 3;
if (em_bits & static_cast<uint8_t>(emphasis::underline)) em_codes[2] = 4;
if (em_bits & static_cast<uint8_t>(emphasis::strikethrough))
em_codes[3] = 9;
FMT_CONSTEXPR ansi_color_escape(emphasis em) noexcept {
uint8_t em_codes[num_emphases] = {};
if (has_emphasis(em, emphasis::bold)) em_codes[0] = 1;
if (has_emphasis(em, emphasis::faint)) em_codes[1] = 2;
if (has_emphasis(em, emphasis::italic)) em_codes[2] = 3;
if (has_emphasis(em, emphasis::underline)) em_codes[3] = 4;
if (has_emphasis(em, emphasis::blink)) em_codes[4] = 5;
if (has_emphasis(em, emphasis::reverse)) em_codes[5] = 7;
if (has_emphasis(em, emphasis::conceal)) em_codes[6] = 8;
if (has_emphasis(em, emphasis::strikethrough)) em_codes[7] = 9;
size_t index = 0;
for (int i = 0; i < 4; ++i) {
for (size_t i = 0; i < num_emphases; ++i) {
if (!em_codes[i]) continue;
buffer[index++] = static_cast<Char>('\x1b');
buffer[index++] = static_cast<Char>('[');
@ -408,67 +383,76 @@ template <typename Char> struct ansi_color_escape {
}
buffer[index++] = static_cast<Char>(0);
}
FMT_CONSTEXPR operator const Char*() const FMT_NOEXCEPT { return buffer; }
FMT_CONSTEXPR operator const Char*() const noexcept { return buffer; }
FMT_CONSTEXPR const Char* begin() const FMT_NOEXCEPT { return buffer; }
FMT_CONSTEXPR const Char* end() const FMT_NOEXCEPT {
FMT_CONSTEXPR const Char* begin() const noexcept { return buffer; }
FMT_CONSTEXPR_CHAR_TRAITS const Char* end() const noexcept {
return buffer + std::char_traits<Char>::length(buffer);
}
private:
Char buffer[7u + 3u * 4u + 1u];
static constexpr size_t num_emphases = 8;
Char buffer[7u + 3u * num_emphases + 1u];
static FMT_CONSTEXPR void to_esc(uint8_t c, Char* out,
char delimiter) FMT_NOEXCEPT {
char delimiter) noexcept {
out[0] = static_cast<Char>('0' + c / 100);
out[1] = static_cast<Char>('0' + c / 10 % 10);
out[2] = static_cast<Char>('0' + c % 10);
out[3] = static_cast<Char>(delimiter);
}
static FMT_CONSTEXPR bool has_emphasis(emphasis em, emphasis mask) noexcept {
return static_cast<uint8_t>(em) & static_cast<uint8_t>(mask);
}
};
template <typename Char>
FMT_CONSTEXPR ansi_color_escape<Char> make_foreground_color(
detail::color_type foreground) FMT_NOEXCEPT {
return ansi_color_escape<Char>(foreground, detail::data::foreground_color);
detail::color_type foreground) noexcept {
return ansi_color_escape<Char>(foreground, "\x1b[38;2;");
}
template <typename Char>
FMT_CONSTEXPR ansi_color_escape<Char> make_background_color(
detail::color_type background) FMT_NOEXCEPT {
return ansi_color_escape<Char>(background, detail::data::background_color);
detail::color_type background) noexcept {
return ansi_color_escape<Char>(background, "\x1b[48;2;");
}
template <typename Char>
FMT_CONSTEXPR ansi_color_escape<Char> make_emphasis(emphasis em) FMT_NOEXCEPT {
FMT_CONSTEXPR ansi_color_escape<Char> make_emphasis(emphasis em) noexcept {
return ansi_color_escape<Char>(em);
}
template <typename Char>
inline void fputs(const Char* chars, FILE* stream) FMT_NOEXCEPT {
std::fputs(chars, stream);
template <typename Char> inline void fputs(const Char* chars, FILE* stream) {
int result = std::fputs(chars, stream);
if (result < 0)
FMT_THROW(system_error(errno, FMT_STRING("cannot write to file")));
}
template <>
inline void fputs<wchar_t>(const wchar_t* chars, FILE* stream) FMT_NOEXCEPT {
std::fputws(chars, stream);
template <> inline void fputs<wchar_t>(const wchar_t* chars, FILE* stream) {
int result = std::fputws(chars, stream);
if (result < 0)
FMT_THROW(system_error(errno, FMT_STRING("cannot write to file")));
}
template <typename Char> inline void reset_color(FILE* stream) FMT_NOEXCEPT {
fputs(detail::data::reset_color, stream);
template <typename Char> inline void reset_color(FILE* stream) {
fputs("\x1b[0m", stream);
}
template <> inline void reset_color<wchar_t>(FILE* stream) FMT_NOEXCEPT {
fputs(detail::data::wreset_color, stream);
template <> inline void reset_color<wchar_t>(FILE* stream) {
fputs(L"\x1b[0m", stream);
}
template <typename Char>
inline void reset_color(buffer<Char>& buffer) FMT_NOEXCEPT {
const char* begin = data::reset_color;
const char* end = begin + sizeof(data::reset_color) - 1;
buffer.append(begin, end);
template <typename Char> inline void reset_color(buffer<Char>& buffer) {
auto reset_color = string_view("\x1b[0m");
buffer.append(reset_color.begin(), reset_color.end());
}
template <typename T> struct styled_arg {
const T& value;
text_style style;
};
template <typename Char>
void vformat_to(buffer<Char>& buf, const text_style& ts,
basic_string_view<Char> format_str,
@ -489,18 +473,23 @@ void vformat_to(buffer<Char>& buf, const text_style& ts,
auto background = detail::make_background_color<Char>(ts.get_background());
buf.append(background.begin(), background.end());
}
detail::vformat_to(buf, format_str, args);
detail::vformat_to(buf, format_str, args, {});
if (has_style) detail::reset_color<Char>(buf);
}
} // namespace detail
FMT_END_DETAIL_NAMESPACE
template <typename S, typename Char = char_t<S>>
void vprint(std::FILE* f, const text_style& ts, const S& format,
basic_format_args<buffer_context<type_identity_t<Char>>> args) {
basic_memory_buffer<Char> buf;
detail::vformat_to(buf, ts, to_string_view(format), args);
buf.push_back(Char(0));
detail::fputs(buf.data(), f);
detail::vformat_to(buf, ts, detail::to_string_view(format), args);
if (detail::is_utf8()) {
detail::print(f, basic_string_view<Char>(buf.begin(), buf.size()));
} else {
buf.push_back(Char(0));
detail::fputs(buf.data(), f);
}
}
/**
@ -519,15 +508,19 @@ template <typename S, typename... Args,
void print(std::FILE* f, const text_style& ts, const S& format_str,
const Args&... args) {
vprint(f, ts, format_str,
fmt::make_args_checked<Args...>(format_str, args...));
fmt::make_format_args<buffer_context<char_t<S>>>(args...));
}
/**
\rst
Formats a string and prints it to stdout using ANSI escape sequences to
specify text formatting.
Example:
**Example**::
fmt::print(fmt::emphasis::bold | fg(fmt::color::red),
"Elapsed time: {0:.2f} seconds", 1.23);
\endrst
*/
template <typename S, typename... Args,
FMT_ENABLE_IF(detail::is_string<S>::value)>
@ -540,7 +533,7 @@ inline std::basic_string<Char> vformat(
const text_style& ts, const S& format_str,
basic_format_args<buffer_context<type_identity_t<Char>>> args) {
basic_memory_buffer<Char> buf;
detail::vformat_to(buf, ts, to_string_view(format_str), args);
detail::vformat_to(buf, ts, detail::to_string_view(format_str), args);
return fmt::to_string(buf);
}
@ -559,8 +552,8 @@ inline std::basic_string<Char> vformat(
template <typename S, typename... Args, typename Char = char_t<S>>
inline std::basic_string<Char> format(const text_style& ts, const S& format_str,
const Args&... args) {
return vformat(ts, to_string_view(format_str),
fmt::make_args_checked<Args...>(format_str, args...));
return fmt::vformat(ts, detail::to_string_view(format_str),
fmt::make_format_args<buffer_context<Char>>(args...));
}
/**
@ -571,9 +564,9 @@ template <typename OutputIt, typename Char,
OutputIt vformat_to(
OutputIt out, const text_style& ts, basic_string_view<Char> format_str,
basic_format_args<buffer_context<type_identity_t<Char>>> args) {
decltype(detail::get_buffer<Char>(out)) buf(detail::get_buffer_init(out));
auto&& buf = detail::get_buffer<Char>(out);
detail::vformat_to(buf, ts, format_str, args);
return detail::get_iterator(buf);
return detail::get_iterator(buf, out);
}
/**
@ -594,10 +587,65 @@ template <typename OutputIt, typename S, typename... Args,
inline auto format_to(OutputIt out, const text_style& ts, const S& format_str,
Args&&... args) ->
typename std::enable_if<enable, OutputIt>::type {
return vformat_to(out, ts, to_string_view(format_str),
fmt::make_args_checked<Args...>(format_str, args...));
return vformat_to(out, ts, detail::to_string_view(format_str),
fmt::make_format_args<buffer_context<char_t<S>>>(args...));
}
template <typename T, typename Char>
struct formatter<detail::styled_arg<T>, Char> : formatter<T, Char> {
template <typename FormatContext>
auto format(const detail::styled_arg<T>& arg, FormatContext& ctx) const
-> decltype(ctx.out()) {
const auto& ts = arg.style;
const auto& value = arg.value;
auto out = ctx.out();
bool has_style = false;
if (ts.has_emphasis()) {
has_style = true;
auto emphasis = detail::make_emphasis<Char>(ts.get_emphasis());
out = std::copy(emphasis.begin(), emphasis.end(), out);
}
if (ts.has_foreground()) {
has_style = true;
auto foreground =
detail::make_foreground_color<Char>(ts.get_foreground());
out = std::copy(foreground.begin(), foreground.end(), out);
}
if (ts.has_background()) {
has_style = true;
auto background =
detail::make_background_color<Char>(ts.get_background());
out = std::copy(background.begin(), background.end(), out);
}
out = formatter<T, Char>::format(value, ctx);
if (has_style) {
auto reset_color = string_view("\x1b[0m");
out = std::copy(reset_color.begin(), reset_color.end(), out);
}
return out;
}
};
/**
\rst
Returns an argument that will be formatted using ANSI escape sequences,
to be used in a formatting function.
**Example**::
fmt::print("Elapsed time: {0:.2f} seconds",
fmt::styled(1.23, fmt::fg(fmt::color::green) |
fmt::bg(fmt::color::blue)));
\endrst
*/
template <typename T>
FMT_CONSTEXPR auto styled(const T& value, text_style ts)
-> detail::styled_arg<remove_cvref_t<T>> {
return detail::styled_arg<remove_cvref_t<T>>{value, ts};
}
FMT_MODULE_EXPORT_END
FMT_END_NAMESPACE
#endif // FMT_COLOR_H_

3704
third-party/fmt/core.h generated vendored
View file

File diff suppressed because it is too large Load diff

3625
third-party/fmt/format-inl.h generated vendored
View file

File diff suppressed because it is too large Load diff

93
third-party/fmt/format.cc generated vendored
View file

@ -10,90 +10,35 @@
FMT_BEGIN_NAMESPACE
namespace detail {
template <typename T>
int format_float(char* buf, std::size_t size, const char* format, int precision,
T value) {
#ifdef FMT_FUZZ
if (precision > 100000)
throw std::runtime_error(
"fuzz mode - avoid large allocation inside snprintf");
#endif
// Suppress the warning about nonliteral format string.
int (*snprintf_ptr)(char*, size_t, const char*, ...) = FMT_SNPRINTF;
return precision < 0 ? snprintf_ptr(buf, size, format, value)
: snprintf_ptr(buf, size, format, precision, value);
}
template FMT_API dragonbox::decimal_fp<float> dragonbox::to_decimal(float x)
FMT_NOEXCEPT;
template FMT_API dragonbox::decimal_fp<double> dragonbox::to_decimal(double x)
FMT_NOEXCEPT;
// DEPRECATED! This function exists for ABI compatibility.
template <typename Char>
typename basic_format_context<std::back_insert_iterator<buffer<Char>>,
Char>::iterator
vformat_to(buffer<Char>& buf, basic_string_view<Char> format_str,
basic_format_args<basic_format_context<
std::back_insert_iterator<buffer<type_identity_t<Char>>>,
type_identity_t<Char>>>
args) {
using iterator = std::back_insert_iterator<buffer<char>>;
using context = basic_format_context<
std::back_insert_iterator<buffer<type_identity_t<Char>>>,
type_identity_t<Char>>;
auto out = iterator(buf);
format_handler<iterator, Char, context> h(out, format_str, args, {});
parse_format_string<false>(format_str, h);
return out;
}
template basic_format_context<std::back_insert_iterator<buffer<char>>,
char>::iterator
vformat_to(buffer<char>&, string_view,
basic_format_args<basic_format_context<
std::back_insert_iterator<buffer<type_identity_t<char>>>,
type_identity_t<char>>>);
} // namespace detail
template struct FMT_INSTANTIATION_DEF_API detail::basic_data<void>;
// Workaround a bug in MSVC2013 that prevents instantiation of format_float.
int (*instantiate_format_float)(double, int, detail::float_specs,
detail::buffer<char>&) = detail::format_float;
template FMT_API auto dragonbox::to_decimal(float x) noexcept
-> dragonbox::decimal_fp<float>;
template FMT_API auto dragonbox::to_decimal(double x) noexcept
-> dragonbox::decimal_fp<double>;
#ifndef FMT_STATIC_THOUSANDS_SEPARATOR
template FMT_API detail::locale_ref::locale_ref(const std::locale& loc);
template FMT_API std::locale detail::locale_ref::get<std::locale>() const;
template FMT_API locale_ref::locale_ref(const std::locale& loc);
template FMT_API auto locale_ref::get<std::locale>() const -> std::locale;
#endif
// Explicit instantiations for char.
template FMT_API std::string detail::grouping_impl<char>(locale_ref);
template FMT_API char detail::thousands_sep_impl(locale_ref);
template FMT_API char detail::decimal_point_impl(locale_ref);
template FMT_API auto thousands_sep_impl(locale_ref)
-> thousands_sep_result<char>;
template FMT_API auto decimal_point_impl(locale_ref) -> char;
template FMT_API void detail::buffer<char>::append(const char*, const char*);
template FMT_API void buffer<char>::append(const char*, const char*);
template FMT_API void detail::vformat_to(
detail::buffer<char>&, string_view,
basic_format_args<FMT_BUFFER_CONTEXT(char)>, detail::locale_ref);
template FMT_API int detail::snprintf_float(double, int, detail::float_specs,
detail::buffer<char>&);
template FMT_API int detail::snprintf_float(long double, int,
detail::float_specs,
detail::buffer<char>&);
template FMT_API int detail::format_float(double, int, detail::float_specs,
detail::buffer<char>&);
template FMT_API int detail::format_float(long double, int, detail::float_specs,
detail::buffer<char>&);
template FMT_API void vformat_to(buffer<char>&, string_view,
basic_format_args<FMT_BUFFER_CONTEXT(char)>,
locale_ref);
// Explicit instantiations for wchar_t.
template FMT_API std::string detail::grouping_impl<wchar_t>(locale_ref);
template FMT_API wchar_t detail::thousands_sep_impl(locale_ref);
template FMT_API wchar_t detail::decimal_point_impl(locale_ref);
template FMT_API auto thousands_sep_impl(locale_ref)
-> thousands_sep_result<wchar_t>;
template FMT_API auto decimal_point_impl(locale_ref) -> wchar_t;
template FMT_API void detail::buffer<wchar_t>::append(const wchar_t*,
const wchar_t*);
template FMT_API void buffer<wchar_t>::append(const wchar_t*, const wchar_t*);
} // namespace detail
FMT_END_NAMESPACE

6107
third-party/fmt/format.h generated vendored
View file

File diff suppressed because it is too large Load diff

716
third-party/fmt/ranges.h generated vendored
View file

@ -13,47 +13,13 @@
#define FMT_RANGES_H_
#include <initializer_list>
#include <tuple>
#include <type_traits>
#include "format.h"
// output only up to N items from the range.
#ifndef FMT_RANGE_OUTPUT_LENGTH_LIMIT
# define FMT_RANGE_OUTPUT_LENGTH_LIMIT 256
#endif
FMT_BEGIN_NAMESPACE
template <typename Char> struct formatting_base {
template <typename ParseContext>
FMT_CONSTEXPR auto parse(ParseContext& ctx) -> decltype(ctx.begin()) {
return ctx.begin();
}
};
template <typename Char, typename Enable = void>
struct formatting_range : formatting_base<Char> {
static FMT_CONSTEXPR_DECL const size_t range_length_limit =
FMT_RANGE_OUTPUT_LENGTH_LIMIT; // output only up to N items from the
// range.
Char prefix;
Char delimiter;
Char postfix;
formatting_range() : prefix('{'), delimiter(','), postfix('}') {}
static FMT_CONSTEXPR_DECL const bool add_delimiter_spaces = true;
static FMT_CONSTEXPR_DECL const bool add_prepostfix_space = false;
};
template <typename Char, typename Enable = void>
struct formatting_tuple : formatting_base<Char> {
Char prefix;
Char delimiter;
Char postfix;
formatting_tuple() : prefix('('), delimiter(','), postfix(')') {}
static FMT_CONSTEXPR_DECL const bool add_delimiter_spaces = true;
static FMT_CONSTEXPR_DECL const bool add_prepostfix_space = false;
};
namespace detail {
template <typename RangeT, typename OutputIterator>
@ -75,47 +41,145 @@ OutputIterator copy(char ch, OutputIterator out) {
return out;
}
/// Return true value if T has std::string interface, like std::string_view.
template <typename T> class is_like_std_string {
template <typename OutputIterator>
OutputIterator copy(wchar_t ch, OutputIterator out) {
*out++ = ch;
return out;
}
// Returns true if T has a std::string-like interface, like std::string_view.
template <typename T> class is_std_string_like {
template <typename U>
static auto check(U* p)
-> decltype((void)p->find('a'), p->length(), (void)p->data(), int());
template <typename> static void check(...);
public:
static FMT_CONSTEXPR_DECL const bool value =
is_string<T>::value || !std::is_void<decltype(check<T>(nullptr))>::value;
static constexpr const bool value =
is_string<T>::value ||
std::is_convertible<T, std_string_view<char>>::value ||
!std::is_void<decltype(check<T>(nullptr))>::value;
};
template <typename Char>
struct is_like_std_string<fmt::basic_string_view<Char>> : std::true_type {};
struct is_std_string_like<fmt::basic_string_view<Char>> : std::true_type {};
template <typename T> class is_map {
template <typename U> static auto check(U*) -> typename U::mapped_type;
template <typename> static void check(...);
public:
#ifdef FMT_FORMAT_MAP_AS_LIST
static constexpr const bool value = false;
#else
static constexpr const bool value =
!std::is_void<decltype(check<T>(nullptr))>::value;
#endif
};
template <typename T> class is_set {
template <typename U> static auto check(U*) -> typename U::key_type;
template <typename> static void check(...);
public:
#ifdef FMT_FORMAT_SET_AS_LIST
static constexpr const bool value = false;
#else
static constexpr const bool value =
!std::is_void<decltype(check<T>(nullptr))>::value && !is_map<T>::value;
#endif
};
template <typename... Ts> struct conditional_helper {};
template <typename T, typename _ = void> struct is_range_ : std::false_type {};
#if !FMT_MSC_VER || FMT_MSC_VER > 1800
#if !FMT_MSC_VERSION || FMT_MSC_VERSION > 1800
# define FMT_DECLTYPE_RETURN(val) \
->decltype(val) { return val; } \
static_assert( \
true, "") // This makes it so that a semicolon is required after the
// macro, which helps clang-format handle the formatting.
// C array overload
template <typename T, std::size_t N>
auto range_begin(const T (&arr)[N]) -> const T* {
return arr;
}
template <typename T, std::size_t N>
auto range_end(const T (&arr)[N]) -> const T* {
return arr + N;
}
template <typename T, typename Enable = void>
struct has_member_fn_begin_end_t : std::false_type {};
template <typename T>
struct is_range_<
T, conditional_t<false,
conditional_helper<decltype(std::declval<T>().begin()),
decltype(std::declval<T>().end())>,
void>> : std::true_type {};
struct has_member_fn_begin_end_t<T, void_t<decltype(std::declval<T>().begin()),
decltype(std::declval<T>().end())>>
: std::true_type {};
// Member function overload
template <typename T>
auto range_begin(T&& rng) FMT_DECLTYPE_RETURN(static_cast<T&&>(rng).begin());
template <typename T>
auto range_end(T&& rng) FMT_DECLTYPE_RETURN(static_cast<T&&>(rng).end());
// ADL overload. Only participates in overload resolution if member functions
// are not found.
template <typename T>
auto range_begin(T&& rng)
-> enable_if_t<!has_member_fn_begin_end_t<T&&>::value,
decltype(begin(static_cast<T&&>(rng)))> {
return begin(static_cast<T&&>(rng));
}
template <typename T>
auto range_end(T&& rng) -> enable_if_t<!has_member_fn_begin_end_t<T&&>::value,
decltype(end(static_cast<T&&>(rng)))> {
return end(static_cast<T&&>(rng));
}
template <typename T, typename Enable = void>
struct has_const_begin_end : std::false_type {};
template <typename T, typename Enable = void>
struct has_mutable_begin_end : std::false_type {};
template <typename T>
struct has_const_begin_end<
T,
void_t<
decltype(detail::range_begin(std::declval<const remove_cvref_t<T>&>())),
decltype(detail::range_end(std::declval<const remove_cvref_t<T>&>()))>>
: std::true_type {};
template <typename T>
struct has_mutable_begin_end<
T, void_t<decltype(detail::range_begin(std::declval<T>())),
decltype(detail::range_end(std::declval<T>())),
enable_if_t<std::is_copy_constructible<T>::value>>>
: std::true_type {};
template <typename T>
struct is_range_<T, void>
: std::integral_constant<bool, (has_const_begin_end<T>::value ||
has_mutable_begin_end<T>::value)> {};
# undef FMT_DECLTYPE_RETURN
#endif
/// tuple_size and tuple_element check.
// tuple_size and tuple_element check.
template <typename T> class is_tuple_like_ {
template <typename U>
static auto check(U* p) -> decltype(std::tuple_size<U>::value, int());
template <typename> static void check(...);
public:
static FMT_CONSTEXPR_DECL const bool value =
static constexpr const bool value =
!std::is_void<decltype(check<T>(nullptr))>::value;
};
// Check for integer_sequence
#if defined(__cpp_lib_integer_sequence) || FMT_MSC_VER >= 1900
#if defined(__cpp_lib_integer_sequence) || FMT_MSC_VERSION >= 1900
template <typename T, T... N>
using integer_sequence = std::integer_sequence<T, N...>;
template <size_t... N> using index_sequence = std::index_sequence<N...>;
@ -138,8 +202,33 @@ template <size_t N>
using make_index_sequence = make_integer_sequence<size_t, N>;
#endif
template <typename T>
using tuple_index_sequence = make_index_sequence<std::tuple_size<T>::value>;
template <typename T, typename C, bool = is_tuple_like_<T>::value>
class is_tuple_formattable_ {
public:
static constexpr const bool value = false;
};
template <typename T, typename C> class is_tuple_formattable_<T, C, true> {
template <std::size_t... Is>
static std::true_type check2(index_sequence<Is...>,
integer_sequence<bool, (Is == Is)...>);
static std::false_type check2(...);
template <std::size_t... Is>
static decltype(check2(
index_sequence<Is...>{},
integer_sequence<
bool, (is_formattable<typename std::tuple_element<Is, T>::type,
C>::value)...>{})) check(index_sequence<Is...>);
public:
static constexpr const bool value =
decltype(check(tuple_index_sequence<T>{}))::value;
};
template <class Tuple, class F, size_t... Is>
void for_each(index_sequence<Is...>, Tuple&& tup, F&& f) FMT_NOEXCEPT {
void for_each(index_sequence<Is...>, Tuple&& tup, F&& f) noexcept {
using std::get;
// using free function get<I>(T) now.
const int _[] = {0, ((void)f(get<Is>(tup)), 0)...};
@ -157,194 +246,429 @@ template <class Tuple, class F> void for_each(Tuple&& tup, F&& f) {
for_each(indexes, std::forward<Tuple>(tup), std::forward<F>(f));
}
#if FMT_MSC_VERSION && FMT_MSC_VERSION < 1920
// Older MSVC doesn't get the reference type correctly for arrays.
template <typename R> struct range_reference_type_impl {
using type = decltype(*detail::range_begin(std::declval<R&>()));
};
template <typename T, std::size_t N> struct range_reference_type_impl<T[N]> {
using type = T&;
};
template <typename T>
using range_reference_type = typename range_reference_type_impl<T>::type;
#else
template <typename Range>
using value_type = remove_cvref_t<decltype(*std::declval<Range>().begin())>;
using range_reference_type =
decltype(*detail::range_begin(std::declval<Range&>()));
#endif
template <typename Arg, FMT_ENABLE_IF(!is_like_std_string<
typename std::decay<Arg>::type>::value)>
FMT_CONSTEXPR const char* format_str_quoted(bool add_space, const Arg&) {
return add_space ? " {}" : "{}";
// We don't use the Range's value_type for anything, but we do need the Range's
// reference type, with cv-ref stripped.
template <typename Range>
using uncvref_type = remove_cvref_t<range_reference_type<Range>>;
template <typename Range>
using uncvref_first_type =
remove_cvref_t<decltype(std::declval<range_reference_type<Range>>().first)>;
template <typename Range>
using uncvref_second_type = remove_cvref_t<
decltype(std::declval<range_reference_type<Range>>().second)>;
template <typename OutputIt> OutputIt write_delimiter(OutputIt out) {
*out++ = ',';
*out++ = ' ';
return out;
}
template <typename Arg, FMT_ENABLE_IF(is_like_std_string<
typename std::decay<Arg>::type>::value)>
FMT_CONSTEXPR const char* format_str_quoted(bool add_space, const Arg&) {
return add_space ? " \"{}\"" : "\"{}\"";
template <typename Char, typename OutputIt>
auto write_range_entry(OutputIt out, basic_string_view<Char> str) -> OutputIt {
return write_escaped_string(out, str);
}
FMT_CONSTEXPR const char* format_str_quoted(bool add_space, const char*) {
return add_space ? " \"{}\"" : "\"{}\"";
}
FMT_CONSTEXPR const wchar_t* format_str_quoted(bool add_space, const wchar_t*) {
return add_space ? L" \"{}\"" : L"\"{}\"";
template <typename Char, typename OutputIt, typename T,
FMT_ENABLE_IF(std::is_convertible<T, std_string_view<char>>::value)>
inline auto write_range_entry(OutputIt out, const T& str) -> OutputIt {
auto sv = std_string_view<Char>(str);
return write_range_entry<Char>(out, basic_string_view<Char>(sv));
}
FMT_CONSTEXPR const char* format_str_quoted(bool add_space, const char) {
return add_space ? " '{}'" : "'{}'";
template <typename Char, typename OutputIt, typename Arg,
FMT_ENABLE_IF(std::is_same<Arg, Char>::value)>
OutputIt write_range_entry(OutputIt out, const Arg v) {
return write_escaped_char(out, v);
}
FMT_CONSTEXPR const wchar_t* format_str_quoted(bool add_space, const wchar_t) {
return add_space ? L" '{}'" : L"'{}'";
template <
typename Char, typename OutputIt, typename Arg,
FMT_ENABLE_IF(!is_std_string_like<typename std::decay<Arg>::type>::value &&
!std::is_same<Arg, Char>::value)>
OutputIt write_range_entry(OutputIt out, const Arg& v) {
return write<Char>(out, v);
}
} // namespace detail
template <typename T> struct is_tuple_like {
static FMT_CONSTEXPR_DECL const bool value =
static constexpr const bool value =
detail::is_tuple_like_<T>::value && !detail::is_range_<T>::value;
};
template <typename T, typename C> struct is_tuple_formattable {
static constexpr const bool value =
detail::is_tuple_formattable_<T, C>::value;
};
template <typename TupleT, typename Char>
struct formatter<TupleT, Char, enable_if_t<fmt::is_tuple_like<TupleT>::value>> {
struct formatter<TupleT, Char,
enable_if_t<fmt::is_tuple_like<TupleT>::value &&
fmt::is_tuple_formattable<TupleT, Char>::value>> {
private:
// C++11 generic lambda for format()
basic_string_view<Char> separator_ = detail::string_literal<Char, ',', ' '>{};
basic_string_view<Char> opening_bracket_ =
detail::string_literal<Char, '('>{};
basic_string_view<Char> closing_bracket_ =
detail::string_literal<Char, ')'>{};
// C++11 generic lambda for format().
template <typename FormatContext> struct format_each {
template <typename T> void operator()(const T& v) {
if (i > 0) {
if (formatting.add_prepostfix_space) {
*out++ = ' ';
}
out = detail::copy(formatting.delimiter, out);
}
out = format_to(out,
detail::format_str_quoted(
(formatting.add_delimiter_spaces && i > 0), v),
v);
if (i > 0) out = detail::copy_str<Char>(separator, out);
out = detail::write_range_entry<Char>(out, v);
++i;
}
formatting_tuple<Char>& formatting;
size_t& i;
typename std::add_lvalue_reference<decltype(
std::declval<FormatContext>().out())>::type out;
int i;
typename FormatContext::iterator& out;
basic_string_view<Char> separator;
};
public:
formatting_tuple<Char> formatting;
FMT_CONSTEXPR formatter() {}
template <typename ParseContext>
FMT_CONSTEXPR auto parse(ParseContext& ctx) -> decltype(ctx.begin()) {
return formatting.parse(ctx);
FMT_CONSTEXPR void set_separator(basic_string_view<Char> sep) {
separator_ = sep;
}
template <typename FormatContext = format_context>
auto format(const TupleT& values, FormatContext& ctx) -> decltype(ctx.out()) {
auto out = ctx.out();
size_t i = 0;
detail::copy(formatting.prefix, out);
detail::for_each(values, format_each<FormatContext>{formatting, i, out});
if (formatting.add_prepostfix_space) {
*out++ = ' ';
}
detail::copy(formatting.postfix, out);
return ctx.out();
}
};
template <typename T, typename Char> struct is_range {
static FMT_CONSTEXPR_DECL const bool value =
detail::is_range_<T>::value && !detail::is_like_std_string<T>::value &&
!std::is_convertible<T, std::basic_string<Char>>::value &&
!std::is_constructible<detail::std_string_view<Char>, T>::value;
};
template <typename T, typename Char>
struct formatter<
T, Char,
enable_if_t<fmt::is_range<T, Char>::value
// Workaround a bug in MSVC 2017 and earlier.
#if !FMT_MSC_VER || FMT_MSC_VER >= 1927
&&
(has_formatter<detail::value_type<T>, format_context>::value ||
detail::has_fallback_formatter<detail::value_type<T>,
format_context>::value)
#endif
>> {
formatting_range<Char> formatting;
template <typename ParseContext>
FMT_CONSTEXPR auto parse(ParseContext& ctx) -> decltype(ctx.begin()) {
return formatting.parse(ctx);
FMT_CONSTEXPR void set_brackets(basic_string_view<Char> open,
basic_string_view<Char> close) {
opening_bracket_ = open;
closing_bracket_ = close;
}
template <typename FormatContext>
typename FormatContext::iterator format(const T& values, FormatContext& ctx) {
auto out = detail::copy(formatting.prefix, ctx.out());
size_t i = 0;
auto it = values.begin();
auto end = values.end();
for (; it != end; ++it) {
if (i > 0) {
if (formatting.add_prepostfix_space) *out++ = ' ';
out = detail::copy(formatting.delimiter, out);
}
out = format_to(out,
detail::format_str_quoted(
(formatting.add_delimiter_spaces && i > 0), *it),
*it);
if (++i > formatting.range_length_limit) {
out = format_to(out, " ... <other elements>");
break;
}
}
if (formatting.add_prepostfix_space) *out++ = ' ';
return detail::copy(formatting.postfix, out);
}
};
template <typename Char, typename... T> struct tuple_arg_join : detail::view {
const std::tuple<T...>& tuple;
basic_string_view<Char> sep;
tuple_arg_join(const std::tuple<T...>& t, basic_string_view<Char> s)
: tuple{t}, sep{s} {}
};
template <typename Char, typename... T>
struct formatter<tuple_arg_join<Char, T...>, Char> {
template <typename ParseContext>
FMT_CONSTEXPR auto parse(ParseContext& ctx) -> decltype(ctx.begin()) {
return ctx.begin();
}
template <typename FormatContext = format_context>
auto format(const TupleT& values, FormatContext& ctx) const
-> decltype(ctx.out()) {
auto out = ctx.out();
out = detail::copy_str<Char>(opening_bracket_, out);
detail::for_each(values, format_each<FormatContext>{0, out, separator_});
out = detail::copy_str<Char>(closing_bracket_, out);
return out;
}
};
template <typename T, typename Char> struct is_range {
static constexpr const bool value =
detail::is_range_<T>::value && !detail::is_std_string_like<T>::value &&
!std::is_convertible<T, std::basic_string<Char>>::value &&
!std::is_convertible<T, detail::std_string_view<Char>>::value;
};
namespace detail {
template <typename Context> struct range_mapper {
using mapper = arg_mapper<Context>;
template <typename T,
FMT_ENABLE_IF(has_formatter<remove_cvref_t<T>, Context>::value)>
static auto map(T&& value) -> T&& {
return static_cast<T&&>(value);
}
template <typename T,
FMT_ENABLE_IF(!has_formatter<remove_cvref_t<T>, Context>::value)>
static auto map(T&& value)
-> decltype(mapper().map(static_cast<T&&>(value))) {
return mapper().map(static_cast<T&&>(value));
}
};
template <typename Char, typename Element>
using range_formatter_type = conditional_t<
is_formattable<Element, Char>::value,
formatter<remove_cvref_t<decltype(range_mapper<buffer_context<Char>>{}.map(
std::declval<Element>()))>,
Char>,
fallback_formatter<Element, Char>>;
template <typename R>
using maybe_const_range =
conditional_t<has_const_begin_end<R>::value, const R, R>;
// Workaround a bug in MSVC 2015 and earlier.
#if !FMT_MSC_VERSION || FMT_MSC_VERSION >= 1910
template <typename R, typename Char>
struct is_formattable_delayed
: disjunction<
is_formattable<uncvref_type<maybe_const_range<R>>, Char>,
has_fallback_formatter<uncvref_type<maybe_const_range<R>>, Char>> {};
#endif
} // namespace detail
template <typename T, typename Char, typename Enable = void>
struct range_formatter;
template <typename T, typename Char>
struct range_formatter<
T, Char,
enable_if_t<conjunction<
std::is_same<T, remove_cvref_t<T>>,
disjunction<is_formattable<T, Char>,
detail::has_fallback_formatter<T, Char>>>::value>> {
private:
detail::range_formatter_type<Char, T> underlying_;
bool custom_specs_ = false;
basic_string_view<Char> separator_ = detail::string_literal<Char, ',', ' '>{};
basic_string_view<Char> opening_bracket_ =
detail::string_literal<Char, '['>{};
basic_string_view<Char> closing_bracket_ =
detail::string_literal<Char, ']'>{};
template <class U>
FMT_CONSTEXPR static auto maybe_set_debug_format(U& u, bool set)
-> decltype(u.set_debug_format(set)) {
u.set_debug_format(set);
}
template <class U>
FMT_CONSTEXPR static void maybe_set_debug_format(U&, ...) {}
FMT_CONSTEXPR void maybe_set_debug_format(bool set) {
maybe_set_debug_format(underlying_, set);
}
public:
FMT_CONSTEXPR range_formatter() { maybe_set_debug_format(true); }
FMT_CONSTEXPR auto underlying() -> detail::range_formatter_type<Char, T>& {
return underlying_;
}
FMT_CONSTEXPR void set_separator(basic_string_view<Char> sep) {
separator_ = sep;
}
FMT_CONSTEXPR void set_brackets(basic_string_view<Char> open,
basic_string_view<Char> close) {
opening_bracket_ = open;
closing_bracket_ = close;
}
template <typename ParseContext>
FMT_CONSTEXPR auto parse(ParseContext& ctx) -> decltype(ctx.begin()) {
auto it = ctx.begin();
auto end = ctx.end();
if (it != end && *it == 'n') {
set_brackets({}, {});
++it;
}
if (it == end || *it == '}') return it;
if (*it != ':')
FMT_THROW(format_error("no other top-level range formatters supported"));
maybe_set_debug_format(false);
custom_specs_ = true;
++it;
ctx.advance_to(it);
return underlying_.parse(ctx);
}
template <typename R, class FormatContext>
auto format(R&& range, FormatContext& ctx) const -> decltype(ctx.out()) {
detail::range_mapper<buffer_context<Char>> mapper;
auto out = ctx.out();
out = detail::copy_str<Char>(opening_bracket_, out);
int i = 0;
auto it = detail::range_begin(range);
auto end = detail::range_end(range);
for (; it != end; ++it) {
if (i > 0) out = detail::copy_str<Char>(separator_, out);
;
ctx.advance_to(out);
out = underlying_.format(mapper.map(*it), ctx);
++i;
}
out = detail::copy_str<Char>(closing_bracket_, out);
return out;
}
};
enum class range_format { disabled, map, set, sequence, string, debug_string };
namespace detail {
template <typename T> struct range_format_kind_ {
static constexpr auto value = std::is_same<range_reference_type<T>, T>::value
? range_format::disabled
: is_map<T>::value ? range_format::map
: is_set<T>::value ? range_format::set
: range_format::sequence;
};
template <range_format K, typename R, typename Char, typename Enable = void>
struct range_default_formatter;
template <range_format K>
using range_format_constant = std::integral_constant<range_format, K>;
template <range_format K, typename R, typename Char>
struct range_default_formatter<
K, R, Char,
enable_if_t<(K == range_format::sequence || K == range_format::map ||
K == range_format::set)>> {
using range_type = detail::maybe_const_range<R>;
range_formatter<detail::uncvref_type<range_type>, Char> underlying_;
FMT_CONSTEXPR range_default_formatter() { init(range_format_constant<K>()); }
FMT_CONSTEXPR void init(range_format_constant<range_format::set>) {
underlying_.set_brackets(detail::string_literal<Char, '{'>{},
detail::string_literal<Char, '}'>{});
}
FMT_CONSTEXPR void init(range_format_constant<range_format::map>) {
underlying_.set_brackets(detail::string_literal<Char, '{'>{},
detail::string_literal<Char, '}'>{});
underlying_.underlying().set_brackets({}, {});
underlying_.underlying().set_separator(
detail::string_literal<Char, ':', ' '>{});
}
FMT_CONSTEXPR void init(range_format_constant<range_format::sequence>) {}
template <typename ParseContext>
FMT_CONSTEXPR auto parse(ParseContext& ctx) -> decltype(ctx.begin()) {
return underlying_.parse(ctx);
}
template <typename FormatContext>
typename FormatContext::iterator format(
const tuple_arg_join<Char, T...>& value, FormatContext& ctx) {
return format(value, ctx, detail::make_index_sequence<sizeof...(T)>{});
auto format(range_type& range, FormatContext& ctx) const
-> decltype(ctx.out()) {
return underlying_.format(range, ctx);
}
};
} // namespace detail
template <typename T, typename Char, typename Enable = void>
struct range_format_kind
: conditional_t<
is_range<T, Char>::value, detail::range_format_kind_<T>,
std::integral_constant<range_format, range_format::disabled>> {};
template <typename R, typename Char>
struct formatter<
R, Char,
enable_if_t<conjunction<bool_constant<range_format_kind<R, Char>::value !=
range_format::disabled>
// Workaround a bug in MSVC 2015 and earlier.
#if !FMT_MSC_VERSION || FMT_MSC_VERSION >= 1910
,
detail::is_formattable_delayed<R, Char>
#endif
>::value>>
: detail::range_default_formatter<range_format_kind<R, Char>::value, R,
Char> {
};
template <typename Char, typename... T> struct tuple_join_view : detail::view {
const std::tuple<T...>& tuple;
basic_string_view<Char> sep;
tuple_join_view(const std::tuple<T...>& t, basic_string_view<Char> s)
: tuple(t), sep{s} {}
};
template <typename Char, typename... T>
using tuple_arg_join = tuple_join_view<Char, T...>;
// Define FMT_TUPLE_JOIN_SPECIFIERS to enable experimental format specifiers
// support in tuple_join. It is disabled by default because of issues with
// the dynamic width and precision.
#ifndef FMT_TUPLE_JOIN_SPECIFIERS
# define FMT_TUPLE_JOIN_SPECIFIERS 0
#endif
template <typename Char, typename... T>
struct formatter<tuple_join_view<Char, T...>, Char> {
template <typename ParseContext>
FMT_CONSTEXPR auto parse(ParseContext& ctx) -> decltype(ctx.begin()) {
return do_parse(ctx, std::integral_constant<size_t, sizeof...(T)>());
}
template <typename FormatContext>
auto format(const tuple_join_view<Char, T...>& value,
FormatContext& ctx) const -> typename FormatContext::iterator {
return do_format(value, ctx,
std::integral_constant<size_t, sizeof...(T)>());
}
private:
template <typename FormatContext, size_t... N>
typename FormatContext::iterator format(
const tuple_arg_join<Char, T...>& value, FormatContext& ctx,
detail::index_sequence<N...>) {
return format_args(value, ctx, std::get<N>(value.tuple)...);
std::tuple<formatter<typename std::decay<T>::type, Char>...> formatters_;
template <typename ParseContext>
FMT_CONSTEXPR auto do_parse(ParseContext& ctx,
std::integral_constant<size_t, 0>)
-> decltype(ctx.begin()) {
return ctx.begin();
}
template <typename ParseContext, size_t N>
FMT_CONSTEXPR auto do_parse(ParseContext& ctx,
std::integral_constant<size_t, N>)
-> decltype(ctx.begin()) {
auto end = ctx.begin();
#if FMT_TUPLE_JOIN_SPECIFIERS
end = std::get<sizeof...(T) - N>(formatters_).parse(ctx);
if (N > 1) {
auto end1 = do_parse(ctx, std::integral_constant<size_t, N - 1>());
if (end != end1)
FMT_THROW(format_error("incompatible format specs for tuple elements"));
}
#endif
return end;
}
template <typename FormatContext>
typename FormatContext::iterator format_args(
const tuple_arg_join<Char, T...>&, FormatContext& ctx) {
// NOTE: for compilers that support C++17, this empty function instantiation
// can be replaced with a constexpr branch in the variadic overload.
auto do_format(const tuple_join_view<Char, T...>&, FormatContext& ctx,
std::integral_constant<size_t, 0>) const ->
typename FormatContext::iterator {
return ctx.out();
}
template <typename FormatContext, typename Arg, typename... Args>
typename FormatContext::iterator format_args(
const tuple_arg_join<Char, T...>& value, FormatContext& ctx,
const Arg& arg, const Args&... args) {
using base = formatter<typename std::decay<Arg>::type, Char>;
auto out = ctx.out();
out = base{}.format(arg, ctx);
if (sizeof...(Args) > 0) {
template <typename FormatContext, size_t N>
auto do_format(const tuple_join_view<Char, T...>& value, FormatContext& ctx,
std::integral_constant<size_t, N>) const ->
typename FormatContext::iterator {
auto out = std::get<sizeof...(T) - N>(formatters_)
.format(std::get<sizeof...(T) - N>(value.tuple), ctx);
if (N > 1) {
out = std::copy(value.sep.begin(), value.sep.end(), out);
ctx.advance_to(out);
return format_args(value, ctx, args...);
return do_format(value, ctx, std::integral_constant<size_t, N - 1>());
}
return out;
}
};
FMT_MODULE_EXPORT_BEGIN
/**
\rst
Returns an object that formats `tuple` with elements separated by `sep`.
@ -357,14 +681,15 @@ struct formatter<tuple_arg_join<Char, T...>, Char> {
\endrst
*/
template <typename... T>
FMT_CONSTEXPR tuple_arg_join<char, T...> join(const std::tuple<T...>& tuple,
string_view sep) {
FMT_CONSTEXPR auto join(const std::tuple<T...>& tuple, string_view sep)
-> tuple_join_view<char, T...> {
return {tuple, sep};
}
template <typename... T>
FMT_CONSTEXPR tuple_arg_join<wchar_t, T...> join(const std::tuple<T...>& tuple,
wstring_view sep) {
FMT_CONSTEXPR auto join(const std::tuple<T...>& tuple,
basic_string_view<wchar_t> sep)
-> tuple_join_view<wchar_t, T...> {
return {tuple, sep};
}
@ -380,17 +705,12 @@ FMT_CONSTEXPR tuple_arg_join<wchar_t, T...> join(const std::tuple<T...>& tuple,
\endrst
*/
template <typename T>
arg_join<const T*, const T*, char> join(std::initializer_list<T> list,
string_view sep) {
return join(std::begin(list), std::end(list), sep);
}
template <typename T>
arg_join<const T*, const T*, wchar_t> join(std::initializer_list<T> list,
wstring_view sep) {
auto join(std::initializer_list<T> list, string_view sep)
-> join_view<const T*, const T*> {
return join(std::begin(list), std::end(list), sep);
}
FMT_MODULE_EXPORT_END
FMT_END_NAMESPACE
#endif // FMT_RANGES_H_

2
vendor.yaml
View file

@ -24,3 +24,5 @@ third-party/tiny_gltf:
- "PR #1632 - Ensure stb_image is using UTF-8 on windows"
third-party/SQLiteCpp:
sha: c68f651a10a335fe3c24c31baa64f1d9b97d68be
third-party/fmt:
git: https://github.com/fmtlib/fmt/tree/9.1.0