2023-04-24 23:46:55 -04:00
|
|
|
// TODO - eventually replace our `goalc` tests with this setup
|
|
|
|
// A simple test runner framework for debugging / iterating on the formatter
|
|
|
|
// Tests are defined in files as such:
|
|
|
|
|
|
|
|
/*
|
|
|
|
===
|
|
|
|
TEST NAME
|
|
|
|
===
|
|
|
|
|
|
|
|
INPUT
|
|
|
|
|
|
|
|
---
|
|
|
|
|
|
|
|
EXPECTED OUTPUT
|
|
|
|
|
|
|
|
*/
|
|
|
|
|
|
|
|
// Test files can contain multiple tests, upon running we will recurse a directory
|
|
|
|
// looking for any `.test` files and run them through the framework
|
|
|
|
//
|
|
|
|
// Any differences will be diff'd and displayed
|
|
|
|
|
|
|
|
#include "common/formatter/formatter.h"
|
|
|
|
#include "common/util/FileUtil.h"
|
|
|
|
#include "common/util/string_util.h"
|
|
|
|
|
|
|
|
#include "gtest/gtest.h"
|
|
|
|
|
2024-03-05 22:11:52 -05:00
|
|
|
#include "fmt/color.h"
|
|
|
|
#include "fmt/core.h"
|
2023-04-24 23:46:55 -04:00
|
|
|
|
|
|
|
struct TestDefinition {
|
|
|
|
std::string name;
|
|
|
|
std::string input;
|
|
|
|
std::string output;
|
|
|
|
};
|
|
|
|
|
2023-06-06 20:34:50 -04:00
|
|
|
std::vector<TestDefinition> get_test_definitions(const fs::path& file_path) {
|
2023-06-18 17:19:35 -04:00
|
|
|
std::vector<TestDefinition> tests;
|
2023-04-24 23:46:55 -04:00
|
|
|
// Read in the file, and run the test
|
|
|
|
const auto contents = str_util::split(file_util::read_text_file(file_path));
|
2023-06-18 17:19:35 -04:00
|
|
|
if (contents.empty() || (contents.size() == 1 && contents.at(0).empty())) {
|
|
|
|
return tests;
|
|
|
|
}
|
2023-04-24 23:46:55 -04:00
|
|
|
TestDefinition curr_test;
|
2023-04-29 16:13:57 -04:00
|
|
|
size_t i = 0;
|
2023-04-24 23:46:55 -04:00
|
|
|
while (i < contents.size()) {
|
|
|
|
const auto& line = contents.at(i);
|
|
|
|
if (line == "===") {
|
|
|
|
curr_test = TestDefinition();
|
|
|
|
curr_test.name = contents.at(i + 1);
|
|
|
|
i += 3;
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
// Parse the input and output
|
|
|
|
if (!curr_test.name.empty() && line.empty()) {
|
|
|
|
i++;
|
|
|
|
while (true) {
|
|
|
|
if (contents.at(i) == "---") {
|
|
|
|
i++;
|
|
|
|
curr_test.input = str_util::trim(curr_test.input);
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
curr_test.input += contents.at(i) + "\n";
|
|
|
|
i++;
|
|
|
|
}
|
|
|
|
i++;
|
|
|
|
while (true) {
|
|
|
|
if (i == contents.size() || contents.at(i) == "===") {
|
|
|
|
curr_test.output = str_util::trim(curr_test.output);
|
|
|
|
tests.push_back(curr_test);
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
curr_test.output += contents.at(i) + "\n";
|
|
|
|
i++;
|
|
|
|
}
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
}
|
2023-06-06 20:34:50 -04:00
|
|
|
return tests;
|
|
|
|
}
|
|
|
|
|
|
|
|
bool has_important_tests(const fs::path& file_path) {
|
|
|
|
const auto& tests = get_test_definitions(file_path);
|
|
|
|
for (const auto& test : tests) {
|
|
|
|
if (str_util::starts_with(test.name, "!")) {
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
2023-10-20 21:24:31 -04:00
|
|
|
// TODO - consider adding a test that auto-formats all of goal_src (there should be no errors)
|
|
|
|
|
2023-06-06 20:34:50 -04:00
|
|
|
bool run_tests(const fs::path& file_path, const bool only_important_tests) {
|
|
|
|
const auto& tests = get_test_definitions(file_path);
|
2023-04-24 23:46:55 -04:00
|
|
|
// Run the tests, report successes and failures
|
|
|
|
bool test_failed = false;
|
2023-06-04 13:19:29 -04:00
|
|
|
fmt::print("{}:\n", fmt::styled(file_util::base_name(file_path.string()),
|
|
|
|
fmt::emphasis::bold | fg(fmt::color::cyan)));
|
2023-04-24 23:46:55 -04:00
|
|
|
for (const auto& test : tests) {
|
2023-06-04 13:19:29 -04:00
|
|
|
if (only_important_tests && !str_util::starts_with(test.name, "!")) {
|
|
|
|
continue;
|
|
|
|
}
|
2024-02-27 20:12:44 -05:00
|
|
|
if (str_util::contains(test.output, "TODO")) {
|
|
|
|
// ignore the output
|
|
|
|
fmt::print(" ⚠️ - {}\n", test.name);
|
|
|
|
continue;
|
|
|
|
}
|
2023-04-24 23:46:55 -04:00
|
|
|
const auto formatted_result = formatter::format_code(test.input);
|
2023-10-20 21:24:31 -04:00
|
|
|
if (formatted_result && str_util::starts_with(test.name, "!?")) {
|
|
|
|
fmt::print("FORMATTED RESULT:\n\n{}\n\n", formatted_result.value());
|
|
|
|
}
|
2023-05-28 13:22:00 -04:00
|
|
|
if (!formatted_result) {
|
|
|
|
// Unable to parse, was that expected?
|
|
|
|
if (test.output == "__THROWS__") {
|
|
|
|
fmt::print(" ✅ - {}\n", test.name);
|
|
|
|
} else {
|
|
|
|
fmt::print(" ❌ - {}\n", test.name);
|
|
|
|
fmt::print("Unable to Format\n");
|
|
|
|
test_failed = true;
|
|
|
|
}
|
|
|
|
} else if (formatted_result != test.output) {
|
2023-04-24 23:46:55 -04:00
|
|
|
fmt::print(" ❌ - {}\n", test.name);
|
2023-05-28 13:22:00 -04:00
|
|
|
fmt::print("{}\n", str_util::diff(test.output, formatted_result.value()));
|
2023-04-24 23:46:55 -04:00
|
|
|
test_failed = true;
|
|
|
|
} else {
|
|
|
|
fmt::print(" ✅ - {}\n", test.name);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return test_failed;
|
|
|
|
}
|
|
|
|
|
2023-06-06 20:34:50 -04:00
|
|
|
bool find_and_run_tests() {
|
2023-10-20 21:24:31 -04:00
|
|
|
// TODO - fails when it finds no tests
|
2023-08-05 15:23:09 -04:00
|
|
|
try {
|
|
|
|
// Enumerate test files
|
|
|
|
const auto test_files = file_util::find_files_recursively(
|
2023-08-08 20:53:16 -04:00
|
|
|
file_util::get_file_path({"test/common/formatter/corpus"}), std::regex("^.*\\.test.gc$"));
|
2023-08-05 15:23:09 -04:00
|
|
|
bool failed = false;
|
|
|
|
// First do a pass to see if any tests are meant to be prioritized for debugging
|
|
|
|
bool only_important_tests = false;
|
|
|
|
for (const auto& file : test_files) {
|
|
|
|
only_important_tests = has_important_tests(file);
|
|
|
|
if (only_important_tests) {
|
|
|
|
break;
|
|
|
|
}
|
2023-06-06 20:34:50 -04:00
|
|
|
}
|
2023-08-05 15:23:09 -04:00
|
|
|
for (const auto& file : test_files) {
|
|
|
|
// don't fail fast, but any failure means we return false
|
|
|
|
if (failed) {
|
|
|
|
run_tests(file, only_important_tests);
|
|
|
|
} else {
|
|
|
|
failed = run_tests(file, only_important_tests);
|
|
|
|
}
|
2023-06-04 13:19:29 -04:00
|
|
|
}
|
2023-08-05 15:23:09 -04:00
|
|
|
return !failed;
|
|
|
|
} catch (std::exception& e) {
|
|
|
|
return false;
|
2023-04-24 23:46:55 -04:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
TEST(Formatter, FormatterTests) {
|
2023-06-06 20:34:50 -04:00
|
|
|
EXPECT_TRUE(find_and_run_tests());
|
2023-04-24 23:46:55 -04:00
|
|
|
}
|