From b0acce3b858715a04fba154a23c6ea5fac083a74 Mon Sep 17 00:00:00 2001 From: Mel Date: Tue, 22 Jul 2025 19:29:21 +0200 Subject: Add option to test suite to auto-adjust expected output in definition Signed-off-by: Mel --- boot/tests/test.c | 128 +++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 113 insertions(+), 15 deletions(-) (limited to 'boot/tests/test.c') diff --git a/boot/tests/test.c b/boot/tests/test.c index 56d1a77..b8a535d 100644 --- a/boot/tests/test.c +++ b/boot/tests/test.c @@ -83,11 +83,47 @@ usage(void) { fprintf( stderr, - "usage: test [ ...]\n" + "usage: test [options] [ ...]\n" "targets:\n"); for (uint i = 0; i < ARRAY_SIZE(known_targets); ++i) { fprintf(stderr, " %s\n", known_targets[i].name); } + + fprintf( + stderr, + "\noptions:\n" + " -a, --adjust\t\t\t overwrite expected output with actual output for failed tests\n" + " -?, --help\t\t\t print this help message\n"); +} + +enum Option +{ + OPTION_NONE = 0, + OPTION_ADJUST = 1 << 0, // overwrite expected output with actual output for failed tests + OPTION_HELP = 1 << 1, // print help message +}; + +enum Option +options_from_arguments(int argc, const ascii** argv) +{ + enum Option options = OPTION_NONE; + for (uint ai = 1; ai < argc; ++ai) { + const ascii* arg = argv[ai]; + + if (arg[0] != '-') continue; // skip non-option arguments + + if (strcmp(arg, "-a") == 0 || strcmp(arg, "--adjust") == 0) { + options |= OPTION_ADJUST; + } else if (strcmp(arg, "-?") == 0 || strcmp(arg, "--help") == 0) { + options |= OPTION_HELP; + } else { + fprintf(stderr, "unknown option '%s'\n", arg); + usage(); + exit(EXIT_FAILURE); + } + } + + return options; } enum Test_Target @@ -97,6 +133,8 @@ targets_from_arguments(int argc, const ascii** argv) for (uint ai = 1; ai < argc; ++ai) { const ascii* arg = argv[ai]; + if (arg[0] == '-') continue; // skip option arguments + bool found = false; for (uint ti = 0; ti < ARRAY_SIZE(known_targets); ++ti) { struct Known_Target kt = known_targets[ti]; @@ -192,6 +230,29 @@ string_from_file(const ascii* path) return string_from_static_c_string(file_data); } +void +string_to_file(struct String str, const ascii* path) +{ + check(!string_is_empty(str), "string to write to file is empty"); + check(path, "path to write string to is nil"); + + FILE* file = fopen(path, "w"); + if (!file) { + fprintf(stderr, "i couldn't open file '%s' for writing: %s\n", path, strerror(errno)); + exit(EXIT_FAILURE); + } + + size_t written = fwrite(str.data, 1, str.length, file); + if (written < str.length) { + fprintf( + stderr, "i couldn't write complete string to file '%s': %s\n", path, strerror(errno)); + fclose(file); + exit(EXIT_FAILURE); + } + + fclose(file); +} + #define MAX_INPUT_LENGTH 8192 #define MAX_PATH_LENGTH 256 @@ -343,8 +404,32 @@ run_command(const ascii* command) }; } +void +write_failed_output_artifact(const ascii* test_definition_path, struct String output) +{ + struct Temporary_File wrong_output_file = temporary_file_from_string(output); + fprintf(stderr, "'%s': completed with incorrect output. written to %s.\n", test_definition_path, + wrong_output_file.path); + free(wrong_output_file.path); // free but don't delete. +} + +void +adjust_failed_definition_output(const ascii* test_definition_path, struct String output) +{ + struct String test_file = string_from_file(test_definition_path); + struct String test_without_output = string_strip_from_pattern(test_file, OUTPUT_MARK); + + struct String new_test_content = string_concatenate( + ARG_STRING, test_without_output, ARG_ASCII, OUTPUT_MARK "\n\n", ARG_STRING, output, + ARG_END); + + string_to_file(new_test_content, test_definition_path); + fprintf(stderr, "'%s': adjusted test definition output to match actual output.\n", + test_definition_path); +} + bool -run_test(const ascii* test_definition_path, const ascii* base_command) +run_test(const ascii* test_definition_path, const ascii* base_command, bool adjust) { struct String test_file = string_from_file(test_definition_path); @@ -371,10 +456,11 @@ run_test(const ascii* test_definition_path, const ascii* base_command) struct String output = string_trim(result.output); if (!string_equal(output, expected_output)) { - struct Temporary_File wrong_output_file = temporary_file_from_string(output); - fprintf(stderr, "'%s': completed with incorrect output. written to %s.\n", - test_definition_path, wrong_output_file.path); - free(wrong_output_file.path); // free but don't delete. + if (adjust) + adjust_failed_definition_output(test_definition_path, output); + else + write_failed_output_artifact(test_definition_path, output); + success = false; goto end; } @@ -387,7 +473,7 @@ end: } struct Result_Summary -tests(const ascii* base_path, const ascii* base_command) +tests(const ascii* base_path, const ascii* base_command, bool adjust) { struct Result_Summary summary = { 0 }; @@ -412,7 +498,7 @@ tests(const ascii* base_path, const ascii* base_command) if (!is_viable_test_definition_file(test_definition_path)) continue; - bool succeeded = run_test(test_definition_path, base_command); + bool succeeded = run_test(test_definition_path, base_command, adjust); summary.total++; if (succeeded) { @@ -426,26 +512,38 @@ tests(const ascii* base_path, const ascii* base_command) } struct Result_Summary -lex_tests(void) +lex_tests(bool adjust) { const ascii* base_path = "./boot/tests/lex/"; const ascii* base_command = "./build/catboot --test-lex"; - return tests(base_path, base_command); + return tests(base_path, base_command, adjust); } struct Result_Summary -parse_tests(void) +parse_tests(bool adjust) { const ascii* base_path = "./boot/tests/parse/"; const ascii* base_command = "./build/catboot --test-parse"; - return tests(base_path, base_command); + return tests(base_path, base_command, adjust); } int main(int argc, const ascii** argv) { + enum Option options = options_from_arguments(argc, argv); + if (options & OPTION_HELP) { + usage(); + return EXIT_SUCCESS; + } + + bool adjust = false; + if (options & OPTION_ADJUST) { + fprintf(stderr, "--adjust option will overwrite failed test outputs\n"); + adjust = true; + } + enum Test_Target targets = targets_from_arguments(argc, argv); if (targets == TARGET_NONE) { usage(); @@ -466,11 +564,11 @@ main(int argc, const ascii** argv) struct Result_Summary summary = { 0 }; switch (target.target) { case TARGET_LEX: - summary = lex_tests(); + summary = lex_tests(adjust); break; case TARGET_PARSE: - summary = parse_tests(); + summary = parse_tests(adjust); break; default: @@ -508,4 +606,4 @@ main(int argc, const ascii** argv) result_summary_print(full_summary); return EXIT_SUCCESS; -} \ No newline at end of file +} -- cgit 1.4.1