Add option to test suite to auto-adjust expected output in definition

Signed-off-by: Mel <mel@rnrd.eu>

1 files changed, 113 insertions, 15 deletions

diff --git a/boot/tests/test.c b/boot/tests/test.c
index 56d1a77..b8a535d 100644
--- a/boot/tests/test.c
+++ b/boot/tests/test.c
@@ -83,11 +83,47 @@ usage(void)
 {
     fprintf(
         stderr,
-        "usage: test [<target> ...]\n"
+        "usage: test [options] [<target> ...]\n"
         "targets:\n");
     for (uint i = 0; i < ARRAY_SIZE(known_targets); ++i) {
         fprintf(stderr, "  %s\n", known_targets[i].name);
     }
+
+    fprintf(
+        stderr,
+        "\noptions:\n"
+        "  -a, --adjust\t\t\t overwrite expected output with actual output for failed tests\n"
+        "  -?, --help\t\t\t print this help message\n");
+}
+
+enum Option
+{
+    OPTION_NONE = 0,
+    OPTION_ADJUST = 1 << 0, // overwrite expected output with actual output for failed tests
+    OPTION_HELP = 1 << 1,   // print help message
+};
+
+enum Option
+options_from_arguments(int argc, const ascii** argv)
+{
+    enum Option options = OPTION_NONE;
+    for (uint ai = 1; ai < argc; ++ai) {
+        const ascii* arg = argv[ai];
+
+        if (arg[0] != '-') continue; // skip non-option arguments
+
+        if (strcmp(arg, "-a") == 0 || strcmp(arg, "--adjust") == 0) {
+            options |= OPTION_ADJUST;
+        } else if (strcmp(arg, "-?") == 0 || strcmp(arg, "--help") == 0) {
+            options |= OPTION_HELP;
+        } else {
+            fprintf(stderr, "unknown option '%s'\n", arg);
+            usage();
+            exit(EXIT_FAILURE);
+        }
+    }
+
+    return options;
 }
 
 enum Test_Target
@@ -97,6 +133,8 @@ targets_from_arguments(int argc, const ascii** argv)
     for (uint ai = 1; ai < argc; ++ai) {
         const ascii* arg = argv[ai];
 
+        if (arg[0] == '-') continue; // skip option arguments
+
         bool found = false;
         for (uint ti = 0; ti < ARRAY_SIZE(known_targets); ++ti) {
             struct Known_Target kt = known_targets[ti];
@@ -192,6 +230,29 @@ string_from_file(const ascii* path)
     return string_from_static_c_string(file_data);
 }
 
+void
+string_to_file(struct String str, const ascii* path)
+{
+    check(!string_is_empty(str), "string to write to file is empty");
+    check(path, "path to write string to is nil");
+
+    FILE* file = fopen(path, "w");
+    if (!file) {
+        fprintf(stderr, "i couldn't open file '%s' for writing: %s\n", path, strerror(errno));
+        exit(EXIT_FAILURE);
+    }
+
+    size_t written = fwrite(str.data, 1, str.length, file);
+    if (written < str.length) {
+        fprintf(
+            stderr, "i couldn't write complete string to file '%s': %s\n", path, strerror(errno));
+        fclose(file);
+        exit(EXIT_FAILURE);
+    }
+
+    fclose(file);
+}
+
 #define MAX_INPUT_LENGTH 8192
 #define MAX_PATH_LENGTH 256
 
@@ -343,8 +404,32 @@ run_command(const ascii* command)
     };
 }
 
+void
+write_failed_output_artifact(const ascii* test_definition_path, struct String output)
+{
+    struct Temporary_File wrong_output_file = temporary_file_from_string(output);
+    fprintf(stderr, "'%s': completed with incorrect output. written to %s.\n", test_definition_path,
+            wrong_output_file.path);
+    free(wrong_output_file.path); // free but don't delete.
+}
+
+void
+adjust_failed_definition_output(const ascii* test_definition_path, struct String output)
+{
+    struct String test_file = string_from_file(test_definition_path);
+    struct String test_without_output = string_strip_from_pattern(test_file, OUTPUT_MARK);
+
+    struct String new_test_content = string_concatenate(
+        ARG_STRING, test_without_output, ARG_ASCII, OUTPUT_MARK "\n\n", ARG_STRING, output,
+        ARG_END);
+
+    string_to_file(new_test_content, test_definition_path);
+    fprintf(stderr, "'%s': adjusted test definition output to match actual output.\n",
+            test_definition_path);
+}
+
 bool
-run_test(const ascii* test_definition_path, const ascii* base_command)
+run_test(const ascii* test_definition_path, const ascii* base_command, bool adjust)
 {
     struct String test_file = string_from_file(test_definition_path);
 
@@ -371,10 +456,11 @@ run_test(const ascii* test_definition_path, const ascii* base_command)
 
     struct String output = string_trim(result.output);
     if (!string_equal(output, expected_output)) {
-        struct Temporary_File wrong_output_file = temporary_file_from_string(output);
-        fprintf(stderr, "'%s': completed with incorrect output. written to %s.\n",
-                test_definition_path, wrong_output_file.path);
-        free(wrong_output_file.path); // free but don't delete.
+        if (adjust)
+            adjust_failed_definition_output(test_definition_path, output);
+        else
+            write_failed_output_artifact(test_definition_path, output);
+
         success = false;
         goto end;
     }
@@ -387,7 +473,7 @@ end:
 }
 
 struct Result_Summary
-tests(const ascii* base_path, const ascii* base_command)
+tests(const ascii* base_path, const ascii* base_command, bool adjust)
 {
     struct Result_Summary summary = { 0 };
 
@@ -412,7 +498,7 @@ tests(const ascii* base_path, const ascii* base_command)
 
         if (!is_viable_test_definition_file(test_definition_path)) continue;
 
-        bool succeeded = run_test(test_definition_path, base_command);
+        bool succeeded = run_test(test_definition_path, base_command, adjust);
 
         summary.total++;
         if (succeeded) {
@@ -426,26 +512,38 @@ tests(const ascii* base_path, const ascii* base_command)
 }
 
 struct Result_Summary
-lex_tests(void)
+lex_tests(bool adjust)
 {
     const ascii* base_path = "./boot/tests/lex/";
     const ascii* base_command = "./build/catboot --test-lex";
 
-    return tests(base_path, base_command);
+    return tests(base_path, base_command, adjust);
 }
 
 struct Result_Summary
-parse_tests(void)
+parse_tests(bool adjust)
 {
     const ascii* base_path = "./boot/tests/parse/";
     const ascii* base_command = "./build/catboot --test-parse";
 
-    return tests(base_path, base_command);
+    return tests(base_path, base_command, adjust);
 }
 
 int
 main(int argc, const ascii** argv)
 {
+    enum Option options = options_from_arguments(argc, argv);
+    if (options & OPTION_HELP) {
+        usage();
+        return EXIT_SUCCESS;
+    }
+
+    bool adjust = false;
+    if (options & OPTION_ADJUST) {
+        fprintf(stderr, "--adjust option will overwrite failed test outputs\n");
+        adjust = true;
+    }
+
     enum Test_Target targets = targets_from_arguments(argc, argv);
     if (targets == TARGET_NONE) {
         usage();
@@ -466,11 +564,11 @@ main(int argc, const ascii** argv)
         struct Result_Summary summary = { 0 };
         switch (target.target) {
         case TARGET_LEX:
-            summary = lex_tests();
+            summary = lex_tests(adjust);
             break;
 
         case TARGET_PARSE:
-            summary = parse_tests();
+            summary = parse_tests(adjust);
             break;
 
         default:
@@ -508,4 +606,4 @@ main(int argc, const ascii** argv)
     result_summary_print(full_summary);
 
     return EXIT_SUCCESS;
-}
\ No newline at end of file
+}