Test suite runner prototype for `catboot`

Signed-off-by: Mel <mel@rnrd.eu>

3 files changed, 534 insertions, 0 deletions

diff --git a/boot/tests/lex/basic.cskt b/boot/tests/lex/basic.cskt
new file mode 100644
index 0000000..7380498
--- /dev/null
+++ b/boot/tests/lex/basic.cskt
@@ -0,0 +1,12 @@
+very simple test to check if
+the lexer works at all. :)
+
+<<<
+
+main = fun () int {
+  return 1 + 2
+}
+
+>>>
+
+NAME ASSIGN WORD_FUN ROUND_OPEN ROUND_CLOSE NAME CURLY_OPEN NEWLINE WORD_RETURN LITERAL_INTEGER PLUS LITERAL_INTEGER NEWLINE CURLY_CLOSE END_OF_FILE
diff --git a/boot/tests/parse/basic.cskt b/boot/tests/parse/basic.cskt
new file mode 100644
index 0000000..fcc6af0
--- /dev/null
+++ b/boot/tests/parse/basic.cskt
@@ -0,0 +1,14 @@
+very simple test to check if
+the parser works at all. :)
+
+<<<
+
+main = fun () int {
+  return 1 + 2
+}
+
+>>>
+
+(expr (binary = (expr (name main)) (expr (function  (returns (type name int)) (block
+	(return (expr (binary + (expr 1) (expr 2))))
+)))))
\ No newline at end of file
diff --git a/boot/tests/test.c b/boot/tests/test.c
new file mode 100644
index 0000000..5c0eb90
--- /dev/null
+++ b/boot/tests/test.c
@@ -0,0 +1,508 @@
+/*
+ * a test suite runner for the catskill bootstrap compiler.
+ * valid test files are situated in one of the suite folders in `tests`,
+ * and demarcate the input beginning with `<<<` and expected output beginning
+ * with `>>>`.
+ * NOTE: currently this is basically a rough prototype, please handle with care!
+ *
+ * Copyright (c) 2025, Mel G. <mel@rnrd.eu>
+ *
+ * SPDX-License-Identifier: MPL-2.0
+ */
+
+#define _GNU_SOURCE
+
+#include <dirent.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <sys/mman.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+
+#include "../common.c"
+
+#define MAX_TESTS 256
+
+#define INPUT_MARK "<<<"
+#define OUTPUT_MARK ">>>"
+
+enum Result
+{
+    RESULT_NONE,
+    RESULT_OK,
+    RESULT_FAIL,
+};
+
+struct Result_Summary
+{
+    uint total;
+    uint ok;
+    uint fail;
+};
+
+struct Result_Summary
+result_summary_merge(struct Result_Summary a, struct Result_Summary b)
+{
+    return (struct Result_Summary){
+        .total = a.total + b.total,
+        .ok = a.ok + b.ok,
+        .fail = a.fail + b.fail,
+    };
+}
+
+void
+result_summary_print(struct Result_Summary summary)
+{
+    fprintf(stderr, "[%lu/%lu/%lu]\n", summary.ok, summary.fail, summary.total);
+}
+
+enum Test_Target
+{
+    TARGET_NONE = 0,
+    TARGET_LEX = 1 << 0,
+    TARGET_PARSE = 1 << 1,
+
+    TARGET_ALL = TARGET_LEX | TARGET_PARSE,
+};
+
+struct Known_Target
+{
+    const ascii* name;
+    enum Test_Target target;
+} known_targets[] = {
+    { "lex", TARGET_LEX },
+    { "parse", TARGET_PARSE },
+
+    { "all", TARGET_ALL },
+};
+
+void
+usage(void)
+{
+    fprintf(
+        stderr,
+        "usage: test [<target> ...]\n"
+        "targets:\n");
+    for (uint i = 0; i < ARRAY_SIZE(known_targets); ++i) {
+        fprintf(stderr, "  %s\n", known_targets[i].name);
+    }
+}
+
+enum Test_Target
+targets_from_arguments(int argc, const ascii** argv)
+{
+    enum Test_Target target = TARGET_NONE;
+    for (uint ai = 1; ai < argc; ++ai) {
+        const ascii* arg = argv[ai];
+
+        bool found = false;
+        for (uint ti = 0; ti < ARRAY_SIZE(known_targets); ++ti) {
+            struct Known_Target kt = known_targets[ti];
+            if (strcmp(arg, kt.name) == 0) {
+                found = true;
+                target |= kt.target;
+                break;
+            }
+        }
+
+        // TODO: parameterize `failure`
+        if (!found) failure("unknown test target");
+    }
+
+    return target;
+}
+
+bool
+is_viable_test_definition_file(const ascii* path)
+{
+    struct stat st;
+    if (stat(path, &st) != 0) {
+        fprintf(stderr, "couldn't stat file '%s': %s\n", path, strerror(errno));
+        return false;
+    }
+
+    // is it a regular file?
+    if (!S_ISREG(st.st_mode)) {
+        fprintf(stderr, "file '%s' is not a regular file.\n", path);
+        return false;
+    }
+
+    // can i read it?
+    if (!(st.st_mode & S_IRUSR)) {
+        fprintf(stderr, "i couldn't read file '%s'.\n", path);
+        return false;
+    }
+
+    return true;
+}
+
+#define MAX_OUTPUT_LENGTH 65536
+
+// NOTE: below are multiple `String` extension functions,
+// some of which should definitely be integrated into the common
+// library.
+
+bool
+string_equal(struct String a, struct String b)
+{
+    return strcmp(a.data, b.data) == 0;
+}
+
+struct String
+string_from_pipe(FILE* pipe)
+{
+    // we never free this memory, but tests shouldn't be long-running enough
+    // for this to be a problem.
+    uint bytes_read;
+    ascii* buffer = malloc(MAX_OUTPUT_LENGTH + 1);
+    if (!buffer) failure("failed to allocate memory for command output\n");
+
+    while ((bytes_read = fread(buffer, 1, MAX_OUTPUT_LENGTH, pipe)) > 0) {
+        if (bytes_read == MAX_OUTPUT_LENGTH) {
+            fprintf(
+                stderr, "i truncated the command output to %d bytes, something is probably wrong\n",
+                MAX_OUTPUT_LENGTH);
+            break;
+        }
+    }
+
+    return string_from_static_c_string(buffer);
+}
+
+// map file into string.
+// TODO: merge this with the `read_file` in `boot/catboot.c` into the common lib.
+struct String
+string_from_file(const ascii* path)
+{
+    struct stat stat_info;
+    if (stat(path, &stat_info) == -1) failure("i couldn't open that file, sorry :(");
+
+    check(stat_info.st_size > 0, "file is empty, i can't map an empty file");
+
+    const int32 file_descriptor = open(path, O_RDONLY);
+    check(file_descriptor != -1, "i couldn't open that file, sorry :(");
+
+    const flags mmap_prot = PROT_READ;
+    const flags mmap_flags = MAP_PRIVATE;
+    const unknown* file_data =
+        mmap(nil, stat_info.st_size, mmap_prot, mmap_flags, file_descriptor, 0);
+
+    return string_from_static_c_string(file_data);
+}
+
+#define MAX_INPUT_LENGTH 8192
+#define MAX_PATH_LENGTH 256
+
+struct String
+string_strip_until_pattern(struct String str, const ascii* pattern)
+{
+    check(pattern, "pattern given to `string_strip_until_pattern` is nil");
+
+    uint pattern_length = strlen(pattern);
+    ascii* pattern_start = strstr(str.data, pattern);
+    if (!pattern_start) return str;
+
+    return string_from_c_string(pattern_start + pattern_length);
+}
+
+struct String
+string_strip_from_pattern(struct String str, const ascii* pattern)
+{
+    check(pattern, "pattern given to `string_strip_after_pattern` is nil");
+
+    ascii* pattern_start = strstr(str.data, pattern);
+    if (!pattern_start) return str;
+
+    uint new_string_length = pattern_start - str.data;
+    check(new_string_length < MAX_INPUT_LENGTH - 1, "`string_strip_after_pattern` input too long");
+
+    ascii buffer[MAX_INPUT_LENGTH];
+    memcpy(buffer, str.data, new_string_length);
+    buffer[new_string_length] = '\0';
+
+    return string_from_c_string(buffer);
+}
+
+struct String
+string_trim_left(struct String str)
+{
+    if (string_is_empty(str)) return str;
+
+    ascii* c;
+    for (c = str.data; *c == ' ' || *c == '\n'; c++) {}
+
+    return string_from_static_c_string(c);
+}
+
+struct String
+string_trim_right(struct String str)
+{
+    if (string_is_empty(str)) return str;
+
+    ascii* c;
+    for (c = &str.data[string_length(str) - 1]; (*c == ' ' || *c == '\n') && c >= str.data; c--) {}
+
+    uint length = c - str.data + 1;
+    return string_new(str.data, length);
+}
+
+struct String
+string_trim(struct String str)
+{
+    return string_trim_right(string_trim_left(str));
+}
+
+struct Temporary_File
+{
+    ascii* path;
+};
+
+struct Temporary_File
+temporary_file_from_string(struct String str)
+{
+    const ascii* name_template = "/tmp/catboot_test_artifact_XXXXXX";
+
+    struct Temporary_File file = {
+        .path = malloc(MAX_PATH_LENGTH),
+    };
+    check(file.path, "failed to allocate file name buffer memory");
+    strcpy(file.path, name_template);
+
+    integer fd = mkstemp(file.path);
+    if (fd < 0) {
+        // TODO: `failure` and `check` functions that print errno content
+        fprintf(stderr, "i couldn't create a temporary input file: %s\n", strerror(errno));
+        exit(EXIT_FAILURE);
+    }
+
+    integer write_result = write(fd, str.data, str.length);
+    if (write_result < 0) {
+        fprintf(stderr, "i couldn't write to temporary input file: %s\n", strerror(errno));
+        exit(EXIT_FAILURE);
+    }
+
+    check(write_result == str.length, "couldn't write complete input to temporary file");
+
+    close(fd);
+
+    return file;
+}
+
+void
+temporary_file_delete(struct Temporary_File file)
+{
+    if (unlink(file.path) < 0)
+        fprintf(stderr, "warning, i failed removing temporary file at '%s'... weird...", file.path);
+
+    free(file.path);
+}
+
+struct Command_Result
+{
+    bool did_succeed;
+    struct String output;
+};
+
+struct Command_Result
+run_command(const ascii* command)
+{
+    FILE* command_pipe = popen(command, "r");
+    if (!command_pipe) {
+        fprintf(stderr, "i couldn't run '%s': %s\n", command, strerror(errno));
+        return (struct Command_Result){ .did_succeed = false };
+    }
+
+    struct String output = string_trim(string_from_pipe(command_pipe));
+
+    int32 status = pclose(command_pipe);
+    if (status == -1) {
+        fprintf(stderr, "i couldn't finish running '%s': %s\n", command, strerror(errno));
+        return (struct Command_Result){ .did_succeed = false };
+    }
+
+    // i think (??) `pclose` just runs `waitpid` and returns it's status directly,
+    // so we should probably check it with the same macros as `waitpid`, but i'm not 100% sure.
+    bool did_succeed = true;
+    if (WIFEXITED(status)) {
+        integer exit_code = WEXITSTATUS(status);
+        if (exit_code != 0) {
+            fprintf(stderr, "command '%s' exited with non-zero code (%ld)\n", command, exit_code);
+            did_succeed = false;
+        }
+    } else if (WIFSIGNALED(status)) {
+        integer signal_number = WTERMSIG(status);
+        fprintf(stderr, "command '%s' was terminated by signal (%ld)\n", command, signal_number);
+        did_succeed = false;
+    }
+
+    return (struct Command_Result){
+        .did_succeed = did_succeed,
+        .output = output,
+    };
+}
+
+bool
+run_test(const ascii* test_definition_path, const ascii* base_command)
+{
+    struct String test_file = string_from_file(test_definition_path);
+
+    struct String test_content = string_strip_until_pattern(test_file, INPUT_MARK);
+
+    struct String input = string_strip_from_pattern(test_content, OUTPUT_MARK);
+    struct String expected_output = string_strip_until_pattern(test_content, OUTPUT_MARK);
+
+    input = string_trim(input), expected_output = string_trim(expected_output);
+
+    struct Temporary_File input_file = temporary_file_from_string(input);
+
+    ascii command[MAX_PATH_LENGTH + 64];
+    snprintf(command, sizeof(command), "%s %s", base_command, input_file.path);
+
+    struct Command_Result result = run_command(command);
+
+    bool success = true;
+    if (!result.did_succeed) {
+        fprintf(stderr, "'%s': did not complete successfully.\n", test_definition_path);
+        success = false;
+        goto end;
+    }
+
+    struct String output = string_trim(result.output);
+    if (!string_equal(output, expected_output)) {
+        struct Temporary_File wrong_output_file = temporary_file_from_string(output);
+        fprintf(stderr, "'%s': completed with incorrect output. written to %s.\n",
+                test_definition_path, wrong_output_file.path);
+        temporary_file_delete(wrong_output_file);
+        success = false;
+    }
+
+end:
+    temporary_file_delete(input_file);
+    return success;
+}
+
+struct Result_Summary
+tests(const ascii* base_path, const ascii* base_command)
+{
+    struct Result_Summary summary = { 0 };
+
+    // iterate through all test files in the base path
+    DIR* dir = opendir(base_path);
+    if (!dir) {
+        fprintf(stderr, "failed to open directory '%s': %s\n", base_path, strerror(errno));
+        return summary;
+    }
+
+    struct dirent* dir_entry;
+    while ((dir_entry = readdir(dir)) != NULL) {
+        // skip . and ..
+        if (strcmp(dir_entry->d_name, ".") == 0 || strcmp(dir_entry->d_name, "..") == 0) {
+            continue;
+        }
+
+        char test_definition_path[MAX_PATH_LENGTH];
+        snprintf(
+            test_definition_path, sizeof(test_definition_path), "%s%s", base_path,
+            dir_entry->d_name);
+
+        if (!is_viable_test_definition_file(test_definition_path)) continue;
+
+        bool succeeded = run_test(test_definition_path, base_command);
+
+        summary.total++;
+        if (succeeded) {
+            summary.ok++;
+        } else {
+            summary.fail++;
+        }
+    }
+
+    return summary;
+}
+
+struct Result_Summary
+lex_tests(void)
+{
+    const ascii* base_path = "./boot/tests/lex/";
+    const ascii* base_command = "./build/catboot --test-lex";
+
+    return tests(base_path, base_command);
+}
+
+struct Result_Summary
+parse_tests(void)
+{
+    const ascii* base_path = "./boot/tests/parse/";
+    const ascii* base_command = "./build/catboot --test-parse";
+
+    return tests(base_path, base_command);
+}
+
+int
+main(int argc, const ascii** argv)
+{
+    enum Test_Target targets = targets_from_arguments(argc, argv);
+    if (targets == TARGET_NONE) {
+        usage();
+        return EXIT_FAILURE;
+    }
+
+    struct Result_Summary full_summary = { 0 };
+
+    for (uint t = 0; t < ARRAY_SIZE(known_targets); ++t) {
+        struct Known_Target target = known_targets[t];
+        if (!(targets & target.target)) continue;
+        if (target.target == TARGET_ALL) continue;
+
+        if (t > 0) fprintf(stderr, "\n");
+
+        fprintf(stderr, "running tests for target '%s'...\n", target.name);
+
+        struct Result_Summary summary = { 0 };
+        switch (target.target) {
+        case TARGET_LEX:
+            summary = lex_tests();
+            break;
+
+        case TARGET_PARSE:
+            summary = parse_tests();
+            break;
+
+        default:
+            failure("unknown test target");
+        }
+
+        if (summary.total == 0) {
+            fprintf(stderr, "no tests were run for target '%s'. ", target.name);
+        } else if (summary.fail == 0) {
+            fprintf(stderr, "all %lu tests passed for target '%s'. ", summary.total, target.name);
+        } else {
+            fprintf(stderr, "%lu/%lu tests failed for target '%s'. ", summary.fail, summary.total,
+                    target.name);
+        }
+        result_summary_print(summary);
+
+        full_summary = result_summary_merge(full_summary, summary);
+    }
+
+    fprintf(stderr, "\n");
+
+    if (full_summary.total == 0) {
+        fprintf(stderr, "no tests were run.\n");
+
+        return EXIT_SUCCESS;
+    }
+    if (full_summary.fail > 0) {
+        fprintf(stderr, "all tests finished, with failures. ");
+        result_summary_print(full_summary);
+
+        return EXIT_FAILURE;
+    }
+
+    fprintf(stderr, "all tests finished successfully. ");
+    result_summary_print(full_summary);
+
+    return EXIT_SUCCESS;
+}
\ No newline at end of file