path: root/boot/tests/test.c

/*
 * a test suite runner for the catskill bootstrap compiler.
 * valid test files are situated in one of the suite folders in `tests`,
 * and demarcate the input beginning with `<<<` and expected output beginning
 * with `>>>`.
 * NOTE: currently this is basically a rough prototype, please handle with care!
 *
 * Copyright (c) 2025, Mel G. <mel@rnrd.eu>
 *
 * SPDX-License-Identifier: MPL-2.0
 */

#define _GNU_SOURCE

#include <dirent.h>
#include <errno.h>
#include <fcntl.h>
#include <stdio.h>
#include <stdlib.h>
#include <sys/mman.h>
#include <sys/stat.h>
#include <sys/types.h>

#include "../common.c"

#define MAX_TESTS 256

#define INPUT_MARK "<<<"
#define OUTPUT_MARK ">>>"

enum Result
{
    RESULT_NONE,
    RESULT_OK,
    RESULT_FAIL,
};

struct Result_Summary
{
    uint total;
    uint ok;
    uint fail;
};

struct Result_Summary
result_summary_merge(struct Result_Summary a, struct Result_Summary b)
{
    return (struct Result_Summary){
        .total = a.total + b.total,
        .ok = a.ok + b.ok,
        .fail = a.fail + b.fail,
    };
}

void
result_summary_print(struct Result_Summary summary)
{
    fprintf(stderr, "[%lu/%lu/%lu]\n", summary.ok, summary.fail, summary.total);
}

enum Test_Target
{
    TARGET_NONE = 0,
    TARGET_LEX = 1 << 0,
    TARGET_PARSE = 1 << 1,

    TARGET_ALL = TARGET_LEX | TARGET_PARSE,
};

struct Known_Target
{
    const ascii* name;
    enum Test_Target target;
} known_targets[] = {
    { "lex", TARGET_LEX },
    { "parse", TARGET_PARSE },

    { "all", TARGET_ALL },
};

void
usage(void)
{
    fprintf(
        stderr,
        "usage: test [<target> ...]\n"
        "targets:\n");
    for (uint i = 0; i < ARRAY_SIZE(known_targets); ++i) {
        fprintf(stderr, "  %s\n", known_targets[i].name);
    }
}

enum Test_Target
targets_from_arguments(int argc, const ascii** argv)
{
    enum Test_Target target = TARGET_NONE;
    for (uint ai = 1; ai < argc; ++ai) {
        const ascii* arg = argv[ai];

        bool found = false;
        for (uint ti = 0; ti < ARRAY_SIZE(known_targets); ++ti) {
            struct Known_Target kt = known_targets[ti];
            if (strcmp(arg, kt.name) == 0) {
                found = true;
                target |= kt.target;
                break;
            }
        }

        // TODO: parameterize `failure`
        if (!found) failure("unknown test target");
    }

    return target;
}

bool
is_viable_test_definition_file(const ascii* path)
{
    struct stat st;
    if (stat(path, &st) != 0) {
        fprintf(stderr, "couldn't stat file '%s': %s\n", path, strerror(errno));
        return false;
    }

    // is it a regular file?
    if (!S_ISREG(st.st_mode)) {
        fprintf(stderr, "file '%s' is not a regular file.\n", path);
        return false;
    }

    // can i read it?
    if (!(st.st_mode & S_IRUSR)) {
        fprintf(stderr, "i couldn't read file '%s'.\n", path);
        return false;
    }

    return true;
}

#define MAX_OUTPUT_LENGTH 65536

// NOTE: below are multiple `String` extension functions,
// some of which should definitely be integrated into the common
// library.

bool
string_equal(struct String a, struct String b)
{
    return strcmp(a.data, b.data) == 0;
}

struct String
string_from_pipe(FILE* pipe)
{
    // we never free this memory, but tests shouldn't be long-running enough
    // for this to be a problem.
    uint bytes_read;
    ascii* buffer = malloc(MAX_OUTPUT_LENGTH + 1);
    if (!buffer) failure("failed to allocate memory for command output\n");

    while ((bytes_read = fread(buffer, 1, MAX_OUTPUT_LENGTH, pipe)) > 0) {
        if (bytes_read == MAX_OUTPUT_LENGTH) {
            fprintf(
                stderr, "i truncated the command output to %d bytes, something is probably wrong\n",
                MAX_OUTPUT_LENGTH);
            break;
        }
    }

    return string_from_static_c_string(buffer);
}

// map file into string.
// TODO: merge this with the `read_file` in `boot/catboot.c` into the common lib.
struct String
string_from_file(const ascii* path)
{
    struct stat stat_info;
    if (stat(path, &stat_info) == -1) failure("i couldn't open that file, sorry :(");

    check(stat_info.st_size > 0, "file is empty, i can't map an empty file");

    const int32 file_descriptor = open(path, O_RDONLY);
    check(file_descriptor != -1, "i couldn't open that file, sorry :(");

    const flags mmap_prot = PROT_READ;
    const flags mmap_flags = MAP_PRIVATE;
    const unknown* file_data =
        mmap(nil, stat_info.st_size, mmap_prot, mmap_flags, file_descriptor, 0);

    return string_from_static_c_string(file_data);
}

#define MAX_INPUT_LENGTH 8192
#define MAX_PATH_LENGTH 256

struct String
string_strip_until_pattern(struct String str, const ascii* pattern)
{
    check(pattern, "pattern given to `string_strip_until_pattern` is nil");

    uint pattern_length = strlen(pattern);
    ascii* pattern_start = strstr(str.data, pattern);
    if (!pattern_start) return str;

    return string_from_c_string(pattern_start + pattern_length);
}

struct String
string_strip_from_pattern(struct String str, const ascii* pattern)
{
    check(pattern, "pattern given to `string_strip_after_pattern` is nil");

    ascii* pattern_start = strstr(str.data, pattern);
    if (!pattern_start) return str;

    uint new_string_length = pattern_start - str.data;
    check(new_string_length < MAX_INPUT_LENGTH - 1, "`string_strip_after_pattern` input too long");

    ascii buffer[MAX_INPUT_LENGTH];
    memcpy(buffer, str.data, new_string_length);
    buffer[new_string_length] = '\0';

    return string_from_c_string(buffer);
}

struct String
string_trim_left(struct String str)
{
    if (string_is_empty(str)) return str;

    ascii* c;
    for (c = str.data; *c == ' ' || *c == '\n'; c++) {}

    return string_from_static_c_string(c);
}

struct String
string_trim_right(struct String str)
{
    if (string_is_empty(str)) return str;

    ascii* c;
    for (c = &str.data[string_length(str) - 1]; (*c == ' ' || *c == '\n') && c >= str.data; c--) {}

    uint length = c - str.data + 1;
    return string_new(str.data, length);
}

struct String
string_trim(struct String str)
{
    return string_trim_right(string_trim_left(str));
}

struct Temporary_File
{
    ascii* path;
};

struct Temporary_File
temporary_file_from_string(struct String str)
{
    const ascii* name_template = "/tmp/catboot_test_artifact_XXXXXX";

    struct Temporary_File file = {
        .path = malloc(MAX_PATH_LENGTH),
    };
    check(file.path, "failed to allocate file name buffer memory");
    strcpy(file.path, name_template);

    integer fd = mkstemp(file.path);
    if (fd < 0) {
        // TODO: `failure` and `check` functions that print errno content
        fprintf(stderr, "i couldn't create a temporary input file: %s\n", strerror(errno));
        exit(EXIT_FAILURE);
    }

    integer write_result = write(fd, str.data, str.length);
    if (write_result < 0) {
        fprintf(stderr, "i couldn't write to temporary input file: %s\n", strerror(errno));
        exit(EXIT_FAILURE);
    }

    check(write_result == str.length, "couldn't write complete input to temporary file");

    close(fd);

    return file;
}

void
temporary_file_delete(struct Temporary_File file)
{
    if (unlink(file.path) < 0)
        fprintf(stderr, "warning, i failed removing temporary file at '%s'... weird...", file.path);

    free(file.path);
}

struct Command_Result
{
    bool did_succeed;
    struct String output;
};

struct Command_Result
run_command(const ascii* command)
{
    FILE* command_pipe = popen(command, "r");
    if (!command_pipe) {
        fprintf(stderr, "i couldn't run '%s': %s\n", command, strerror(errno));
        return (struct Command_Result){ .did_succeed = false };
    }

    struct String output = string_trim(string_from_pipe(command_pipe));

    int32 status = pclose(command_pipe);
    if (status == -1) {
        fprintf(stderr, "i couldn't finish running '%s': %s\n", command, strerror(errno));
        return (struct Command_Result){ .did_succeed = false };
    }

    // i think (??) `pclose` just runs `waitpid` and returns it's status directly,
    // so we should probably check it with the same macros as `waitpid`, but i'm not 100% sure.
    bool did_succeed = true;
    if (WIFEXITED(status)) {
        integer exit_code = WEXITSTATUS(status);
        if (exit_code != 0) {
            fprintf(stderr, "command '%s' exited with non-zero code (%ld)\n", command, exit_code);
            did_succeed = false;
        }
    } else if (WIFSIGNALED(status)) {
        integer signal_number = WTERMSIG(status);
        fprintf(stderr, "command '%s' was terminated by signal (%ld)\n", command, signal_number);
        did_succeed = false;
    }

    return (struct Command_Result){
        .did_succeed = did_succeed,
        .output = output,
    };
}

bool
run_test(const ascii* test_definition_path, const ascii* base_command)
{
    struct String test_file = string_from_file(test_definition_path);

    struct String test_content = string_strip_until_pattern(test_file, INPUT_MARK);

    struct String input = string_strip_from_pattern(test_content, OUTPUT_MARK);
    struct String expected_output = string_strip_until_pattern(test_content, OUTPUT_MARK);

    input = string_trim(input), expected_output = string_trim(expected_output);

    struct Temporary_File input_file = temporary_file_from_string(input);

    ascii command[MAX_PATH_LENGTH + 64];
    snprintf(command, sizeof(command), "%s %s", base_command, input_file.path);

    struct Command_Result result = run_command(command);

    bool success = true;
    if (!result.did_succeed) {
        fprintf(stderr, "'%s': did not complete successfully.\n", test_definition_path);
        success = false;
        goto end;
    }

    struct String output = string_trim(result.output);
    if (!string_equal(output, expected_output)) {
        struct Temporary_File wrong_output_file = temporary_file_from_string(output);
        fprintf(stderr, "'%s': completed with incorrect output. written to %s.\n",
                test_definition_path, wrong_output_file.path);
        free(wrong_output_file.path); // free but don't delete.
        success = false;
        goto end;
    }

    fprintf(stderr, "'%s': completed successfully.\n", test_definition_path);

end:
    temporary_file_delete(input_file);
    return success;
}

struct Result_Summary
tests(const ascii* base_path, const ascii* base_command)
{
    struct Result_Summary summary = { 0 };

    // iterate through all test files in the base path
    DIR* dir = opendir(base_path);
    if (!dir) {
        fprintf(stderr, "failed to open directory '%s': %s\n", base_path, strerror(errno));
        return summary;
    }

    struct dirent* dir_entry;
    while ((dir_entry = readdir(dir)) != NULL) {
        // skip . and ..
        if (strcmp(dir_entry->d_name, ".") == 0 || strcmp(dir_entry->d_name, "..") == 0) {
            continue;
        }

        char test_definition_path[MAX_PATH_LENGTH];
        snprintf(
            test_definition_path, sizeof(test_definition_path), "%s%s", base_path,
            dir_entry->d_name);

        if (!is_viable_test_definition_file(test_definition_path)) continue;

        bool succeeded = run_test(test_definition_path, base_command);

        summary.total++;
        if (succeeded) {
            summary.ok++;
        } else {
            summary.fail++;
        }
    }

    return summary;
}

struct Result_Summary
lex_tests(void)
{
    const ascii* base_path = "./boot/tests/lex/";
    const ascii* base_command = "./build/catboot --test-lex";

    return tests(base_path, base_command);
}

struct Result_Summary
parse_tests(void)
{
    const ascii* base_path = "./boot/tests/parse/";
    const ascii* base_command = "./build/catboot --test-parse";

    return tests(base_path, base_command);
}

int
main(int argc, const ascii** argv)
{
    enum Test_Target targets = targets_from_arguments(argc, argv);
    if (targets == TARGET_NONE) {
        usage();
        return EXIT_FAILURE;
    }

    struct Result_Summary full_summary = { 0 };

    for (uint t = 0; t < ARRAY_SIZE(known_targets); ++t) {
        struct Known_Target target = known_targets[t];
        if (!(targets & target.target)) continue;
        if (target.target == TARGET_ALL) continue;

        if (t > 0) fprintf(stderr, "\n");

        fprintf(stderr, "running tests for target '%s'...\n", target.name);

        struct Result_Summary summary = { 0 };
        switch (target.target) {
        case TARGET_LEX:
            summary = lex_tests();
            break;

        case TARGET_PARSE:
            summary = parse_tests();
            break;

        default:
            failure("unknown test target");
        }

        if (summary.total == 0) {
            fprintf(stderr, "no tests were run for target '%s'. ", target.name);
        } else if (summary.fail == 0) {
            fprintf(stderr, "all %lu tests passed for target '%s'. ", summary.total, target.name);
        } else {
            fprintf(stderr, "%lu/%lu tests failed for target '%s'. ", summary.fail, summary.total,
                    target.name);
        }
        result_summary_print(summary);

        full_summary = result_summary_merge(full_summary, summary);
    }

    fprintf(stderr, "\n");

    if (full_summary.total == 0) {
        fprintf(stderr, "no tests were run.\n");

        return EXIT_SUCCESS;
    }
    if (full_summary.fail > 0) {
        fprintf(stderr, "all tests finished, with failures. ");
        result_summary_print(full_summary);

        return EXIT_FAILURE;
    }

    fprintf(stderr, "all tests finished successfully. ");
    result_summary_print(full_summary);

    return EXIT_SUCCESS;
}