/* * a test suite runner for the catskill bootstrap compiler. * valid test files are situated in one of the suite folders in `tests`, * and demarcate the input beginning with `<<<` and expected output beginning * with `>>>`. * NOTE: currently this is basically a rough prototype, please handle with care! * * Copyright (c) 2025, Mel G. * * SPDX-License-Identifier: MPL-2.0 */ #define _GNU_SOURCE #include #include #include #include #include #include #include #include #include "../common.c" #define MAX_TESTS 256 #define INPUT_MARK "<<<" #define OUTPUT_MARK ">>>" enum Result { RESULT_NONE, RESULT_OK, RESULT_FAIL, }; struct Result_Summary { uint total; uint ok; uint fail; }; struct Result_Summary result_summary_merge(struct Result_Summary a, struct Result_Summary b) { return (struct Result_Summary){ .total = a.total + b.total, .ok = a.ok + b.ok, .fail = a.fail + b.fail, }; } void result_summary_print(struct Result_Summary summary) { fprintf(stderr, "[%lu/%lu/%lu]\n", summary.ok, summary.fail, summary.total); } enum Test_Target { TARGET_NONE = 0, TARGET_LEX = 1 << 0, TARGET_PARSE = 1 << 1, TARGET_ALL = TARGET_LEX | TARGET_PARSE, }; struct Known_Target { const ascii* name; enum Test_Target target; } known_targets[] = { { "lex", TARGET_LEX }, { "parse", TARGET_PARSE }, { "all", TARGET_ALL }, }; void usage(void) { fprintf( stderr, "usage: test [ ...]\n" "targets:\n"); for (uint i = 0; i < ARRAY_SIZE(known_targets); ++i) { fprintf(stderr, " %s\n", known_targets[i].name); } } enum Test_Target targets_from_arguments(int argc, const ascii** argv) { enum Test_Target target = TARGET_NONE; for (uint ai = 1; ai < argc; ++ai) { const ascii* arg = argv[ai]; bool found = false; for (uint ti = 0; ti < ARRAY_SIZE(known_targets); ++ti) { struct Known_Target kt = known_targets[ti]; if (strcmp(arg, kt.name) == 0) { found = true; target |= kt.target; break; } } // TODO: parameterize `failure` if (!found) failure("unknown test target"); } return target; } bool is_viable_test_definition_file(const ascii* path) { struct stat st; if (stat(path, &st) != 0) { fprintf(stderr, "couldn't stat file '%s': %s\n", path, strerror(errno)); return false; } // is it a regular file? if (!S_ISREG(st.st_mode)) { fprintf(stderr, "file '%s' is not a regular file.\n", path); return false; } // can i read it? if (!(st.st_mode & S_IRUSR)) { fprintf(stderr, "i couldn't read file '%s'.\n", path); return false; } return true; } #define MAX_OUTPUT_LENGTH 65536 // NOTE: below are multiple `String` extension functions, // some of which should definitely be integrated into the common // library. bool string_equal(struct String a, struct String b) { return strcmp(a.data, b.data) == 0; } struct String string_from_pipe(FILE* pipe) { // we never free this memory, but tests shouldn't be long-running enough // for this to be a problem. uint bytes_read; ascii* buffer = malloc(MAX_OUTPUT_LENGTH + 1); if (!buffer) failure("failed to allocate memory for command output\n"); while ((bytes_read = fread(buffer, 1, MAX_OUTPUT_LENGTH, pipe)) > 0) { if (bytes_read == MAX_OUTPUT_LENGTH) { fprintf( stderr, "i truncated the command output to %d bytes, something is probably wrong\n", MAX_OUTPUT_LENGTH); break; } } return string_from_static_c_string(buffer); } // map file into string. // TODO: merge this with the `read_file` in `boot/catboot.c` into the common lib. struct String string_from_file(const ascii* path) { struct stat stat_info; if (stat(path, &stat_info) == -1) failure("i couldn't open that file, sorry :("); check(stat_info.st_size > 0, "file is empty, i can't map an empty file"); const int32 file_descriptor = open(path, O_RDONLY); check(file_descriptor != -1, "i couldn't open that file, sorry :("); const flags mmap_prot = PROT_READ; const flags mmap_flags = MAP_PRIVATE; const unknown* file_data = mmap(nil, stat_info.st_size, mmap_prot, mmap_flags, file_descriptor, 0); return string_from_static_c_string(file_data); } #define MAX_INPUT_LENGTH 8192 #define MAX_PATH_LENGTH 256 struct String string_strip_until_pattern(struct String str, const ascii* pattern) { check(pattern, "pattern given to `string_strip_until_pattern` is nil"); uint pattern_length = strlen(pattern); ascii* pattern_start = strstr(str.data, pattern); if (!pattern_start) return str; return string_from_c_string(pattern_start + pattern_length); } struct String string_strip_from_pattern(struct String str, const ascii* pattern) { check(pattern, "pattern given to `string_strip_after_pattern` is nil"); ascii* pattern_start = strstr(str.data, pattern); if (!pattern_start) return str; uint new_string_length = pattern_start - str.data; check(new_string_length < MAX_INPUT_LENGTH - 1, "`string_strip_after_pattern` input too long"); ascii buffer[MAX_INPUT_LENGTH]; memcpy(buffer, str.data, new_string_length); buffer[new_string_length] = '\0'; return string_from_c_string(buffer); } struct String string_trim_left(struct String str) { if (string_is_empty(str)) return str; ascii* c; for (c = str.data; *c == ' ' || *c == '\n'; c++) {} return string_from_static_c_string(c); } struct String string_trim_right(struct String str) { if (string_is_empty(str)) return str; ascii* c; for (c = &str.data[string_length(str) - 1]; (*c == ' ' || *c == '\n') && c >= str.data; c--) {} uint length = c - str.data + 1; return string_new(str.data, length); } struct String string_trim(struct String str) { return string_trim_right(string_trim_left(str)); } struct Temporary_File { ascii* path; }; struct Temporary_File temporary_file_from_string(struct String str) { const ascii* name_template = "/tmp/catboot_test_artifact_XXXXXX"; struct Temporary_File file = { .path = malloc(MAX_PATH_LENGTH), }; check(file.path, "failed to allocate file name buffer memory"); strcpy(file.path, name_template); integer fd = mkstemp(file.path); if (fd < 0) { // TODO: `failure` and `check` functions that print errno content fprintf(stderr, "i couldn't create a temporary input file: %s\n", strerror(errno)); exit(EXIT_FAILURE); } integer write_result = write(fd, str.data, str.length); if (write_result < 0) { fprintf(stderr, "i couldn't write to temporary input file: %s\n", strerror(errno)); exit(EXIT_FAILURE); } check(write_result == str.length, "couldn't write complete input to temporary file"); close(fd); return file; } void temporary_file_delete(struct Temporary_File file) { if (unlink(file.path) < 0) fprintf(stderr, "warning, i failed removing temporary file at '%s'... weird...", file.path); free(file.path); } struct Command_Result { bool did_succeed; struct String output; }; struct Command_Result run_command(const ascii* command) { FILE* command_pipe = popen(command, "r"); if (!command_pipe) { fprintf(stderr, "i couldn't run '%s': %s\n", command, strerror(errno)); return (struct Command_Result){ .did_succeed = false }; } struct String output = string_trim(string_from_pipe(command_pipe)); int32 status = pclose(command_pipe); if (status == -1) { fprintf(stderr, "i couldn't finish running '%s': %s\n", command, strerror(errno)); return (struct Command_Result){ .did_succeed = false }; } // i think (??) `pclose` just runs `waitpid` and returns it's status directly, // so we should probably check it with the same macros as `waitpid`, but i'm not 100% sure. bool did_succeed = true; if (WIFEXITED(status)) { integer exit_code = WEXITSTATUS(status); if (exit_code != 0) { fprintf(stderr, "command '%s' exited with non-zero code (%ld)\n", command, exit_code); did_succeed = false; } } else if (WIFSIGNALED(status)) { integer signal_number = WTERMSIG(status); fprintf(stderr, "command '%s' was terminated by signal (%ld)\n", command, signal_number); did_succeed = false; } return (struct Command_Result){ .did_succeed = did_succeed, .output = output, }; } bool run_test(const ascii* test_definition_path, const ascii* base_command) { struct String test_file = string_from_file(test_definition_path); struct String test_content = string_strip_until_pattern(test_file, INPUT_MARK); struct String input = string_strip_from_pattern(test_content, OUTPUT_MARK); struct String expected_output = string_strip_until_pattern(test_content, OUTPUT_MARK); input = string_trim(input), expected_output = string_trim(expected_output); struct Temporary_File input_file = temporary_file_from_string(input); ascii command[MAX_PATH_LENGTH + 64]; snprintf(command, sizeof(command), "%s %s", base_command, input_file.path); struct Command_Result result = run_command(command); bool success = true; if (!result.did_succeed) { fprintf(stderr, "'%s': did not complete successfully.\n", test_definition_path); success = false; goto end; } struct String output = string_trim(result.output); if (!string_equal(output, expected_output)) { struct Temporary_File wrong_output_file = temporary_file_from_string(output); fprintf(stderr, "'%s': completed with incorrect output. written to %s.\n", test_definition_path, wrong_output_file.path); temporary_file_delete(wrong_output_file); success = false; } end: temporary_file_delete(input_file); return success; } struct Result_Summary tests(const ascii* base_path, const ascii* base_command) { struct Result_Summary summary = { 0 }; // iterate through all test files in the base path DIR* dir = opendir(base_path); if (!dir) { fprintf(stderr, "failed to open directory '%s': %s\n", base_path, strerror(errno)); return summary; } struct dirent* dir_entry; while ((dir_entry = readdir(dir)) != NULL) { // skip . and .. if (strcmp(dir_entry->d_name, ".") == 0 || strcmp(dir_entry->d_name, "..") == 0) { continue; } char test_definition_path[MAX_PATH_LENGTH]; snprintf( test_definition_path, sizeof(test_definition_path), "%s%s", base_path, dir_entry->d_name); if (!is_viable_test_definition_file(test_definition_path)) continue; bool succeeded = run_test(test_definition_path, base_command); summary.total++; if (succeeded) { summary.ok++; } else { summary.fail++; } } return summary; } struct Result_Summary lex_tests(void) { const ascii* base_path = "./boot/tests/lex/"; const ascii* base_command = "./build/catboot --test-lex"; return tests(base_path, base_command); } struct Result_Summary parse_tests(void) { const ascii* base_path = "./boot/tests/parse/"; const ascii* base_command = "./build/catboot --test-parse"; return tests(base_path, base_command); } int main(int argc, const ascii** argv) { enum Test_Target targets = targets_from_arguments(argc, argv); if (targets == TARGET_NONE) { usage(); return EXIT_FAILURE; } struct Result_Summary full_summary = { 0 }; for (uint t = 0; t < ARRAY_SIZE(known_targets); ++t) { struct Known_Target target = known_targets[t]; if (!(targets & target.target)) continue; if (target.target == TARGET_ALL) continue; if (t > 0) fprintf(stderr, "\n"); fprintf(stderr, "running tests for target '%s'...\n", target.name); struct Result_Summary summary = { 0 }; switch (target.target) { case TARGET_LEX: summary = lex_tests(); break; case TARGET_PARSE: summary = parse_tests(); break; default: failure("unknown test target"); } if (summary.total == 0) { fprintf(stderr, "no tests were run for target '%s'. ", target.name); } else if (summary.fail == 0) { fprintf(stderr, "all %lu tests passed for target '%s'. ", summary.total, target.name); } else { fprintf(stderr, "%lu/%lu tests failed for target '%s'. ", summary.fail, summary.total, target.name); } result_summary_print(summary); full_summary = result_summary_merge(full_summary, summary); } fprintf(stderr, "\n"); if (full_summary.total == 0) { fprintf(stderr, "no tests were run.\n"); return EXIT_SUCCESS; } if (full_summary.fail > 0) { fprintf(stderr, "all tests finished, with failures. "); result_summary_print(full_summary); return EXIT_FAILURE; } fprintf(stderr, "all tests finished successfully. "); result_summary_print(full_summary); return EXIT_SUCCESS; }