From 1aef923b312f2bc9c0963381eae8619c187081c6 Mon Sep 17 00:00:00 2001 From: Mel Date: Thu, 28 May 2026 00:36:13 +0200 Subject: Use semicolon tokens as statement separators Signed-off-by: Mel --- boot/lex.c | 8 +++++++- boot/tests/parse/semicolon_separators.cskt | 25 +++++++++++++++++++++++++ docs/grammar.md | 4 ++-- 3 files changed, 34 insertions(+), 3 deletions(-) create mode 100644 boot/tests/parse/semicolon_separators.cskt diff --git a/boot/lex.c b/boot/lex.c index e585e1a..6c3e86f 100644 --- a/boot/lex.c +++ b/boot/lex.c @@ -53,6 +53,7 @@ enum Token_Kind TOKEN_SQUARE_CLOSE, TOKEN_COMMA, + TOKEN_SEMICOLON, TOKEN_AMPERSAND, TOKEN_DOT, TOKEN_DOT_DOT, @@ -176,6 +177,8 @@ token_kind_to_string(enum Token_Kind kind) case TOKEN_COMMA: return "COMMA"; + case TOKEN_SEMICOLON: + return "SEMICOLON"; case TOKEN_AMPERSAND: return "AMPERSAND"; case TOKEN_DOT: @@ -331,7 +334,8 @@ token_is_empty(const struct Token* t) bool token_ends_statement(const struct Token* t) { - return token_is(t, TOKEN_END_OF_FILE) || token_is(t, TOKEN_NEWLINE); + return token_is(t, TOKEN_END_OF_FILE) || token_is(t, TOKEN_NEWLINE) + || token_is(t, TOKEN_SEMICOLON); } bool @@ -574,6 +578,8 @@ lexer_symbol_token(struct Lexer* l, struct Lexer_Char current) case ',': RET{ TOKEN_COMMA, 1 }; + case ';': + RET{ TOKEN_SEMICOLON, 1 }; case '&': { lexer_match_chars(l, '&', '=', &a, &b); diff --git a/boot/tests/parse/semicolon_separators.cskt b/boot/tests/parse/semicolon_separators.cskt new file mode 100644 index 0000000..384ca71 --- /dev/null +++ b/boot/tests/parse/semicolon_separators.cskt @@ -0,0 +1,25 @@ +a `;` terminates the current statement just like a newline, +so multiple statements can sit on the same source line without +forcing a line break between them. + +<<< + +var a int = 1; var b int = 2 +a = a + b; b = a - b +a; b; c; d; e; f; g; +;;;;0;;;; + +>>> + +(variable (declaration a (type name int) (initializer (expr 1)))) +(variable (declaration b (type name int) (initializer (expr 2)))) +(expr (binary = (expr (name a)) (expr (binary + (expr (name a)) (expr (name b)))))) +(expr (binary = (expr (name b)) (expr (binary - (expr (name a)) (expr (name b)))))) +(expr (name a)) +(expr (name b)) +(expr (name c)) +(expr (name d)) +(expr (name e)) +(expr (name f)) +(expr (name g)) +(expr 0) diff --git a/docs/grammar.md b/docs/grammar.md index 2fca56e..18a8550 100644 --- a/docs/grammar.md +++ b/docs/grammar.md @@ -72,7 +72,7 @@ STRING_LITERAL = '"' { not_quote } '"' digit = "0".."9" # multi-character symbols are matched eagerly, e.g. "<<=" before "<<". -symbol = "(" | ")" | "{" | "}" | "[" | "]" | "," +symbol = "(" | ")" | "{" | "}" | "[" | "]" | "," | ";" | "." | ".." | "..." | "!" | "?" | "~" | "|" | "^" | "+" | "-" | "*" | "/" | "%" | "&" | "=" @@ -90,7 +90,7 @@ symbol = "(" | ")" | "{" | "}" | "[" | "]" | "," ```ebnf # a translation unit is a stream of statements at the top level. unit = { statement statement_end } -statement_end = NEWLINE | EOF +statement_end = NEWLINE | ";" | EOF statement = statement_declaration | statement_conditional -- cgit 1.4.1