From 514ceb979c4ce79bfee2234cf981292ded714f66 Mon Sep 17 00:00:00 2001
From: Mel <einebeere@gmail.com>
Date: Sat, 13 Nov 2021 19:08:06 +0100
Subject: Lex string embeds.

---
 src/lex/token.rs | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

(limited to 'src/lex/token.rs')

diff --git a/src/lex/token.rs b/src/lex/token.rs
index 77b0df6..c771b4a 100644
--- a/src/lex/token.rs
+++ b/src/lex/token.rs
@@ -49,8 +49,17 @@ pub enum TokenVariant {
     // Literals
     Int(u32),
     Float(f32),
+    
+    // String Literal Tokens
     Str(String),
-
+    // StrOpen and StrClose are necessary for string embeds.
+    // A normal string looks like [StrOpen Str("Hello!") StrClose] in a token stream while a
+    // string with an embed would look something like [StrOpen Str("Hello") StrEmbed("world") Str("!") S StrClose].
+    StrOpen,
+    StrClose,
+    // The string embed has to be lexed by a *seperate* lexer.
+    StrEmbed(String),
+    
     Ident(String),
 
     // Keywords
@@ -71,4 +80,4 @@ pub enum TokenVariant {
 
     Unknown(char),
     Eof,
-}
+}
\ No newline at end of file
-- 
cgit 1.4.1