Lex string embeds.

1 files changed, 11 insertions, 2 deletions

diff --git a/src/lex/token.rs b/src/lex/token.rs
index 77b0df6..c771b4a 100644
--- a/src/lex/token.rs
+++ b/src/lex/token.rs
@@ -49,8 +49,17 @@ pub enum TokenVariant {
     // Literals
     Int(u32),
     Float(f32),
+    
+    // String Literal Tokens
     Str(String),
-
+    // StrOpen and StrClose are necessary for string embeds.
+    // A normal string looks like [StrOpen Str("Hello!") StrClose] in a token stream while a
+    // string with an embed would look something like [StrOpen Str("Hello") StrEmbed("world") Str("!") S StrClose].
+    StrOpen,
+    StrClose,
+    // The string embed has to be lexed by a *seperate* lexer.
+    StrEmbed(String),
+    
     Ident(String),
 
     // Keywords
@@ -71,4 +80,4 @@ pub enum TokenVariant {
 
     Unknown(char),
     Eof,
-}
+}
\ No newline at end of file