Basic scanner for lang

3 files changed, 99 insertions, 0 deletions

diff --git a/pkg/lang/scanner/token/kind.go b/pkg/lang/scanner/token/kind.go
new file mode 100644
index 0000000..e24ce2f
--- /dev/null
+++ b/pkg/lang/scanner/token/kind.go
@@ -0,0 +1,71 @@
+package token
+
+type TokenKind int
+
+const (
+	EOF TokenKind = iota
+	EOL
+
+	// Keywords
+	KwVar
+	KwFn
+	KwObject
+
+	KwIf
+	KwElif
+	KwElse
+	KwFor
+	KwTry
+	KwCatch
+	KwFinally
+
+	KwReturn
+	KwContinue
+	KwBreak
+	KwThrow
+
+	KwIn
+
+	KwNull
+	KwTrue
+	KwFalse
+
+	KwThis
+
+	KwUse
+	KwFrom
+	KwBy
+
+	// Data Tokens
+	Ident
+	Int
+	Float
+	String
+
+	// Punctuation
+	Assign
+	Plus
+	Minus
+	Star
+	Slash
+	Percent
+	Bang
+
+	Eq
+	Neq
+	Lt
+	Gt
+	Lte
+	Gte
+
+	LParen
+	RParen
+	LBrace
+	RBrace
+	LBracket
+	RBracket
+
+	Comma
+	Dot
+	SemiColon
+)
diff --git a/pkg/lang/scanner/token/loc.go b/pkg/lang/scanner/token/loc.go
new file mode 100644
index 0000000..c4b073a
--- /dev/null
+++ b/pkg/lang/scanner/token/loc.go
@@ -0,0 +1,6 @@
+package token
+
+type Loc struct {
+	Row int
+	Col int
+}
diff --git a/pkg/lang/scanner/token/token.go b/pkg/lang/scanner/token/token.go
new file mode 100644
index 0000000..840a420
--- /dev/null
+++ b/pkg/lang/scanner/token/token.go
@@ -0,0 +1,22 @@
+package token
+
+type Token struct {
+	Kind TokenKind
+	At   Loc
+	Data any
+}
+
+func Simple(kind TokenKind, at Loc) Token {
+	return Token{
+		Kind: kind,
+		At:   at,
+	}
+}
+
+func New(kind TokenKind, at Loc, data any) Token {
+	return Token{
+		Kind: kind,
+		At:   at,
+		Data: data,
+	}
+}