More scanner tests and tight idents fix

3 files changed, 85 insertions, 7 deletions

diff --git a/pkg/lang/scanner/scanner.go b/pkg/lang/scanner/scanner.go
index fdb313b..08481d8 100644
--- a/pkg/lang/scanner/scanner.go
+++ b/pkg/lang/scanner/scanner.go
@@ -190,7 +190,7 @@ func (s *Scanner) scanIdentifierOrKeyword() (token.Token, error) {
 	var buf strings.Builder
 
 	for {
-		c, eof, err := s.next()
+		c, eof, err := s.peek()
 		if err != nil {
 			return token.Token{}, err
 		}
@@ -199,6 +199,10 @@ func (s *Scanner) scanIdentifierOrKeyword() (token.Token, error) {
 			break
 		}
 
+		if _, _, err = s.next(); err != nil {
+			return token.Token{}, err
+		}
+
 		buf.WriteRune(c)
 	}
 
diff --git a/pkg/lang/scanner/scanner_test.go b/pkg/lang/scanner/scanner_test.go
index 2948a58..cc19f8f 100644
--- a/pkg/lang/scanner/scanner_test.go
+++ b/pkg/lang/scanner/scanner_test.go
@@ -10,7 +10,17 @@ import (
 )
 
 func TestBasic(t *testing.T) {
-	source := "var x = 1"
+	source := `
+	fn basic() {
+		var x = 1
+		var y = x + 1
+		if x < y {
+			say("x is less than y")
+		} else {
+			say("x is greater than or equal to y")
+		}
+		return true
+	}`
 
 	s := scanner.New(strings.NewReader(source))
 
@@ -18,11 +28,71 @@ func TestBasic(t *testing.T) {
 	require.NoError(t, err)
 
 	expected := []token.Token{
-		token.Simple(token.KwVar, token.Loc{Row: 0, Col: 0}),
-		token.New(token.Ident, token.Loc{Row: 0, Col: 4}, "x"),
-		token.Simple(token.Assign, token.Loc{Row: 0, Col: 6}),
-		token.New(token.Int, token.Loc{Row: 0, Col: 8}, uint64(1)),
-		token.Simple(token.EOF, token.Loc{Row: 0, Col: 9}),
+		token.Simple(token.KwFn, token.NewLoc(1, 1)),
+		token.New(token.Ident, token.NewLoc(1, 4), "basic"),
+		token.Simple(token.LParen, token.NewLoc(1, 9)),
+		token.Simple(token.RParen, token.NewLoc(1, 10)),
+		token.Simple(token.LBrace, token.NewLoc(1, 12)),
+
+		token.Simple(token.KwVar, token.NewLoc(2, 2)),
+		token.New(token.Ident, token.NewLoc(2, 6), "x"),
+		token.Simple(token.Assign, token.NewLoc(2, 8)),
+		token.New(token.Int, token.NewLoc(2, 10), uint64(1)),
+
+		token.Simple(token.KwVar, token.NewLoc(3, 2)),
+		token.New(token.Ident, token.NewLoc(3, 6), "y"),
+		token.Simple(token.Assign, token.NewLoc(3, 8)),
+		token.New(token.Ident, token.NewLoc(3, 10), "x"),
+		token.Simple(token.Plus, token.NewLoc(3, 12)),
+		token.New(token.Int, token.NewLoc(3, 14), uint64(1)),
+
+		token.Simple(token.KwIf, token.NewLoc(4, 2)),
+		token.New(token.Ident, token.NewLoc(4, 5), "x"),
+		token.Simple(token.Lt, token.NewLoc(4, 7)),
+		token.New(token.Ident, token.NewLoc(4, 9), "y"),
+		token.Simple(token.LBrace, token.NewLoc(4, 11)),
+
+		token.New(token.Ident, token.NewLoc(5, 3), "say"),
+		token.Simple(token.LParen, token.NewLoc(5, 6)),
+		token.New(token.String, token.NewLoc(5, 7), "x is less than y"),
+		token.Simple(token.RParen, token.NewLoc(5, 25)),
+
+		token.Simple(token.RBrace, token.NewLoc(6, 2)),
+		token.Simple(token.KwElse, token.NewLoc(6, 4)),
+		token.Simple(token.LBrace, token.NewLoc(6, 9)),
+
+		token.New(token.Ident, token.NewLoc(7, 3), "say"),
+		token.Simple(token.LParen, token.NewLoc(7, 6)),
+		token.New(token.String, token.NewLoc(7, 7), "x is greater than or equal to y"),
+		token.Simple(token.RParen, token.NewLoc(7, 40)),
+
+		token.Simple(token.RBrace, token.NewLoc(8, 2)),
+
+		token.Simple(token.KwReturn, token.NewLoc(9, 2)),
+		token.Simple(token.KwTrue, token.NewLoc(9, 9)),
+
+		token.Simple(token.RBrace, token.NewLoc(10, 1)),
+
+		token.Simple(token.EOF, token.NewLoc(10, 2)),
+	}
+
+	require.Equal(t, expected, tokens)
+}
+
+func TestTightIdent(t *testing.T) {
+	source := `say(message)`
+
+	s := scanner.New(strings.NewReader(source))
+
+	tokens, err := s.Scan()
+	require.NoError(t, err)
+
+	expected := []token.Token{
+		token.New(token.Ident, token.NewLoc(0, 0), "say"),
+		token.Simple(token.LParen, token.NewLoc(0, 3)),
+		token.New(token.Ident, token.NewLoc(0, 4), "message"),
+		token.Simple(token.RParen, token.NewLoc(0, 11)),
+		token.Simple(token.EOF, token.NewLoc(0, 12)),
 	}
 
 	require.Equal(t, expected, tokens)
diff --git a/pkg/lang/scanner/token/loc.go b/pkg/lang/scanner/token/loc.go
index c4b073a..7936cba 100644
--- a/pkg/lang/scanner/token/loc.go
+++ b/pkg/lang/scanner/token/loc.go
@@ -4,3 +4,7 @@ type Loc struct {
 	Row int
 	Col int
 }
+
+func NewLoc(row, col int) Loc {
+	return Loc{row, col}
+}