Emit EOL tokens in scanner

2 files changed, 53 insertions, 5 deletions

diff --git a/pkg/lang/scanner/scanner.go b/pkg/lang/scanner/scanner.go
index 5a1c92a..19e3462 100644
--- a/pkg/lang/scanner/scanner.go
+++ b/pkg/lang/scanner/scanner.go
@@ -53,10 +53,15 @@ func (s *Scanner) scanToken() (token.Token, error) {
 		return token.Token{}, ErrScannerFinished
 	}
 
-	if err := s.skipWhitespace(); err != nil {
+	hadNewline, firstNewline, err := s.skipWhitespace()
+	if err != nil {
 		return token.Token{}, err
 	}
 
+	if hadNewline {
+		return token.New(token.EOL, firstNewline, nil), nil
+	}
+
 	c, eof, err := s.peek()
 	if err != nil {
 		return token.Token{}, err
@@ -294,11 +299,19 @@ func (s *Scanner) scanNumber() (token.Token, error) {
 	return token.New(token.Int, loc, num), nil
 }
 
-func (s *Scanner) skipWhitespace() error {
+func (s *Scanner) skipWhitespace() (bool, token.Loc, error) {
+	hadNewline := false
+	firstNewline := token.Loc{}
+
 	for {
 		c, eof, err := s.peek()
 		if err != nil {
-			return err
+			return false, token.Loc{}, err
+		}
+
+		if c == '\n' && !hadNewline {
+			firstNewline = s.loc()
+			hadNewline = true
 		}
 
 		if eof || !unicode.IsSpace(c) {
@@ -306,11 +319,11 @@ func (s *Scanner) skipWhitespace() error {
 		}
 
 		if _, _, err = s.next(); err != nil {
-			return err
+			return false, token.Loc{}, err
 		}
 	}
 
-	return nil
+	return hadNewline, firstNewline, nil
 }
 
 func (s *Scanner) loc() token.Loc {
diff --git a/pkg/lang/scanner/scanner_test.go b/pkg/lang/scanner/scanner_test.go
index 3af7fa4..896cdfd 100644
--- a/pkg/lang/scanner/scanner_test.go
+++ b/pkg/lang/scanner/scanner_test.go
@@ -28,16 +28,20 @@ func TestBasic(t *testing.T) {
 	require.NoError(t, err)
 
 	expected := []token.Token{
+		token.Simple(token.EOL, token.NewLoc(0, 0)),
+
 		token.Simple(token.KwFn, token.NewLoc(1, 1)),
 		token.New(token.Ident, token.NewLoc(1, 4), "basic"),
 		token.Simple(token.LParen, token.NewLoc(1, 9)),
 		token.Simple(token.RParen, token.NewLoc(1, 10)),
 		token.Simple(token.LBrace, token.NewLoc(1, 12)),
+		token.Simple(token.EOL, token.NewLoc(1, 13)),
 
 		token.Simple(token.KwVar, token.NewLoc(2, 2)),
 		token.New(token.Ident, token.NewLoc(2, 6), "x"),
 		token.Simple(token.Assign, token.NewLoc(2, 8)),
 		token.New(token.Int, token.NewLoc(2, 10), uint64(1)),
+		token.Simple(token.EOL, token.NewLoc(2, 11)),
 
 		token.Simple(token.KwVar, token.NewLoc(3, 2)),
 		token.New(token.Ident, token.NewLoc(3, 6), "y"),
@@ -45,31 +49,38 @@ func TestBasic(t *testing.T) {
 		token.New(token.Ident, token.NewLoc(3, 10), "x"),
 		token.Simple(token.Plus, token.NewLoc(3, 12)),
 		token.New(token.Int, token.NewLoc(3, 14), uint64(1)),
+		token.Simple(token.EOL, token.NewLoc(3, 15)),
 
 		token.Simple(token.KwIf, token.NewLoc(4, 2)),
 		token.New(token.Ident, token.NewLoc(4, 5), "x"),
 		token.Simple(token.Lt, token.NewLoc(4, 7)),
 		token.New(token.Ident, token.NewLoc(4, 9), "y"),
 		token.Simple(token.LBrace, token.NewLoc(4, 11)),
+		token.Simple(token.EOL, token.NewLoc(4, 12)),
 
 		token.New(token.Ident, token.NewLoc(5, 3), "say"),
 		token.Simple(token.LParen, token.NewLoc(5, 6)),
 		token.New(token.String, token.NewLoc(5, 7), "x is less than y"),
 		token.Simple(token.RParen, token.NewLoc(5, 25)),
+		token.Simple(token.EOL, token.NewLoc(5, 26)),
 
 		token.Simple(token.RBrace, token.NewLoc(6, 2)),
 		token.Simple(token.KwElse, token.NewLoc(6, 4)),
 		token.Simple(token.LBrace, token.NewLoc(6, 9)),
+		token.Simple(token.EOL, token.NewLoc(6, 10)),
 
 		token.New(token.Ident, token.NewLoc(7, 3), "say"),
 		token.Simple(token.LParen, token.NewLoc(7, 6)),
 		token.New(token.String, token.NewLoc(7, 7), "x is greater than or equal to y"),
 		token.Simple(token.RParen, token.NewLoc(7, 40)),
+		token.Simple(token.EOL, token.NewLoc(7, 41)),
 
 		token.Simple(token.RBrace, token.NewLoc(8, 2)),
+		token.Simple(token.EOL, token.NewLoc(8, 3)),
 
 		token.Simple(token.KwReturn, token.NewLoc(9, 2)),
 		token.Simple(token.KwTrue, token.NewLoc(9, 9)),
+		token.Simple(token.EOL, token.NewLoc(9, 13)),
 
 		token.Simple(token.RBrace, token.NewLoc(10, 1)),
 
@@ -118,3 +129,27 @@ func TestTightNumber(t *testing.T) {
 	require.Equal(t, expected, tokens)
 }
 
+func TestNewlineStacking(t *testing.T) {
+	source := `
+	x
+
+
+	y
+	`
+
+	s := scanner.New(strings.NewReader(source))
+
+	tokens, err := s.Scan()
+	require.NoError(t, err)
+
+	expected := []token.Token{
+		token.Simple(token.EOL, token.NewLoc(0, 0)),
+		token.New(token.Ident, token.NewLoc(1, 1), "x"),
+		token.Simple(token.EOL, token.NewLoc(1, 2)),
+		token.New(token.Ident, token.NewLoc(4, 1), "y"),
+		token.Simple(token.EOL, token.NewLoc(4, 2)),
+		token.Simple(token.EOF, token.NewLoc(5, 1)),
+	}
+
+	require.Equal(t, expected, tokens)
+}