Skip comments in scanner HEAD main

2 files changed, 33 insertions, 3 deletions

diff --git a/pkg/lang/scanner/scanner.go b/pkg/lang/scanner/scanner.go
index eb7da89..e1f8edf 100644
--- a/pkg/lang/scanner/scanner.go
+++ b/pkg/lang/scanner/scanner.go
@@ -48,7 +48,7 @@ func (s *Scanner) scanToken() (token.Token, error) {
 		return token.Token{}, ErrScannerFinished
 	}
 
-	hadNewline, firstNewline, err := s.skipWhitespace()
+	hadNewline, firstNewline, err := s.skipNonCode()
 	if err != nil {
 		return token.Token{}, err
 	}
@@ -314,7 +314,8 @@ func (s *Scanner) scanNumber() (token.Token, error) {
 	return token.New(token.Int, loc, num), nil
 }
 
-func (s *Scanner) skipWhitespace() (bool, source.Loc, error) {
+func (s *Scanner) skipNonCode() (bool, source.Loc, error) {
+	isInComment := false
 	hadNewline := false
 	firstNewline := source.Loc{}
 
@@ -329,7 +330,13 @@ func (s *Scanner) skipWhitespace() (bool, source.Loc, error) {
 			hadNewline = true
 		}
 
-		if eof || !unicode.IsSpace(c) {
+		if isInComment {
+			if c == '\n' {
+				isInComment = false
+			}
+		} else if c == '#' {
+			isInComment = true
+		} else if eof || !unicode.IsSpace(c) {
 			break
 		}
 
diff --git a/pkg/lang/scanner/scanner_test.go b/pkg/lang/scanner/scanner_test.go
index 4df4b23..fa46152 100644
--- a/pkg/lang/scanner/scanner_test.go
+++ b/pkg/lang/scanner/scanner_test.go
@@ -179,3 +179,26 @@ func TestEmojiInStrings(t *testing.T) {
 
 	require.Equal(t, expected, tokens)
 }
+
+func TestComments(t *testing.T) {
+	src := `
+	# Assignment to 'x'
+	x = 1 # x will be equal to 1
+	`
+
+	s := scanner.New(strings.NewReader(src))
+
+	tokens, err := s.Scan()
+	require.NoError(t, err)
+
+	expected := []token.Token{
+		token.Simple(token.EOL, source.NewLoc(0, 0)),
+		token.New(token.Ident, source.NewLoc(2, 1), "x"),
+		token.Simple(token.Assign, source.NewLoc(2, 3)),
+		token.New(token.Int, source.NewLoc(2, 5), uint64(1)),
+		token.Simple(token.EOL, source.NewLoc(2, 29)),
+		token.Simple(token.EOF, source.NewLoc(3, 1)),
+	}
+
+	require.Equal(t, expected, tokens)
+}