package scanner_test import ( "jinx/pkg/lang/scanner" "jinx/pkg/lang/scanner/token" "strings" "testing" "github.com/stretchr/testify/require" ) func TestBasic(t *testing.T) { source := ` fn basic() { var x = 1 var y = x + 1 if x < y { say("x is less than y") } else { say("x is greater than or equal to y") } return true }` s := scanner.New(strings.NewReader(source)) tokens, err := s.Scan() require.NoError(t, err) expected := []token.Token{ token.Simple(token.EOL, token.NewLoc(0, 0)), token.Simple(token.KwFn, token.NewLoc(1, 1)), token.New(token.Ident, token.NewLoc(1, 4), "basic"), token.Simple(token.LParen, token.NewLoc(1, 9)), token.Simple(token.RParen, token.NewLoc(1, 10)), token.Simple(token.LBrace, token.NewLoc(1, 12)), token.Simple(token.EOL, token.NewLoc(1, 13)), token.Simple(token.KwVar, token.NewLoc(2, 2)), token.New(token.Ident, token.NewLoc(2, 6), "x"), token.Simple(token.Assign, token.NewLoc(2, 8)), token.New(token.Int, token.NewLoc(2, 10), uint64(1)), token.Simple(token.EOL, token.NewLoc(2, 11)), token.Simple(token.KwVar, token.NewLoc(3, 2)), token.New(token.Ident, token.NewLoc(3, 6), "y"), token.Simple(token.Assign, token.NewLoc(3, 8)), token.New(token.Ident, token.NewLoc(3, 10), "x"), token.Simple(token.Plus, token.NewLoc(3, 12)), token.New(token.Int, token.NewLoc(3, 14), uint64(1)), token.Simple(token.EOL, token.NewLoc(3, 15)), token.Simple(token.KwIf, token.NewLoc(4, 2)), token.New(token.Ident, token.NewLoc(4, 5), "x"), token.Simple(token.Lt, token.NewLoc(4, 7)), token.New(token.Ident, token.NewLoc(4, 9), "y"), token.Simple(token.LBrace, token.NewLoc(4, 11)), token.Simple(token.EOL, token.NewLoc(4, 12)), token.New(token.Ident, token.NewLoc(5, 3), "say"), token.Simple(token.LParen, token.NewLoc(5, 6)), token.New(token.String, token.NewLoc(5, 7), "x is less than y"), token.Simple(token.RParen, token.NewLoc(5, 25)), token.Simple(token.EOL, token.NewLoc(5, 26)), token.Simple(token.RBrace, token.NewLoc(6, 2)), token.Simple(token.KwElse, token.NewLoc(6, 4)), token.Simple(token.LBrace, token.NewLoc(6, 9)), token.Simple(token.EOL, token.NewLoc(6, 10)), token.New(token.Ident, token.NewLoc(7, 3), "say"), token.Simple(token.LParen, token.NewLoc(7, 6)), token.New(token.String, token.NewLoc(7, 7), "x is greater than or equal to y"), token.Simple(token.RParen, token.NewLoc(7, 40)), token.Simple(token.EOL, token.NewLoc(7, 41)), token.Simple(token.RBrace, token.NewLoc(8, 2)), token.Simple(token.EOL, token.NewLoc(8, 3)), token.Simple(token.KwReturn, token.NewLoc(9, 2)), token.Simple(token.KwTrue, token.NewLoc(9, 9)), token.Simple(token.EOL, token.NewLoc(9, 13)), token.Simple(token.RBrace, token.NewLoc(10, 1)), token.Simple(token.EOF, token.NewLoc(10, 2)), } require.Equal(t, expected, tokens) } func TestTightIdent(t *testing.T) { source := `say(message)` s := scanner.New(strings.NewReader(source)) tokens, err := s.Scan() require.NoError(t, err) expected := []token.Token{ token.New(token.Ident, token.NewLoc(0, 0), "say"), token.Simple(token.LParen, token.NewLoc(0, 3)), token.New(token.Ident, token.NewLoc(0, 4), "message"), token.Simple(token.RParen, token.NewLoc(0, 11)), token.Simple(token.EOF, token.NewLoc(0, 12)), } require.Equal(t, expected, tokens) } func TestTightNumber(t *testing.T) { source := `1+2+3` s := scanner.New(strings.NewReader(source)) tokens, err := s.Scan() require.NoError(t, err) expected := []token.Token{ token.New(token.Int, token.NewLoc(0, 0), uint64(1)), token.Simple(token.Plus, token.NewLoc(0, 1)), token.New(token.Int, token.NewLoc(0, 2), uint64(2)), token.Simple(token.Plus, token.NewLoc(0, 3)), token.New(token.Int, token.NewLoc(0, 4), uint64(3)), token.Simple(token.EOF, token.NewLoc(0, 5)), } require.Equal(t, expected, tokens) } func TestNewlineStacking(t *testing.T) { source := ` x y ` s := scanner.New(strings.NewReader(source)) tokens, err := s.Scan() require.NoError(t, err) expected := []token.Token{ token.Simple(token.EOL, token.NewLoc(0, 0)), token.New(token.Ident, token.NewLoc(1, 1), "x"), token.Simple(token.EOL, token.NewLoc(1, 2)), token.New(token.Ident, token.NewLoc(4, 1), "y"), token.Simple(token.EOL, token.NewLoc(4, 2)), token.Simple(token.EOF, token.NewLoc(5, 1)), } require.Equal(t, expected, tokens) } func TestEmojiInStrings(t *testing.T) { source := ` say("🇺🇦" + "❤️!") ` s := scanner.New(strings.NewReader(source)) tokens, err := s.Scan() require.NoError(t, err) expected := []token.Token{ token.Simple(token.EOL, token.NewLoc(0, 0)), token.New(token.Ident, token.NewLoc(1, 1), "say"), token.Simple(token.LParen, token.NewLoc(1, 4)), token.New(token.String, token.NewLoc(1, 5), "🇺🇦"), token.Simple(token.Plus, token.NewLoc(1, 10)), token.New(token.String, token.NewLoc(1, 12), "❤️!"), token.Simple(token.RParen, token.NewLoc(1, 17)), token.Simple(token.EOL, token.NewLoc(1, 18)), token.Simple(token.EOF, token.NewLoc(2, 1)), } require.Equal(t, expected, tokens) }