package scanner_test import ( "jinx/pkg/lang/scanner" "jinx/pkg/lang/scanner/token" "jinx/pkg/libs/source" "strings" "testing" "github.com/stretchr/testify/require" ) func TestBasic(t *testing.T) { src := ` fn basic() { var x = 1 var y = x + 1 if x < y { say("x is less than y") } else { say("x is greater than or equal to y") } return true }` s := scanner.New(strings.NewReader(src)) tokens, err := s.Scan() require.NoError(t, err) expected := []token.Token{ token.Simple(token.EOL, source.NewLoc(0, 0)), token.Simple(token.KwFn, source.NewLoc(1, 1)), token.New(token.Ident, source.NewLoc(1, 4), "basic"), token.Simple(token.LParen, source.NewLoc(1, 9)), token.Simple(token.RParen, source.NewLoc(1, 10)), token.Simple(token.LBrace, source.NewLoc(1, 12)), token.Simple(token.EOL, source.NewLoc(1, 13)), token.Simple(token.KwVar, source.NewLoc(2, 2)), token.New(token.Ident, source.NewLoc(2, 6), "x"), token.Simple(token.Assign, source.NewLoc(2, 8)), token.New(token.Int, source.NewLoc(2, 10), uint64(1)), token.Simple(token.EOL, source.NewLoc(2, 11)), token.Simple(token.KwVar, source.NewLoc(3, 2)), token.New(token.Ident, source.NewLoc(3, 6), "y"), token.Simple(token.Assign, source.NewLoc(3, 8)), token.New(token.Ident, source.NewLoc(3, 10), "x"), token.Simple(token.Plus, source.NewLoc(3, 12)), token.New(token.Int, source.NewLoc(3, 14), uint64(1)), token.Simple(token.EOL, source.NewLoc(3, 15)), token.Simple(token.KwIf, source.NewLoc(4, 2)), token.New(token.Ident, source.NewLoc(4, 5), "x"), token.Simple(token.Lt, source.NewLoc(4, 7)), token.New(token.Ident, source.NewLoc(4, 9), "y"), token.Simple(token.LBrace, source.NewLoc(4, 11)), token.Simple(token.EOL, source.NewLoc(4, 12)), token.New(token.Ident, source.NewLoc(5, 3), "say"), token.Simple(token.LParen, source.NewLoc(5, 6)), token.New(token.String, source.NewLoc(5, 7), "x is less than y"), token.Simple(token.RParen, source.NewLoc(5, 25)), token.Simple(token.EOL, source.NewLoc(5, 26)), token.Simple(token.RBrace, source.NewLoc(6, 2)), token.Simple(token.KwElse, source.NewLoc(6, 4)), token.Simple(token.LBrace, source.NewLoc(6, 9)), token.Simple(token.EOL, source.NewLoc(6, 10)), token.New(token.Ident, source.NewLoc(7, 3), "say"), token.Simple(token.LParen, source.NewLoc(7, 6)), token.New(token.String, source.NewLoc(7, 7), "x is greater than or equal to y"), token.Simple(token.RParen, source.NewLoc(7, 40)), token.Simple(token.EOL, source.NewLoc(7, 41)), token.Simple(token.RBrace, source.NewLoc(8, 2)), token.Simple(token.EOL, source.NewLoc(8, 3)), token.Simple(token.KwReturn, source.NewLoc(9, 2)), token.Simple(token.KwTrue, source.NewLoc(9, 9)), token.Simple(token.EOL, source.NewLoc(9, 13)), token.Simple(token.RBrace, source.NewLoc(10, 1)), token.Simple(token.EOF, source.NewLoc(10, 2)), } require.Equal(t, expected, tokens) } func TestTightIdent(t *testing.T) { src := `say(message)` s := scanner.New(strings.NewReader(src)) tokens, err := s.Scan() require.NoError(t, err) expected := []token.Token{ token.New(token.Ident, source.NewLoc(0, 0), "say"), token.Simple(token.LParen, source.NewLoc(0, 3)), token.New(token.Ident, source.NewLoc(0, 4), "message"), token.Simple(token.RParen, source.NewLoc(0, 11)), token.Simple(token.EOF, source.NewLoc(0, 12)), } require.Equal(t, expected, tokens) } func TestTightNumber(t *testing.T) { src := `1+2+3` s := scanner.New(strings.NewReader(src)) tokens, err := s.Scan() require.NoError(t, err) expected := []token.Token{ token.New(token.Int, source.NewLoc(0, 0), uint64(1)), token.Simple(token.Plus, source.NewLoc(0, 1)), token.New(token.Int, source.NewLoc(0, 2), uint64(2)), token.Simple(token.Plus, source.NewLoc(0, 3)), token.New(token.Int, source.NewLoc(0, 4), uint64(3)), token.Simple(token.EOF, source.NewLoc(0, 5)), } require.Equal(t, expected, tokens) } func TestNewlineStacking(t *testing.T) { src := ` x y ` s := scanner.New(strings.NewReader(src)) tokens, err := s.Scan() require.NoError(t, err) expected := []token.Token{ token.Simple(token.EOL, source.NewLoc(0, 0)), token.New(token.Ident, source.NewLoc(1, 1), "x"), token.Simple(token.EOL, source.NewLoc(1, 2)), token.New(token.Ident, source.NewLoc(4, 1), "y"), token.Simple(token.EOL, source.NewLoc(4, 2)), token.Simple(token.EOF, source.NewLoc(5, 1)), } require.Equal(t, expected, tokens) } func TestEmojiInStrings(t *testing.T) { src := ` say("🇺🇦" + "❤️!") ` s := scanner.New(strings.NewReader(src)) tokens, err := s.Scan() require.NoError(t, err) expected := []token.Token{ token.Simple(token.EOL, source.NewLoc(0, 0)), token.New(token.Ident, source.NewLoc(1, 1), "say"), token.Simple(token.LParen, source.NewLoc(1, 4)), token.New(token.String, source.NewLoc(1, 5), "🇺🇦"), token.Simple(token.Plus, source.NewLoc(1, 10)), token.New(token.String, source.NewLoc(1, 12), "❤️!"), token.Simple(token.RParen, source.NewLoc(1, 17)), token.Simple(token.EOL, source.NewLoc(1, 18)), token.Simple(token.EOF, source.NewLoc(2, 1)), } require.Equal(t, expected, tokens) }