1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
|
package scanner_test
import (
"jinx/pkg/lang/scanner"
"jinx/pkg/lang/scanner/token"
"jinx/pkg/libs/source"
"strings"
"testing"
"github.com/stretchr/testify/require"
)
func TestBasic(t *testing.T) {
src := `
fn basic() {
var x = 1
var y = x + 1
if x < y {
say("x is less than y")
} else {
say("x is greater than or equal to y")
}
return true
}`
s := scanner.New(strings.NewReader(src))
tokens, err := s.Scan()
require.NoError(t, err)
expected := []token.Token{
token.Simple(token.EOL, source.NewLoc(0, 0)),
token.Simple(token.KwFn, source.NewLoc(1, 1)),
token.New(token.Ident, source.NewLoc(1, 4), "basic"),
token.Simple(token.LParen, source.NewLoc(1, 9)),
token.Simple(token.RParen, source.NewLoc(1, 10)),
token.Simple(token.LBrace, source.NewLoc(1, 12)),
token.Simple(token.EOL, source.NewLoc(1, 13)),
token.Simple(token.KwVar, source.NewLoc(2, 2)),
token.New(token.Ident, source.NewLoc(2, 6), "x"),
token.Simple(token.Assign, source.NewLoc(2, 8)),
token.New(token.Int, source.NewLoc(2, 10), uint64(1)),
token.Simple(token.EOL, source.NewLoc(2, 11)),
token.Simple(token.KwVar, source.NewLoc(3, 2)),
token.New(token.Ident, source.NewLoc(3, 6), "y"),
token.Simple(token.Assign, source.NewLoc(3, 8)),
token.New(token.Ident, source.NewLoc(3, 10), "x"),
token.Simple(token.Plus, source.NewLoc(3, 12)),
token.New(token.Int, source.NewLoc(3, 14), uint64(1)),
token.Simple(token.EOL, source.NewLoc(3, 15)),
token.Simple(token.KwIf, source.NewLoc(4, 2)),
token.New(token.Ident, source.NewLoc(4, 5), "x"),
token.Simple(token.Lt, source.NewLoc(4, 7)),
token.New(token.Ident, source.NewLoc(4, 9), "y"),
token.Simple(token.LBrace, source.NewLoc(4, 11)),
token.Simple(token.EOL, source.NewLoc(4, 12)),
token.New(token.Ident, source.NewLoc(5, 3), "say"),
token.Simple(token.LParen, source.NewLoc(5, 6)),
token.New(token.String, source.NewLoc(5, 7), "x is less than y"),
token.Simple(token.RParen, source.NewLoc(5, 25)),
token.Simple(token.EOL, source.NewLoc(5, 26)),
token.Simple(token.RBrace, source.NewLoc(6, 2)),
token.Simple(token.KwElse, source.NewLoc(6, 4)),
token.Simple(token.LBrace, source.NewLoc(6, 9)),
token.Simple(token.EOL, source.NewLoc(6, 10)),
token.New(token.Ident, source.NewLoc(7, 3), "say"),
token.Simple(token.LParen, source.NewLoc(7, 6)),
token.New(token.String, source.NewLoc(7, 7), "x is greater than or equal to y"),
token.Simple(token.RParen, source.NewLoc(7, 40)),
token.Simple(token.EOL, source.NewLoc(7, 41)),
token.Simple(token.RBrace, source.NewLoc(8, 2)),
token.Simple(token.EOL, source.NewLoc(8, 3)),
token.Simple(token.KwReturn, source.NewLoc(9, 2)),
token.Simple(token.KwTrue, source.NewLoc(9, 9)),
token.Simple(token.EOL, source.NewLoc(9, 13)),
token.Simple(token.RBrace, source.NewLoc(10, 1)),
token.Simple(token.EOF, source.NewLoc(10, 2)),
}
require.Equal(t, expected, tokens)
}
func TestTightIdent(t *testing.T) {
src := `say(message)`
s := scanner.New(strings.NewReader(src))
tokens, err := s.Scan()
require.NoError(t, err)
expected := []token.Token{
token.New(token.Ident, source.NewLoc(0, 0), "say"),
token.Simple(token.LParen, source.NewLoc(0, 3)),
token.New(token.Ident, source.NewLoc(0, 4), "message"),
token.Simple(token.RParen, source.NewLoc(0, 11)),
token.Simple(token.EOF, source.NewLoc(0, 12)),
}
require.Equal(t, expected, tokens)
}
func TestTightNumber(t *testing.T) {
src := `1+2+3`
s := scanner.New(strings.NewReader(src))
tokens, err := s.Scan()
require.NoError(t, err)
expected := []token.Token{
token.New(token.Int, source.NewLoc(0, 0), uint64(1)),
token.Simple(token.Plus, source.NewLoc(0, 1)),
token.New(token.Int, source.NewLoc(0, 2), uint64(2)),
token.Simple(token.Plus, source.NewLoc(0, 3)),
token.New(token.Int, source.NewLoc(0, 4), uint64(3)),
token.Simple(token.EOF, source.NewLoc(0, 5)),
}
require.Equal(t, expected, tokens)
}
func TestNewlineStacking(t *testing.T) {
src := `
x
y
`
s := scanner.New(strings.NewReader(src))
tokens, err := s.Scan()
require.NoError(t, err)
expected := []token.Token{
token.Simple(token.EOL, source.NewLoc(0, 0)),
token.New(token.Ident, source.NewLoc(1, 1), "x"),
token.Simple(token.EOL, source.NewLoc(1, 2)),
token.New(token.Ident, source.NewLoc(4, 1), "y"),
token.Simple(token.EOL, source.NewLoc(4, 2)),
token.Simple(token.EOF, source.NewLoc(5, 1)),
}
require.Equal(t, expected, tokens)
}
func TestEmojiInStrings(t *testing.T) {
src := `
say("🇺🇦" + "❤️!")
`
s := scanner.New(strings.NewReader(src))
tokens, err := s.Scan()
require.NoError(t, err)
expected := []token.Token{
token.Simple(token.EOL, source.NewLoc(0, 0)),
token.New(token.Ident, source.NewLoc(1, 1), "say"),
token.Simple(token.LParen, source.NewLoc(1, 4)),
token.New(token.String, source.NewLoc(1, 5), "🇺🇦"),
token.Simple(token.Plus, source.NewLoc(1, 10)),
token.New(token.String, source.NewLoc(1, 12), "❤️!"),
token.Simple(token.RParen, source.NewLoc(1, 17)),
token.Simple(token.EOL, source.NewLoc(1, 18)),
token.Simple(token.EOF, source.NewLoc(2, 1)),
}
require.Equal(t, expected, tokens)
}
|
