1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
|
use std::{iter::Peekable, str::Chars};
use super::token::{Location, Token, TokenVariant};
pub struct Lexer<'source> {
location: Location,
chars: Peekable<Chars<'source>>,
done: bool,
}
impl Iterator for Lexer<'_> {
type Item = Token;
fn next(&mut self) -> Option<Self::Item> {
if self.done {
return None;
}
if let None = self.chars.peek() {
self.done = true;
return Some(Token {
location: self.location,
variant: TokenVariant::Eof,
});
}
self.skip_whitespace();
let c = *self.chars.peek()?;
let token = if c.is_numeric() {
self.number()
} else if c == '+' {
self.char_token(TokenVariant::OpPlus)
} else if c == '-' {
self.char_token(TokenVariant::OpMinus)
} else if c == '*' {
self.char_token(TokenVariant::OpStar)
} else if c == '/' {
self.char_token(TokenVariant::OpSlash)
} else if c == '!' {
self.char_token(TokenVariant::OpNot)
} else if c == '(' {
self.char_token(TokenVariant::GroupOpen)
} else if c == ')' {
self.char_token(TokenVariant::GroupClose)
} else {
self.char_token(TokenVariant::Unknown(c))
};
Some(token)
}
}
impl<'s> Lexer<'s> {
pub fn new(source: &'s str) -> Self {
Lexer {
location: Location { col: 0, row: 0 },
chars: source.chars().peekable(),
done: false,
}
}
fn advance(&mut self) -> Option<char> {
let next = self.chars.next();
if let Some(c) = next {
if c == '\n' {
self.location.row += 1;
self.location.col = 0;
} else {
self.location.row += 1;
}
}
next
}
fn skip_whitespace(&mut self) {
while self
.chars
.peek()
.map_or(false, |x| x.is_whitespace() && *x != '\n')
{
self.advance();
}
}
fn char_token(&mut self, variant: TokenVariant) -> Token {
let token = Token {
location: self.location,
variant,
};
self.advance();
token
}
fn number(&mut self) -> Token {
let location = self.location;
let mut is_integer = true;
let mut buffer = String::new();
while self
.chars
.peek()
.map_or(false, |&c| c.is_numeric() || c == '.')
{
let c = self.advance().unwrap();
if c == '.' {
is_integer = false;
}
buffer.push(c);
}
let variant = if is_integer {
let int = buffer.parse().expect("Failed lexing integer token.");
TokenVariant::Int(int)
} else {
let float = buffer.parse().expect("Failed lexing float token.");
TokenVariant::Float(float)
};
Token { location, variant }
}
}
|
