From b09a14147d397904722ee7c25e4defc56135b96f Mon Sep 17 00:00:00 2001 From: Mel Date: Mon, 9 May 2022 00:01:02 +0200 Subject: Extract source walk part of scanner --- pkg/lang/scanner/scanner.go | 119 +++++++++++--------------------------------- 1 file changed, 28 insertions(+), 91 deletions(-) (limited to 'pkg/lang/scanner/scanner.go') diff --git a/pkg/lang/scanner/scanner.go b/pkg/lang/scanner/scanner.go index 19e3462..2991083 100644 --- a/pkg/lang/scanner/scanner.go +++ b/pkg/lang/scanner/scanner.go @@ -1,29 +1,24 @@ package scanner import ( - "bufio" - "errors" "io" "jinx/pkg/lang/scanner/token" + "jinx/pkg/libs/source" "strconv" "strings" "unicode" ) type Scanner struct { - source *bufio.Reader - row int - col int + src source.Walker indent int finished bool } -func New(source io.Reader) *Scanner { +func New(src io.Reader) *Scanner { return &Scanner{ - source: bufio.NewReader(source), - row: 0, - col: 0, + src: *source.NewWalker(src), indent: 0, finished: false, } @@ -62,14 +57,14 @@ func (s *Scanner) scanToken() (token.Token, error) { return token.New(token.EOL, firstNewline, nil), nil } - c, eof, err := s.peek() + c, eof, err := s.src.Peek() if err != nil { return token.Token{}, err } if eof { s.finished = true - return token.Simple(token.EOF, s.loc()), nil + return token.Simple(token.EOF, s.src.Loc()), nil } if c == '"' { @@ -80,8 +75,8 @@ func (s *Scanner) scanToken() (token.Token, error) { return s.scanNumber() } - loc := s.loc() - c, _, err = s.next() + loc := s.src.Loc() + c, _, err = s.src.Next() if err != nil { return token.Token{}, err } @@ -91,7 +86,7 @@ func (s *Scanner) scanToken() (token.Token, error) { case '\n': kind = token.EOL case '=': - if cont, err := s.consume('='); cont && err == nil { + if cont, err := s.src.Consume('='); cont && err == nil { kind = token.Eq } else if cont && err != nil { kind = token.Assign @@ -110,7 +105,7 @@ func (s *Scanner) scanToken() (token.Token, error) { kind = token.Percent case '<': - if cont, err := s.consume('='); cont && err == nil { + if cont, err := s.src.Consume('='); cont && err == nil { kind = token.Lte } else if cont && err != nil { kind = token.Lt @@ -118,7 +113,7 @@ func (s *Scanner) scanToken() (token.Token, error) { return token.Token{}, err } case '>': - if cont, err := s.consume('='); cont && err == nil { + if cont, err := s.src.Consume('='); cont && err == nil { kind = token.Gte } else if cont && err != nil { kind = token.Gt @@ -126,7 +121,7 @@ func (s *Scanner) scanToken() (token.Token, error) { return token.Token{}, err } case '!': - if cont, err := s.consume('='); cont && err == nil { + if cont, err := s.src.Consume('='); cont && err == nil { kind = token.Neq } else if cont && err != nil { kind = token.Bang @@ -162,15 +157,15 @@ func (s *Scanner) scanToken() (token.Token, error) { } func (s *Scanner) scanString() (token.Token, error) { - loc := s.loc() - if _, err := s.consume('"'); err != nil { + loc := s.src.Loc() + if _, err := s.src.Consume('"'); err != nil { return token.Token{}, err } var buf strings.Builder for { - c, eof, err := s.next() + c, eof, err := s.src.Next() if err != nil { return token.Token{}, err } @@ -190,12 +185,12 @@ func (s *Scanner) scanString() (token.Token, error) { } func (s *Scanner) scanIdentifierOrKeyword() (token.Token, error) { - loc := s.loc() + loc := s.src.Loc() var buf strings.Builder for { - c, eof, err := s.peek() + c, eof, err := s.src.Peek() if err != nil { return token.Token{}, err } @@ -204,7 +199,7 @@ func (s *Scanner) scanIdentifierOrKeyword() (token.Token, error) { break } - if _, _, err = s.next(); err != nil { + if _, _, err = s.src.Next(); err != nil { return token.Token{}, err } @@ -270,12 +265,12 @@ func (s *Scanner) scanIdentifierOrKeyword() (token.Token, error) { } func (s *Scanner) scanNumber() (token.Token, error) { - loc := s.loc() + loc := s.src.Loc() var buf strings.Builder for { - c, eof, err := s.peek() + c, eof, err := s.src.Peek() if err != nil { return token.Token{}, err } @@ -284,7 +279,7 @@ func (s *Scanner) scanNumber() (token.Token, error) { break } - if _, _, err = s.next(); err != nil { + if _, _, err = s.src.Next(); err != nil { return token.Token{}, err } @@ -299,18 +294,18 @@ func (s *Scanner) scanNumber() (token.Token, error) { return token.New(token.Int, loc, num), nil } -func (s *Scanner) skipWhitespace() (bool, token.Loc, error) { +func (s *Scanner) skipWhitespace() (bool, source.Loc, error) { hadNewline := false - firstNewline := token.Loc{} + firstNewline := source.Loc{} for { - c, eof, err := s.peek() + c, eof, err := s.src.Peek() if err != nil { - return false, token.Loc{}, err + return false, source.Loc{}, err } if c == '\n' && !hadNewline { - firstNewline = s.loc() + firstNewline = s.src.Loc() hadNewline = true } @@ -318,68 +313,10 @@ func (s *Scanner) skipWhitespace() (bool, token.Loc, error) { break } - if _, _, err = s.next(); err != nil { - return false, token.Loc{}, err + if _, _, err = s.src.Next(); err != nil { + return false, source.Loc{}, err } } return hadNewline, firstNewline, nil } - -func (s *Scanner) loc() token.Loc { - return token.Loc{ - Row: s.row, - Col: s.col, - } -} - -func (s *Scanner) next() (rune, bool, error) { - r, _, err := s.source.ReadRune() - if err != nil { - if errors.Is(err, io.EOF) { - return 0, true, nil - } - - return 0, false, err - } - - if r == '\n' { - s.row++ - s.col = 0 - } else { - s.col++ - } - - return r, false, nil -} - -func (s *Scanner) consume(want rune) (bool, error) { - c, _, err := s.next() - if err != nil { - return false, err - } - - if c != want { - return true, ErrUnexpectedChar{ - Expected: want, - Actual: c, - } - } - - return true, nil -} - -func (s *Scanner) peek() (rune, bool, error) { - r, _, err := s.source.ReadRune() - defer s.source.UnreadRune() - - if err != nil { - if errors.Is(err, io.EOF) { - return 0, true, nil - } - - return 0, false, err - } - - return r, false, nil -} -- cgit 1.4.1