featurebase/pql/scanner.go

package pql

import (
	"bufio"
	"bytes"
	"io"
	"unicode"
)

// Scanner represents a PQL lexical scanner.
type Scanner struct {
	r   io.RuneScanner
	pos Pos
}

// NewScanner returns a new instance of Scanner.
func NewScanner(r io.Reader) *Scanner {
	return &Scanner{r: bufio.NewReader(r)}
}

// Scan returns the next token and position from the underlying reader.
func (s *Scanner) Scan() (tok Token, pos Pos, lit string) {
	pos = s.pos

	// Read next code point.
	ch := s.read()

	// If we see whitespace then consume all contiguous whitespace.
	// If we see a letter, or certain acceptable special characters, then consume
	// as an ident or reserved word. If we see quotes, then scan as string.
	if isWhitespace(ch) {
		s.unread()
		return s.scanWhitespace()
	} else if isIdentFirstChar(ch) {
		s.unread()
		return s.scanIdent()
	} else if isDigit(ch) || ch == '-' {
		s.unread()
		return s.scanNumber()
	} else if ch == '"' || ch == '\'' {
		s.unread()
		return s.scanString()
	}

	// Otherwise parse individual characters.
	switch ch {
	case eof:
		tok = EOF
		return
	case '=':
		tok = EQ
	case ',':
		tok = COMMA
	case '(':
		tok = LPAREN
	case ')':
		tok = RPAREN
	case '[':
		tok = LBRACK
	case ']':
		tok = RBRACK
	default:
		tok = ILLEGAL
	}

	lit = string(ch)
	return
}

// read returns the next code point from the underlying reader and updates the pos.
func (s *Scanner) read() rune {
	// Read next rune from underlying reader.
	ch, _, err := s.r.ReadRune()
	if err != nil {
		return eof
	}

	// Update position information.
	if ch == '\n' {
		s.pos.Line++
		s.pos.Char = 0
	} else {
		s.pos.Char++
	}

	return ch
}

// unread pushes the previously read rune back onto the reader.
func (s *Scanner) unread() {
	if s.pos.Char == 0 {
		s.pos.Line--
	} else {
		s.pos.Char--
	}

	s.r.UnreadRune()
}

// scanWhitespace consumes the current rune and all contiguous whitespace.
func (s *Scanner) scanWhitespace() (tok Token, pos Pos, lit string) {
	pos = s.pos

	var buf bytes.Buffer
	for {
		ch := s.read()
		if ch == eof {
			break
		} else if !isWhitespace(ch) {
			s.unread()
			break
		}
		buf.WriteRune(ch)
	}

	return WS, pos, buf.String()
}

func (s *Scanner) scanIdent() (tok Token, pos Pos, lit string) {
	pos = s.pos

	var buf bytes.Buffer
	for {
		ch := s.read()
		if ch == eof {
			break
		} else if !isIdentChar(ch) {
			s.unread()
			break
		}
		buf.WriteRune(ch)
	}
	lit = buf.String()

	// If the literal matches a keyword then return that keyword.
	if tok = Lookup(lit); tok != IDENT {
		return tok, pos, lit
	}

	return IDENT, pos, lit
}

// scanNumber consumes consecutive digits, optionally starting with a minus sign and up to one '.' character.
func (s *Scanner) scanNumber() (tok Token, pos Pos, lit string) {
	pos = s.pos
	tok = INTEGER

	var buf bytes.Buffer
	var seenDot bool
	first := true
	for {
		ch := s.read()
		if !isDigit(ch) && !(first && ch == '-') && (seenDot || ch != '.') {
			s.unread()
			break
		}
		if ch == '.' {
			seenDot = true
			tok = FLOAT
		}
		buf.WriteRune(ch)
		first = false
	}
	return tok, pos, buf.String()
}

// scanString consumes a single-quoted or double-quoted string.
func (s *Scanner) scanString() (tok Token, pos Pos, lit string) {
	pos = s.pos

	// This must be either a single- or double-quote.
	ending := s.read()

	var buf bytes.Buffer
	for {
		ch := s.read()
		if ch == ending {
			break
		} else if ch == '\n' || ch == eof {
			return BADSTRING, pos, buf.String()
		} else if ch == '\\' {
			next := s.read()
			if next == 'n' {
				buf.WriteRune('\n')
			} else if next == '\\' {
				buf.WriteRune('\\')
			} else if next == '"' {
				buf.WriteRune('"')
			} else if next == '\'' {
				buf.WriteRune('\'')
			} else {
				return BADSTRING, pos, buf.String()
			}
		} else {
			buf.WriteRune(ch)
		}
	}

	return STRING, pos, buf.String()
}

// bufScanner represents a wrapper for scanner to add a buffer.
// It provides a fixed-length circular buffer that can be unread.
type bufScanner struct {
	s   *Scanner
	i   int // buffer index
	n   int // buffer size
	buf [8]struct {
		tok Token
		pos Pos
		lit string
	}
}

// newBufScanner returns a new buffered scanner for a reader.
func newBufScanner(r io.Reader) *bufScanner {
	return &bufScanner{s: NewScanner(r)}
}

// Scan reads the next token from the scanner.
func (s *bufScanner) Scan() (tok Token, pos Pos, lit string) {
	// If we have unread tokens then read them off the buffer first.
	if s.n > 0 {
		s.n--
		return s.curr()
	}

	// Move buffer position forward and save the token.
	s.i = (s.i + 1) % len(s.buf)
	buf := &s.buf[s.i]
	buf.tok, buf.pos, buf.lit = s.s.Scan()

	return s.curr()
}

// unscan pushes the previously token back onto the buffer.
func (s *bufScanner) unscan() { s.n++ }

// curr returns the last read token.
func (s *bufScanner) curr() (tok Token, pos Pos, lit string) {
	buf := &s.buf[(s.i-s.n+len(s.buf))%len(s.buf)]
	return buf.tok, buf.pos, buf.lit
}

// pos returns the current position.
func (s *bufScanner) pos() Pos {
	_, pos, _ := s.curr()
	return pos
}

// isWhitespace returns true if the rune a Unicode space character.
func isWhitespace(ch rune) bool { return unicode.IsSpace(ch) }

// isLetter returns true if the rune is a letter.
func isLetter(ch rune) bool { return (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') }

// isDigit returns true if the rune is a digit.
func isDigit(ch rune) bool { return (ch >= '0' && ch <= '9') }

// isIdentChar returns true if the rune can be used in an unquoted identifier.
func isIdentChar(ch rune) bool {
	return isLetter(ch) || isDigit(ch) || ch == '_' || ch == '-' || ch == '.'
}

// isIdentFirstChar returns true if the rune can be used as the first char in an identifier.
func isIdentFirstChar(ch rune) bool { return isLetter(ch) }

const eof = rune(0)