mirror of
https://github.com/featurebasedb/featurebase.git
synced 2026-08-28 10:54:59 +00:00
replace PQL parser with one created by PEG parser generator
This commit is contained in:
Matt Jaffee
parent
9ad8d8d0b9
commit
47233c8bee
10 changed files with 1747 additions and 722 deletions
15
Makefile
15
Makefile
|
|
@ -1,4 +1,4 @@
|
|||
.PHONY: build check-clean clean cover cover-viz default docker docker-build docker-test generate generate-protoc install install-build-deps install-dep install-protoc install-protoc-gen-gofast prerelease prerelease-build prerelease-upload release release-build require-dep require-protoc require-protoc-gen-gofast test
|
||||
.PHONY: build check-clean clean cover cover-viz default docker docker-build docker-test generate generate-protoc generate-pql install install-build-deps install-dep install-protoc install-protoc-gen-gofast install-peg prerelease prerelease-build prerelease-upload release release-build require-dep require-protoc require-protoc-gen-gofast require-peg test
|
||||
|
||||
CLONE_URL=github.com/pilosa/pilosa
|
||||
VERSION := $(shell git describe --tags 2> /dev/null || echo unknown)
|
||||
|
|
@ -92,8 +92,11 @@ generate-protoc: require-protoc require-protoc-gen-gofast
|
|||
generate-stringer:
|
||||
go generate github.com/pilosa/pilosa
|
||||
|
||||
generate-pql: require-peg
|
||||
cd pql && peg -inline -switch pql.peg && cd ..
|
||||
|
||||
# `go generate` all needed packages
|
||||
generate: generate-protoc generate-stringer
|
||||
generate: generate-protoc generate-stringer generate-pql
|
||||
|
||||
# Create Docker image from Dockerfile
|
||||
docker:
|
||||
|
|
@ -128,7 +131,10 @@ require-protoc-gen-gofast:
|
|||
require-protoc:
|
||||
$(call require,protoc)
|
||||
|
||||
install-build-deps: install-dep install-protoc-gen-gofast install-protoc install-stringer
|
||||
require-peg:
|
||||
$(call require,peg)
|
||||
|
||||
install-build-deps: install-dep install-protoc-gen-gofast install-protoc install-stringer install-peg
|
||||
|
||||
install-dep:
|
||||
go get -u github.com/golang/dep/cmd/dep
|
||||
|
|
@ -141,3 +147,6 @@ install-protoc-gen-gofast:
|
|||
|
||||
install-protoc:
|
||||
@echo This tool cannot automatically install protoc. Please download and install protoc from https://google.github.io/proto-lens/installing-protoc.html
|
||||
|
||||
install-peg:
|
||||
go get github.com/pointlander/peg
|
||||
|
|
|
|||
|
|
@ -653,8 +653,8 @@ func TestHandler_Query_ErrParse(t *testing.T) {
|
|||
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1", strings.NewReader("bad_fn(")))
|
||||
if w.Code != gohttp.StatusBadRequest {
|
||||
t.Fatalf("unexpected status code: %d", w.Code)
|
||||
} else if body := w.Body.String(); body != `{"error":"parsing: expected comma, right paren, or identifier, found \"\" occurred at line 1, char 8"}`+"\n" {
|
||||
t.Fatalf("unexpected body: %s", body)
|
||||
} else if body := w.Body.String(); body != `{"error":"parsing: parsing: \nparse error near PegText (line 1 symbol 1 - line 1 symbol 4):\n\"bad\"\n"}`+"\n" {
|
||||
t.Fatalf("unexpected body: \n%s", body)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
138
pql/ast.go
138
pql/ast.go
|
|
@ -26,6 +26,144 @@ import (
|
|||
// Query represents a PQL query.
|
||||
type Query struct {
|
||||
Calls []*Call
|
||||
|
||||
lastField string
|
||||
lastCond Token
|
||||
inList bool
|
||||
callStack []*Call
|
||||
}
|
||||
|
||||
func (q *Query) startCall(name string) {
|
||||
newCall := &Call{Name: name}
|
||||
q.callStack = append(q.callStack, newCall)
|
||||
|
||||
if len(q.callStack) == 1 {
|
||||
q.Calls = append(q.Calls, newCall)
|
||||
} else {
|
||||
calls := q.callStack[len(q.callStack)-2].Children
|
||||
q.callStack[len(q.callStack)-2].Children = append(calls, newCall)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func (q *Query) endCall() {
|
||||
q.callStack = q.callStack[:len(q.callStack)-1]
|
||||
}
|
||||
|
||||
func (q *Query) addField(field string) {
|
||||
if q.lastField != "" {
|
||||
panic(fmt.Sprintf("addField called with '%s' while field is not empty, it's: %s", field, q.lastField))
|
||||
}
|
||||
q.lastField = field
|
||||
call := q.callStack[len(q.callStack)-1]
|
||||
if call.Args == nil {
|
||||
call.Args = make(map[string]interface{})
|
||||
}
|
||||
}
|
||||
|
||||
func (q *Query) addVal(val interface{}) {
|
||||
if q.lastField == "" {
|
||||
panic(fmt.Sprintf("addVal called with '%s' when lastField is empty", val))
|
||||
}
|
||||
call := q.callStack[len(q.callStack)-1]
|
||||
if q.inList {
|
||||
list := call.Args[q.lastField].([]interface{})
|
||||
call.Args[q.lastField] = append(list, val)
|
||||
return
|
||||
}
|
||||
if q.lastCond != ILLEGAL {
|
||||
if val != nil || q.lastCond != NEQ {
|
||||
panic(fmt.Sprintf("can't add val %s with condition %s", val, q.lastCond))
|
||||
}
|
||||
call.Args[q.lastField] = &Condition{
|
||||
Op: NEQ,
|
||||
Value: val,
|
||||
}
|
||||
} else {
|
||||
call.Args[q.lastField] = val
|
||||
}
|
||||
q.lastField = ""
|
||||
q.lastCond = ILLEGAL
|
||||
}
|
||||
|
||||
func (q *Query) addNumVal(val string) {
|
||||
if q.lastField == "" {
|
||||
panic(fmt.Sprintf("addIntVal called with '%s' when lastField is empty", val))
|
||||
}
|
||||
var ival interface{}
|
||||
var err error
|
||||
if strings.Contains(val, ".") {
|
||||
ival, err = strconv.ParseFloat(val, 64)
|
||||
} else {
|
||||
ival, err = strconv.ParseInt(val, 10, 64)
|
||||
}
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
call := q.callStack[len(q.callStack)-1]
|
||||
if q.inList {
|
||||
if q.lastCond != ILLEGAL {
|
||||
list := call.Args[q.lastField].(*Condition).Value.([]interface{})
|
||||
call.Args[q.lastField] = &Condition{
|
||||
Op: q.lastCond,
|
||||
Value: append(list, ival),
|
||||
}
|
||||
} else {
|
||||
list := call.Args[q.lastField].([]interface{})
|
||||
call.Args[q.lastField] = append(list, ival)
|
||||
}
|
||||
return
|
||||
} else if q.lastCond != ILLEGAL {
|
||||
call.Args[q.lastField] = &Condition{
|
||||
Op: q.lastCond,
|
||||
Value: ival,
|
||||
}
|
||||
} else {
|
||||
call.Args[q.lastField] = ival
|
||||
}
|
||||
q.lastField = ""
|
||||
q.lastCond = ILLEGAL
|
||||
}
|
||||
|
||||
func (q *Query) startList() {
|
||||
call := q.callStack[len(q.callStack)-1]
|
||||
if q.lastCond != ILLEGAL {
|
||||
call.Args[q.lastField] = &Condition{
|
||||
Op: q.lastCond,
|
||||
Value: make([]interface{}, 0),
|
||||
}
|
||||
} else {
|
||||
call.Args[q.lastField] = make([]interface{}, 0)
|
||||
}
|
||||
q.inList = true
|
||||
}
|
||||
|
||||
func (q *Query) endList() {
|
||||
q.inList = false
|
||||
q.lastField = ""
|
||||
q.lastCond = ILLEGAL
|
||||
}
|
||||
|
||||
func (q *Query) addGT() {
|
||||
q.lastCond = GT
|
||||
}
|
||||
func (q *Query) addLT() {
|
||||
q.lastCond = LT
|
||||
}
|
||||
func (q *Query) addGTE() {
|
||||
q.lastCond = GTE
|
||||
}
|
||||
func (q *Query) addLTE() {
|
||||
q.lastCond = LTE
|
||||
}
|
||||
func (q *Query) addEQ() {
|
||||
q.lastCond = EQ
|
||||
}
|
||||
func (q *Query) addNEQ() {
|
||||
q.lastCond = NEQ
|
||||
}
|
||||
func (q *Query) addBTWN() {
|
||||
q.lastCond = BETWEEN
|
||||
}
|
||||
|
||||
// WriteCallN returns the number of mutating calls.
|
||||
|
|
|
|||
299
pql/parser.go
299
pql/parser.go
|
|
@ -15,10 +15,11 @@
|
|||
package pql
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"strconv"
|
||||
"io/ioutil"
|
||||
"strings"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
// TimeFormat is the go-style time format used to parse string dates.
|
||||
|
|
@ -26,13 +27,16 @@ const TimeFormat = "2006-01-02T15:04"
|
|||
|
||||
// Parser represents a parser for the PQL language.
|
||||
type Parser struct {
|
||||
scanner *bufScanner
|
||||
r io.Reader
|
||||
//scanner *bufScanner
|
||||
PQL
|
||||
}
|
||||
|
||||
// NewParser returns a new instance of Parser.
|
||||
func NewParser(r io.Reader) *Parser {
|
||||
return &Parser{
|
||||
scanner: newBufScanner(r),
|
||||
r: r,
|
||||
// scanner: newBufScanner(r),
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -43,287 +47,18 @@ func ParseString(s string) (*Query, error) {
|
|||
|
||||
// Parse parses the next node in the query.
|
||||
func (p *Parser) Parse() (*Query, error) {
|
||||
q := &Query{}
|
||||
for {
|
||||
call, err := p.parseCall()
|
||||
if err == io.EOF {
|
||||
break
|
||||
} else if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
q.Calls = append(q.Calls, call)
|
||||
}
|
||||
|
||||
// Require at least one call.
|
||||
if len(q.Calls) == 0 {
|
||||
return nil, io.ErrUnexpectedEOF
|
||||
}
|
||||
|
||||
return q, nil
|
||||
}
|
||||
|
||||
// parseCall parses the next function call.
|
||||
func (p *Parser) parseCall() (*Call, error) {
|
||||
var c Call
|
||||
|
||||
// Read call name.
|
||||
tok, pos, lit := p.scanIgnoreWhitespace()
|
||||
if tok == EOF {
|
||||
return nil, io.EOF
|
||||
} else if tok != IDENT {
|
||||
return nil, &ParseError{Message: fmt.Sprintf("expected identifier, found: %s", lit), Pos: pos}
|
||||
}
|
||||
c.Name = lit
|
||||
|
||||
// Scan opening parenthesis.
|
||||
if err := p.expect(LPAREN); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Parse children first.
|
||||
children, err := p.parseChildren()
|
||||
buf, err := ioutil.ReadAll(p.r)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, errors.Wrap(err, "reading buffer to parse")
|
||||
}
|
||||
c.Children = children
|
||||
|
||||
// If next token is a closing paren then exit.
|
||||
if tok, pos, lit := p.scanIgnoreWhitespace(); tok == RPAREN {
|
||||
return &c, nil
|
||||
} else if tok == IDENT {
|
||||
p.unscan(1)
|
||||
} else if tok != COMMA {
|
||||
return nil, parseErrorf(pos, "expected comma, right paren, or identifier, found %q", lit)
|
||||
p.PQL = PQL{
|
||||
Buffer: string(buf),
|
||||
}
|
||||
|
||||
// Parse key/value arguments.
|
||||
args, err := p.parseArgs()
|
||||
p.Init()
|
||||
err = p.PQL.Parse()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
c.Args = args
|
||||
|
||||
// Scan closing parenthesis.
|
||||
if err := p.expect(RPAREN); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &c, nil
|
||||
}
|
||||
|
||||
// parseChildren parses call children.
|
||||
func (p *Parser) parseChildren() ([]*Call, error) {
|
||||
var offset int
|
||||
var children []*Call
|
||||
for {
|
||||
// Ensure next two tokens are IDENT+LPAREN.
|
||||
if tok, _, _ := p.scanIgnoreWhitespace(); tok != IDENT {
|
||||
p.unscanIgnoreWhitespace(1 + offset)
|
||||
return children, nil
|
||||
}
|
||||
if tok, _, _ := p.scan(); tok != LPAREN {
|
||||
p.unscanIgnoreWhitespace(2 + offset)
|
||||
return children, nil
|
||||
}
|
||||
|
||||
// Push tokens back on scanner and parse as a call.
|
||||
p.unscan(2)
|
||||
child, err := p.parseCall()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
children = append(children, child)
|
||||
|
||||
// Exit if closing paren.
|
||||
if tok, pos, lit := p.scanIgnoreWhitespace(); tok == RPAREN {
|
||||
p.unscan(1)
|
||||
return children, nil
|
||||
} else if tok != COMMA {
|
||||
return nil, parseErrorf(pos, "expected comma or right paren, found %q", lit)
|
||||
}
|
||||
|
||||
// Make sure comma is unscanned.
|
||||
offset = 1
|
||||
}
|
||||
}
|
||||
|
||||
// parseArgs parses key/value arguments.
|
||||
func (p *Parser) parseArgs() (map[string]interface{}, error) {
|
||||
args := make(map[string]interface{})
|
||||
for {
|
||||
// Parse key.
|
||||
tok, pos, lit := p.scanIgnoreWhitespace()
|
||||
if tok == RPAREN {
|
||||
p.unscan(1)
|
||||
return args, nil
|
||||
} else if tok != IDENT {
|
||||
return nil, parseErrorf(pos, "expected argument key, found %q", lit)
|
||||
}
|
||||
key := lit
|
||||
|
||||
// Expect '=' or a comparison next.
|
||||
var op Token
|
||||
switch tok, pos, lit := p.scanIgnoreWhitespace(); tok {
|
||||
case ASSIGN:
|
||||
case EQ, NEQ, LT, LTE, GT, GTE, BETWEEN:
|
||||
op = tok
|
||||
default:
|
||||
return nil, parseErrorf(pos, "expected equals sign or comparison operator, found %q", lit)
|
||||
}
|
||||
|
||||
// Parse value.
|
||||
var value interface{}
|
||||
tok, pos, lit = p.scanIgnoreWhitespace()
|
||||
switch tok {
|
||||
case IDENT:
|
||||
if lit == "true" {
|
||||
value = true
|
||||
} else if lit == "false" {
|
||||
value = false
|
||||
} else if lit == "null" {
|
||||
value = nil
|
||||
} else {
|
||||
value = lit
|
||||
}
|
||||
case STRING:
|
||||
value = lit
|
||||
case INTEGER:
|
||||
v, err := strconv.ParseInt(lit, 10, 64)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
value = v
|
||||
case FLOAT:
|
||||
v, err := strconv.ParseFloat(lit, 64)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
value = v
|
||||
case LBRACK:
|
||||
v, err := p.parseList()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
value = v
|
||||
default:
|
||||
return nil, parseErrorf(pos, "invalid argument value: %q", lit)
|
||||
}
|
||||
|
||||
// Ensure key doesn't already exist.
|
||||
if _, ok := args[key]; ok {
|
||||
return nil, parseErrorf(pos, "argument key already used: %s", key)
|
||||
}
|
||||
|
||||
// If op is specified then create a condition.
|
||||
if op != 0 {
|
||||
value = &Condition{Op: op, Value: value}
|
||||
}
|
||||
|
||||
// Add key/value pair to arguments.
|
||||
args[key] = value
|
||||
|
||||
// Exit if closing paren.
|
||||
if tok, pos, lit := p.scanIgnoreWhitespace(); tok == RPAREN {
|
||||
p.unscan(1)
|
||||
return args, nil
|
||||
} else if tok != COMMA {
|
||||
return nil, parseErrorf(pos, "expected comma or right paren, found %q", lit)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// parseList parses a list of primitives. This is used by the TopN() filters.
|
||||
func (p *Parser) parseList() ([]interface{}, error) {
|
||||
var values []interface{}
|
||||
for {
|
||||
// Read next value.
|
||||
tok, pos, lit := p.scanIgnoreWhitespace()
|
||||
switch tok {
|
||||
case IDENT:
|
||||
if lit == "true" {
|
||||
values = append(values, true)
|
||||
} else if lit == "false" {
|
||||
values = append(values, false)
|
||||
} else {
|
||||
values = append(values, lit)
|
||||
}
|
||||
case STRING:
|
||||
values = append(values, lit)
|
||||
case INTEGER:
|
||||
v, err := strconv.ParseInt(lit, 10, 64)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
values = append(values, v)
|
||||
default:
|
||||
return nil, parseErrorf(pos, "invalid list value: %q", lit)
|
||||
}
|
||||
|
||||
// Expect a comma or closing bracket next.
|
||||
if tok, pos, lit := p.scanIgnoreWhitespace(); tok == RBRACK {
|
||||
break
|
||||
} else if tok != COMMA {
|
||||
return nil, parseErrorf(pos, "expected comma, found %q", lit)
|
||||
}
|
||||
}
|
||||
return values, nil
|
||||
}
|
||||
|
||||
// scan returns the next token from the scanner.
|
||||
func (p *Parser) scan() (tok Token, pos Pos, lit string) { return p.scanner.Scan() }
|
||||
|
||||
// scanIgnoreWhitespace returns the next non-whitespace token from the scanner.
|
||||
func (p *Parser) scanIgnoreWhitespace() (tok Token, pos Pos, lit string) {
|
||||
tok, pos, lit = p.scan()
|
||||
if tok == WS {
|
||||
tok, pos, lit = p.scan()
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// unscan returns the last n tokens back to the scanner.
|
||||
func (p *Parser) unscan(n int) {
|
||||
for i := 0; i < n; i++ {
|
||||
p.scanner.unscan()
|
||||
}
|
||||
}
|
||||
|
||||
// unscanIgnoreWhitespace returns the last n non-WS tokens back to the scanner.
|
||||
func (p *Parser) unscanIgnoreWhitespace(n int) {
|
||||
for i := 0; i < n; {
|
||||
p.scanner.unscan()
|
||||
if tok, _, _ := p.scanner.curr(); tok != WS {
|
||||
i++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// expect returns an error if the next token is not exp.
|
||||
func (p *Parser) expect(exp Token) error {
|
||||
if tok, pos, lit := p.scan(); tok != exp {
|
||||
return parseErrorf(pos, "expected %s, found %q", exp.String(), lit)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// pos returns the current position.
|
||||
func (p *Parser) pos() Pos { return p.scanner.pos() }
|
||||
|
||||
// ParseError represents an error that occurred while parsing a PQL query.
|
||||
type ParseError struct {
|
||||
Message string
|
||||
Pos Pos
|
||||
}
|
||||
|
||||
// Error returns a string representation of e.
|
||||
func (e *ParseError) Error() string {
|
||||
return fmt.Sprintf("%s occurred at line %d, char %d", e.Message, e.Pos.Line+1, e.Pos.Char+1)
|
||||
}
|
||||
|
||||
// parseErrorf returns a formatted parse error.
|
||||
func parseErrorf(pos Pos, format string, args ...interface{}) *ParseError {
|
||||
return &ParseError{
|
||||
Message: fmt.Sprintf(format, args...),
|
||||
Pos: pos,
|
||||
return nil, errors.Wrap(err, "parsing")
|
||||
}
|
||||
p.Execute()
|
||||
return &p.Query, nil
|
||||
}
|
||||
|
|
|
|||
44
pql/pql.peg
Normal file
44
pql/pql.peg
Normal file
|
|
@ -0,0 +1,44 @@
|
|||
package pql
|
||||
|
||||
type PQL Peg {
|
||||
Query
|
||||
}
|
||||
|
||||
|
||||
Calls <- Call* !.
|
||||
Call <- newline* < [[A-Z]]+ > { p.startCall(buffer[begin:end] ) } open args close newline* { p.endCall() }
|
||||
args <- arg (comma args)? sp / sp
|
||||
arg <- ( Call
|
||||
/ field sp '=' sp value
|
||||
/ field sp COND sp value
|
||||
)
|
||||
COND <- ( '><' { p.addBTWN() }
|
||||
/ '<=' { p.addLTE() }
|
||||
/ '>=' { p.addGTE() }
|
||||
/ '==' { p.addEQ() }
|
||||
/ '!=' { p.addNEQ() }
|
||||
/ '<' { p.addLT() }
|
||||
/ '>' { p.addGT() }
|
||||
)
|
||||
open <- '(' sp
|
||||
value <- ( item
|
||||
/ lbrack { p.startList() } list rbrack { p.endList() }
|
||||
)
|
||||
list <- item (comma list)?
|
||||
item <- ( 'null' { p.addVal(nil) }
|
||||
/ 'true' { p.addVal(true) }
|
||||
/ 'false' { p.addVal(false) }
|
||||
/ < '-'? [0-9]+ ('.'[0-9]*)? > { p.addNumVal(buffer[begin:end]) }
|
||||
/ < '-'? '.'[0-9]+ > { p.addNumVal(buffer[begin:end]) }
|
||||
/ < ([[A-Z]] / [0-9] / '-' / '_' / ':')+ > { p.addVal(buffer[begin:end]) }
|
||||
/ '"' < ([[A-Z]] / [0-9] / '-' / '_' / ':')+ > '"' { p.addVal(buffer[begin:end]) }
|
||||
/ '\'' < ([[A-Z]] / [0-9] / '-' / '_' / ':')+ > '\'' { p.addVal(buffer[begin:end]) }
|
||||
)
|
||||
|
||||
field <- < [[A-Z]] ( [[A-Z]] / [0-9] / '_' )* > { p.addField(buffer[begin:end]) }
|
||||
close <- ')' sp
|
||||
sp <- ( ' ' / '\t' )*
|
||||
comma <- sp ',' sp
|
||||
lbrack <- '[' sp
|
||||
rbrack <- sp ']' sp
|
||||
newline <- sp '\n' sp
|
||||
1518
pql/pql.peg.go
Normal file
1518
pql/pql.peg.go
Normal file
File diff suppressed because it is too large
Load diff
16
pql/pqlpeg_test.go
Normal file
16
pql/pqlpeg_test.go
Normal file
|
|
@ -0,0 +1,16 @@
|
|||
package pql
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestPEG(t *testing.T) {
|
||||
p := PQL{Buffer: `
|
||||
SetBit(Union(Zitmap(row==4), Intersect(Qitmap(blah>4), Ritmap(field="zoo9")), Hitmap(row=ag-bee)), a="4z", b=5) Count(Union(Witmap(row=5.73, frame=.10), Range(zztop><[2, 9]))) TopN(fields=["hello", "goodbye", "zero"])`[1:]}
|
||||
p.Init()
|
||||
err := p.Parse()
|
||||
if err != nil {
|
||||
t.Fatalf("parse error: %v", err)
|
||||
}
|
||||
p.Execute()
|
||||
}
|
||||
303
pql/scanner.go
303
pql/scanner.go
|
|
@ -1,303 +0,0 @@
|
|||
// Copyright 2017 Pilosa Corp.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package pql
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"io"
|
||||
"unicode"
|
||||
)
|
||||
|
||||
// Scanner represents a PQL lexical scanner.
|
||||
type Scanner struct {
|
||||
r io.RuneScanner
|
||||
pos Pos
|
||||
}
|
||||
|
||||
// NewScanner returns a new instance of Scanner.
|
||||
func NewScanner(r io.Reader) *Scanner {
|
||||
return &Scanner{r: bufio.NewReader(r)}
|
||||
}
|
||||
|
||||
// Scan returns the next token and position from the underlying reader.
|
||||
func (s *Scanner) Scan() (tok Token, pos Pos, lit string) {
|
||||
pos = s.pos
|
||||
|
||||
// Read next code point.
|
||||
ch := s.read()
|
||||
|
||||
// If we see whitespace then consume all contiguous whitespace.
|
||||
// If we see a letter, or certain acceptable special characters, then consume
|
||||
// as an ident or reserved word. If we see quotes, then scan as string.
|
||||
if isWhitespace(ch) {
|
||||
s.unread()
|
||||
return s.scanWhitespace()
|
||||
} else if isIdentFirstChar(ch) {
|
||||
s.unread()
|
||||
return s.scanIdent()
|
||||
} else if isDigit(ch) || ch == '-' {
|
||||
s.unread()
|
||||
return s.scanNumber()
|
||||
} else if ch == '"' || ch == '\'' {
|
||||
s.unread()
|
||||
return s.scanString()
|
||||
}
|
||||
|
||||
// Otherwise parse individual characters.
|
||||
switch ch {
|
||||
case eof:
|
||||
return EOF, pos, ""
|
||||
case '=':
|
||||
if next := s.read(); next == '=' {
|
||||
return EQ, pos, "=="
|
||||
}
|
||||
s.unread()
|
||||
return ASSIGN, pos, string(ch)
|
||||
case '!':
|
||||
if next := s.read(); next == '=' {
|
||||
return NEQ, pos, "!="
|
||||
}
|
||||
s.unread()
|
||||
return ASSIGN, pos, string(ch)
|
||||
case '<':
|
||||
if next := s.read(); next == '=' {
|
||||
return LTE, pos, "<="
|
||||
}
|
||||
s.unread()
|
||||
return LT, pos, string(ch)
|
||||
case '>':
|
||||
next := s.read()
|
||||
if next == '=' {
|
||||
return GTE, pos, ">="
|
||||
} else if next == '<' {
|
||||
return BETWEEN, pos, "><"
|
||||
}
|
||||
s.unread()
|
||||
return GT, pos, string(ch)
|
||||
case ',':
|
||||
return COMMA, pos, string(ch)
|
||||
case '(':
|
||||
return LPAREN, pos, string(ch)
|
||||
case ')':
|
||||
return RPAREN, pos, string(ch)
|
||||
case '[':
|
||||
return LBRACK, pos, string(ch)
|
||||
case ']':
|
||||
return RBRACK, pos, string(ch)
|
||||
default:
|
||||
return ILLEGAL, pos, string(ch)
|
||||
}
|
||||
}
|
||||
|
||||
// read returns the next code point from the underlying reader and updates the pos.
|
||||
func (s *Scanner) read() rune {
|
||||
// Read next rune from underlying reader.
|
||||
ch, _, err := s.r.ReadRune()
|
||||
if err != nil {
|
||||
return eof
|
||||
}
|
||||
|
||||
// Update position information.
|
||||
if ch == '\n' {
|
||||
s.pos.Line++
|
||||
s.pos.Char = 0
|
||||
} else {
|
||||
s.pos.Char++
|
||||
}
|
||||
|
||||
return ch
|
||||
}
|
||||
|
||||
// unread pushes the previously read rune back onto the reader.
|
||||
func (s *Scanner) unread() {
|
||||
if s.pos.Char == 0 {
|
||||
s.pos.Line--
|
||||
} else {
|
||||
s.pos.Char--
|
||||
}
|
||||
|
||||
s.r.UnreadRune()
|
||||
}
|
||||
|
||||
// scanWhitespace consumes the current rune and all contiguous whitespace.
|
||||
func (s *Scanner) scanWhitespace() (tok Token, pos Pos, lit string) {
|
||||
pos = s.pos
|
||||
|
||||
var buf bytes.Buffer
|
||||
for {
|
||||
ch := s.read()
|
||||
if ch == eof {
|
||||
break
|
||||
} else if !isWhitespace(ch) {
|
||||
s.unread()
|
||||
break
|
||||
}
|
||||
buf.WriteRune(ch)
|
||||
}
|
||||
|
||||
return WS, pos, buf.String()
|
||||
}
|
||||
|
||||
func (s *Scanner) scanIdent() (tok Token, pos Pos, lit string) {
|
||||
pos = s.pos
|
||||
|
||||
var buf bytes.Buffer
|
||||
for {
|
||||
ch := s.read()
|
||||
if ch == eof {
|
||||
break
|
||||
} else if !isIdentChar(ch) {
|
||||
s.unread()
|
||||
break
|
||||
}
|
||||
buf.WriteRune(ch)
|
||||
}
|
||||
lit = buf.String()
|
||||
|
||||
// If the literal matches a keyword then return that keyword.
|
||||
if tok = Lookup(lit); tok != IDENT {
|
||||
return tok, pos, lit
|
||||
}
|
||||
|
||||
return IDENT, pos, lit
|
||||
}
|
||||
|
||||
// scanNumber consumes consecutive digits, optionally starting with a minus sign and up to one '.' character.
|
||||
func (s *Scanner) scanNumber() (tok Token, pos Pos, lit string) {
|
||||
pos = s.pos
|
||||
tok = INTEGER
|
||||
|
||||
var buf bytes.Buffer
|
||||
var seenDot bool
|
||||
first := true
|
||||
for {
|
||||
ch := s.read()
|
||||
if !isDigit(ch) && !(first && ch == '-') && (seenDot || ch != '.') {
|
||||
s.unread()
|
||||
break
|
||||
}
|
||||
if ch == '.' {
|
||||
seenDot = true
|
||||
tok = FLOAT
|
||||
}
|
||||
buf.WriteRune(ch)
|
||||
first = false
|
||||
}
|
||||
return tok, pos, buf.String()
|
||||
}
|
||||
|
||||
// scanString consumes a single-quoted or double-quoted string.
|
||||
func (s *Scanner) scanString() (tok Token, pos Pos, lit string) {
|
||||
pos = s.pos
|
||||
|
||||
// This must be either a single- or double-quote.
|
||||
ending := s.read()
|
||||
|
||||
var buf bytes.Buffer
|
||||
for {
|
||||
ch := s.read()
|
||||
if ch == ending {
|
||||
break
|
||||
} else if ch == '\n' || ch == eof {
|
||||
return BADSTRING, pos, buf.String()
|
||||
} else if ch == '\\' {
|
||||
next := s.read()
|
||||
if next == 'n' {
|
||||
buf.WriteRune('\n')
|
||||
} else if next == '\\' {
|
||||
buf.WriteRune('\\')
|
||||
} else if next == '"' {
|
||||
buf.WriteRune('"')
|
||||
} else if next == '\'' {
|
||||
buf.WriteRune('\'')
|
||||
} else {
|
||||
return BADSTRING, pos, buf.String()
|
||||
}
|
||||
} else {
|
||||
buf.WriteRune(ch)
|
||||
}
|
||||
}
|
||||
|
||||
return STRING, pos, buf.String()
|
||||
}
|
||||
|
||||
// bufScanner represents a wrapper for scanner to add a buffer.
|
||||
// It provides a fixed-length circular buffer that can be unread.
|
||||
type bufScanner struct {
|
||||
s *Scanner
|
||||
i int // buffer index
|
||||
n int // buffer size
|
||||
buf [8]struct {
|
||||
tok Token
|
||||
pos Pos
|
||||
lit string
|
||||
}
|
||||
}
|
||||
|
||||
// newBufScanner returns a new buffered scanner for a reader.
|
||||
func newBufScanner(r io.Reader) *bufScanner {
|
||||
return &bufScanner{s: NewScanner(r)}
|
||||
}
|
||||
|
||||
// Scan reads the next token from the scanner.
|
||||
func (s *bufScanner) Scan() (tok Token, pos Pos, lit string) {
|
||||
// If we have unread tokens then read them off the buffer first.
|
||||
if s.n > 0 {
|
||||
s.n--
|
||||
return s.curr()
|
||||
}
|
||||
|
||||
// Move buffer position forward and save the token.
|
||||
s.i = (s.i + 1) % len(s.buf)
|
||||
buf := &s.buf[s.i]
|
||||
buf.tok, buf.pos, buf.lit = s.s.Scan()
|
||||
|
||||
return s.curr()
|
||||
}
|
||||
|
||||
// unscan pushes the previously token back onto the buffer.
|
||||
func (s *bufScanner) unscan() { s.n++ }
|
||||
|
||||
// curr returns the last read token.
|
||||
func (s *bufScanner) curr() (tok Token, pos Pos, lit string) {
|
||||
buf := &s.buf[(s.i-s.n+len(s.buf))%len(s.buf)]
|
||||
return buf.tok, buf.pos, buf.lit
|
||||
}
|
||||
|
||||
// pos returns the current position.
|
||||
func (s *bufScanner) pos() Pos {
|
||||
_, pos, _ := s.curr()
|
||||
return pos
|
||||
}
|
||||
|
||||
// isWhitespace returns true if the rune a Unicode space character.
|
||||
func isWhitespace(ch rune) bool { return unicode.IsSpace(ch) }
|
||||
|
||||
// isLetter returns true if the rune is a letter.
|
||||
func isLetter(ch rune) bool { return (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') }
|
||||
|
||||
// isDigit returns true if the rune is a digit.
|
||||
func isDigit(ch rune) bool { return (ch >= '0' && ch <= '9') }
|
||||
|
||||
// isIdentChar returns true if the rune can be used in an unquoted identifier.
|
||||
func isIdentChar(ch rune) bool {
|
||||
return isLetter(ch) || isDigit(ch) || ch == '_' || ch == '-' || ch == '.'
|
||||
}
|
||||
|
||||
// isIdentFirstChar returns true if the rune can be used as the first char in an identifier.
|
||||
func isIdentFirstChar(ch rune) bool { return isLetter(ch) }
|
||||
|
||||
const eof = rune(0)
|
||||
|
|
@ -1,74 +0,0 @@
|
|||
// Copyright 2017 Pilosa Corp.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package pql_test
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/pilosa/pilosa/pql"
|
||||
)
|
||||
|
||||
func TestScanner_Scan(t *testing.T) {
|
||||
var tests = []struct {
|
||||
name string
|
||||
s string
|
||||
tok pql.Token
|
||||
lit string
|
||||
pos pql.Pos
|
||||
}{
|
||||
// Special tokens (EOF, ILLEGAL, WS)
|
||||
{name: "EOF", s: ``, tok: pql.EOF},
|
||||
{name: "ILLEGAL", s: `#`, tok: pql.ILLEGAL, lit: `#`},
|
||||
{name: "WS/SPACE", s: ` `, tok: pql.WS, lit: " "},
|
||||
{name: "WS/TAB", s: "\t", tok: pql.WS, lit: "\t"},
|
||||
{name: "WS/NEWLINE", s: "\n", tok: pql.WS, lit: "\n"},
|
||||
|
||||
{name: "ASSIGN", s: `=`, tok: pql.ASSIGN, lit: `=`},
|
||||
{name: "EQ", s: `==`, tok: pql.EQ, lit: `==`},
|
||||
{name: "NEQ", s: `!=`, tok: pql.NEQ, lit: `!=`},
|
||||
{name: "LT", s: `<`, tok: pql.LT, lit: `<`},
|
||||
{name: "LTE", s: `<=`, tok: pql.LTE, lit: `<=`},
|
||||
{name: "GT", s: `>`, tok: pql.GT, lit: `>`},
|
||||
{name: "GTE", s: `>=`, tok: pql.GTE, lit: `>=`},
|
||||
{name: "BETWEEN", s: `><`, tok: pql.BETWEEN, lit: `><`},
|
||||
{name: "COMMA", s: `,`, tok: pql.COMMA, lit: `,`},
|
||||
{name: "LPAREN", s: `(`, tok: pql.LPAREN, lit: `(`},
|
||||
{name: "RPAREN", s: `)`, tok: pql.RPAREN, lit: `)`},
|
||||
{name: "LBRACK", s: `[`, tok: pql.LBRACK, lit: `[`},
|
||||
{name: "RBRACK", s: `]`, tok: pql.RBRACK, lit: `]`},
|
||||
|
||||
{name: "IDENT", s: `foo`, tok: pql.IDENT, lit: `foo`},
|
||||
{name: "INTEGER", s: `100`, tok: pql.INTEGER, lit: `100`},
|
||||
{name: "FLOAT", s: `100.3`, tok: pql.FLOAT, lit: `100.3`},
|
||||
|
||||
{name: "ALL", s: `all`, tok: pql.ALL, lit: `all`},
|
||||
{name: "ALL/CASE", s: `ALL`, tok: pql.ALL, lit: `ALL`}, // case insensitive
|
||||
}
|
||||
|
||||
for i, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
s := pql.NewScanner(strings.NewReader(tt.s))
|
||||
tok, pos, lit := s.Scan()
|
||||
if tt.tok != tok {
|
||||
t.Errorf("%d. %q token mismatch: exp=%q got=%q <%q>", i, tt.s, tt.tok, tok, lit)
|
||||
} else if tt.pos.Line != pos.Line || tt.pos.Char != pos.Char {
|
||||
t.Errorf("%d. %q pos mismatch: exp=%#v got=%#v", i, tt.s, tt.pos, pos)
|
||||
} else if tt.lit != lit {
|
||||
t.Errorf("%d. %q literal mismatch: exp=%q got=%q", i, tt.s, tt.lit, lit)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
58
pql/token.go
58
pql/token.go
|
|
@ -14,28 +14,12 @@
|
|||
|
||||
package pql
|
||||
|
||||
import "strings"
|
||||
|
||||
// Token is a lexical token of the PQL language.
|
||||
type Token int
|
||||
|
||||
const (
|
||||
// Special tokens
|
||||
ILLEGAL Token = iota
|
||||
EOF
|
||||
WS
|
||||
|
||||
literal_beg
|
||||
IDENT // main
|
||||
STRING // "foo"
|
||||
BADSTRING // bad escape or unclosed string
|
||||
INTEGER // 12345
|
||||
FLOAT // 100.2
|
||||
literal_end
|
||||
|
||||
keyword_beg
|
||||
ALL
|
||||
keyword_end
|
||||
|
||||
ASSIGN // =
|
||||
EQ // ==
|
||||
|
|
@ -45,23 +29,10 @@ const (
|
|||
GT // >
|
||||
GTE // >=
|
||||
BETWEEN // ><
|
||||
COMMA // ,
|
||||
LPAREN // (
|
||||
RPAREN // )
|
||||
LBRACK // (
|
||||
RBRACK // )
|
||||
)
|
||||
|
||||
var tokens = [...]string{
|
||||
ILLEGAL: "ILLEGAL",
|
||||
EOF: "EOF",
|
||||
WS: "WS",
|
||||
|
||||
IDENT: "IDENT",
|
||||
INTEGER: "INTEGER",
|
||||
FLOAT: "FLOAT",
|
||||
|
||||
ALL: "ALL",
|
||||
|
||||
ASSIGN: "=",
|
||||
EQ: "==",
|
||||
|
|
@ -71,20 +42,6 @@ var tokens = [...]string{
|
|||
GT: ">",
|
||||
GTE: ">=",
|
||||
BETWEEN: "><",
|
||||
COMMA: ",",
|
||||
LPAREN: "(",
|
||||
RPAREN: ")",
|
||||
LBRACK: "(",
|
||||
RBRACK: ")",
|
||||
}
|
||||
|
||||
var keywords map[string]Token
|
||||
|
||||
func init() {
|
||||
keywords = make(map[string]Token)
|
||||
for tok := keyword_beg + 1; tok < keyword_end; tok++ {
|
||||
keywords[strings.ToLower(tokens[tok])] = tok
|
||||
}
|
||||
}
|
||||
|
||||
// String returns the string representation of the token.
|
||||
|
|
@ -94,18 +51,3 @@ func (tok Token) String() string {
|
|||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// Lookup returns the token associated with a given string.
|
||||
func Lookup(ident string) Token {
|
||||
if tok, ok := keywords[strings.ToLower(ident)]; ok {
|
||||
return tok
|
||||
}
|
||||
return IDENT
|
||||
}
|
||||
|
||||
// Pos specifies the line and character position of a token.
|
||||
// The Char and Line are both zero-based indexes.
|
||||
type Pos struct {
|
||||
Line int
|
||||
Char int
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue