From 47233c8beeb243b24e8a460af1dbf7befbc8281e Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Tue, 12 Jun 2018 12:24:51 -0500 Subject: [PATCH] replace PQL parser with one created by PEG parser generator --- Makefile | 15 +- http/handler_test.go | 4 +- pql/ast.go | 138 ++++ pql/parser.go | 299 +-------- pql/pql.peg | 44 ++ pql/pql.peg.go | 1518 ++++++++++++++++++++++++++++++++++++++++++ pql/pqlpeg_test.go | 16 + pql/scanner.go | 303 --------- pql/scanner_test.go | 74 -- pql/token.go | 58 -- 10 files changed, 1747 insertions(+), 722 deletions(-) create mode 100644 pql/pql.peg create mode 100644 pql/pql.peg.go create mode 100644 pql/pqlpeg_test.go delete mode 100644 pql/scanner.go delete mode 100644 pql/scanner_test.go diff --git a/Makefile b/Makefile index 821ba33b6..c7aefa3e0 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: build check-clean clean cover cover-viz default docker docker-build docker-test generate generate-protoc install install-build-deps install-dep install-protoc install-protoc-gen-gofast prerelease prerelease-build prerelease-upload release release-build require-dep require-protoc require-protoc-gen-gofast test +.PHONY: build check-clean clean cover cover-viz default docker docker-build docker-test generate generate-protoc generate-pql install install-build-deps install-dep install-protoc install-protoc-gen-gofast install-peg prerelease prerelease-build prerelease-upload release release-build require-dep require-protoc require-protoc-gen-gofast require-peg test CLONE_URL=github.com/pilosa/pilosa VERSION := $(shell git describe --tags 2> /dev/null || echo unknown) @@ -92,8 +92,11 @@ generate-protoc: require-protoc require-protoc-gen-gofast generate-stringer: go generate github.com/pilosa/pilosa +generate-pql: require-peg + cd pql && peg -inline -switch pql.peg && cd .. + # `go generate` all needed packages -generate: generate-protoc generate-stringer +generate: generate-protoc generate-stringer generate-pql # Create Docker image from Dockerfile docker: @@ -128,7 +131,10 @@ require-protoc-gen-gofast: require-protoc: $(call require,protoc) -install-build-deps: install-dep install-protoc-gen-gofast install-protoc install-stringer +require-peg: + $(call require,peg) + +install-build-deps: install-dep install-protoc-gen-gofast install-protoc install-stringer install-peg install-dep: go get -u github.com/golang/dep/cmd/dep @@ -141,3 +147,6 @@ install-protoc-gen-gofast: install-protoc: @echo This tool cannot automatically install protoc. Please download and install protoc from https://google.github.io/proto-lens/installing-protoc.html + +install-peg: + go get github.com/pointlander/peg diff --git a/http/handler_test.go b/http/handler_test.go index 93f9906b2..a6bd2b98e 100644 --- a/http/handler_test.go +++ b/http/handler_test.go @@ -653,8 +653,8 @@ func TestHandler_Query_ErrParse(t *testing.T) { h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1", strings.NewReader("bad_fn("))) if w.Code != gohttp.StatusBadRequest { t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `{"error":"parsing: expected comma, right paren, or identifier, found \"\" occurred at line 1, char 8"}`+"\n" { - t.Fatalf("unexpected body: %s", body) + } else if body := w.Body.String(); body != `{"error":"parsing: parsing: \nparse error near PegText (line 1 symbol 1 - line 1 symbol 4):\n\"bad\"\n"}`+"\n" { + t.Fatalf("unexpected body: \n%s", body) } } diff --git a/pql/ast.go b/pql/ast.go index c3deff9dd..7a91ef987 100644 --- a/pql/ast.go +++ b/pql/ast.go @@ -26,6 +26,144 @@ import ( // Query represents a PQL query. type Query struct { Calls []*Call + + lastField string + lastCond Token + inList bool + callStack []*Call +} + +func (q *Query) startCall(name string) { + newCall := &Call{Name: name} + q.callStack = append(q.callStack, newCall) + + if len(q.callStack) == 1 { + q.Calls = append(q.Calls, newCall) + } else { + calls := q.callStack[len(q.callStack)-2].Children + q.callStack[len(q.callStack)-2].Children = append(calls, newCall) + } + +} + +func (q *Query) endCall() { + q.callStack = q.callStack[:len(q.callStack)-1] +} + +func (q *Query) addField(field string) { + if q.lastField != "" { + panic(fmt.Sprintf("addField called with '%s' while field is not empty, it's: %s", field, q.lastField)) + } + q.lastField = field + call := q.callStack[len(q.callStack)-1] + if call.Args == nil { + call.Args = make(map[string]interface{}) + } +} + +func (q *Query) addVal(val interface{}) { + if q.lastField == "" { + panic(fmt.Sprintf("addVal called with '%s' when lastField is empty", val)) + } + call := q.callStack[len(q.callStack)-1] + if q.inList { + list := call.Args[q.lastField].([]interface{}) + call.Args[q.lastField] = append(list, val) + return + } + if q.lastCond != ILLEGAL { + if val != nil || q.lastCond != NEQ { + panic(fmt.Sprintf("can't add val %s with condition %s", val, q.lastCond)) + } + call.Args[q.lastField] = &Condition{ + Op: NEQ, + Value: val, + } + } else { + call.Args[q.lastField] = val + } + q.lastField = "" + q.lastCond = ILLEGAL +} + +func (q *Query) addNumVal(val string) { + if q.lastField == "" { + panic(fmt.Sprintf("addIntVal called with '%s' when lastField is empty", val)) + } + var ival interface{} + var err error + if strings.Contains(val, ".") { + ival, err = strconv.ParseFloat(val, 64) + } else { + ival, err = strconv.ParseInt(val, 10, 64) + } + if err != nil { + panic(err) + } + call := q.callStack[len(q.callStack)-1] + if q.inList { + if q.lastCond != ILLEGAL { + list := call.Args[q.lastField].(*Condition).Value.([]interface{}) + call.Args[q.lastField] = &Condition{ + Op: q.lastCond, + Value: append(list, ival), + } + } else { + list := call.Args[q.lastField].([]interface{}) + call.Args[q.lastField] = append(list, ival) + } + return + } else if q.lastCond != ILLEGAL { + call.Args[q.lastField] = &Condition{ + Op: q.lastCond, + Value: ival, + } + } else { + call.Args[q.lastField] = ival + } + q.lastField = "" + q.lastCond = ILLEGAL +} + +func (q *Query) startList() { + call := q.callStack[len(q.callStack)-1] + if q.lastCond != ILLEGAL { + call.Args[q.lastField] = &Condition{ + Op: q.lastCond, + Value: make([]interface{}, 0), + } + } else { + call.Args[q.lastField] = make([]interface{}, 0) + } + q.inList = true +} + +func (q *Query) endList() { + q.inList = false + q.lastField = "" + q.lastCond = ILLEGAL +} + +func (q *Query) addGT() { + q.lastCond = GT +} +func (q *Query) addLT() { + q.lastCond = LT +} +func (q *Query) addGTE() { + q.lastCond = GTE +} +func (q *Query) addLTE() { + q.lastCond = LTE +} +func (q *Query) addEQ() { + q.lastCond = EQ +} +func (q *Query) addNEQ() { + q.lastCond = NEQ +} +func (q *Query) addBTWN() { + q.lastCond = BETWEEN } // WriteCallN returns the number of mutating calls. diff --git a/pql/parser.go b/pql/parser.go index 3af0cbc9c..83498f207 100644 --- a/pql/parser.go +++ b/pql/parser.go @@ -15,10 +15,11 @@ package pql import ( - "fmt" "io" - "strconv" + "io/ioutil" "strings" + + "github.com/pkg/errors" ) // TimeFormat is the go-style time format used to parse string dates. @@ -26,13 +27,16 @@ const TimeFormat = "2006-01-02T15:04" // Parser represents a parser for the PQL language. type Parser struct { - scanner *bufScanner + r io.Reader + //scanner *bufScanner + PQL } // NewParser returns a new instance of Parser. func NewParser(r io.Reader) *Parser { return &Parser{ - scanner: newBufScanner(r), + r: r, + // scanner: newBufScanner(r), } } @@ -43,287 +47,18 @@ func ParseString(s string) (*Query, error) { // Parse parses the next node in the query. func (p *Parser) Parse() (*Query, error) { - q := &Query{} - for { - call, err := p.parseCall() - if err == io.EOF { - break - } else if err != nil { - return nil, err - } - q.Calls = append(q.Calls, call) - } - - // Require at least one call. - if len(q.Calls) == 0 { - return nil, io.ErrUnexpectedEOF - } - - return q, nil -} - -// parseCall parses the next function call. -func (p *Parser) parseCall() (*Call, error) { - var c Call - - // Read call name. - tok, pos, lit := p.scanIgnoreWhitespace() - if tok == EOF { - return nil, io.EOF - } else if tok != IDENT { - return nil, &ParseError{Message: fmt.Sprintf("expected identifier, found: %s", lit), Pos: pos} - } - c.Name = lit - - // Scan opening parenthesis. - if err := p.expect(LPAREN); err != nil { - return nil, err - } - - // Parse children first. - children, err := p.parseChildren() + buf, err := ioutil.ReadAll(p.r) if err != nil { - return nil, err + return nil, errors.Wrap(err, "reading buffer to parse") } - c.Children = children - - // If next token is a closing paren then exit. - if tok, pos, lit := p.scanIgnoreWhitespace(); tok == RPAREN { - return &c, nil - } else if tok == IDENT { - p.unscan(1) - } else if tok != COMMA { - return nil, parseErrorf(pos, "expected comma, right paren, or identifier, found %q", lit) + p.PQL = PQL{ + Buffer: string(buf), } - - // Parse key/value arguments. - args, err := p.parseArgs() + p.Init() + err = p.PQL.Parse() if err != nil { - return nil, err - } - c.Args = args - - // Scan closing parenthesis. - if err := p.expect(RPAREN); err != nil { - return nil, err - } - - return &c, nil -} - -// parseChildren parses call children. -func (p *Parser) parseChildren() ([]*Call, error) { - var offset int - var children []*Call - for { - // Ensure next two tokens are IDENT+LPAREN. - if tok, _, _ := p.scanIgnoreWhitespace(); tok != IDENT { - p.unscanIgnoreWhitespace(1 + offset) - return children, nil - } - if tok, _, _ := p.scan(); tok != LPAREN { - p.unscanIgnoreWhitespace(2 + offset) - return children, nil - } - - // Push tokens back on scanner and parse as a call. - p.unscan(2) - child, err := p.parseCall() - if err != nil { - return nil, err - } - children = append(children, child) - - // Exit if closing paren. - if tok, pos, lit := p.scanIgnoreWhitespace(); tok == RPAREN { - p.unscan(1) - return children, nil - } else if tok != COMMA { - return nil, parseErrorf(pos, "expected comma or right paren, found %q", lit) - } - - // Make sure comma is unscanned. - offset = 1 - } -} - -// parseArgs parses key/value arguments. -func (p *Parser) parseArgs() (map[string]interface{}, error) { - args := make(map[string]interface{}) - for { - // Parse key. - tok, pos, lit := p.scanIgnoreWhitespace() - if tok == RPAREN { - p.unscan(1) - return args, nil - } else if tok != IDENT { - return nil, parseErrorf(pos, "expected argument key, found %q", lit) - } - key := lit - - // Expect '=' or a comparison next. - var op Token - switch tok, pos, lit := p.scanIgnoreWhitespace(); tok { - case ASSIGN: - case EQ, NEQ, LT, LTE, GT, GTE, BETWEEN: - op = tok - default: - return nil, parseErrorf(pos, "expected equals sign or comparison operator, found %q", lit) - } - - // Parse value. - var value interface{} - tok, pos, lit = p.scanIgnoreWhitespace() - switch tok { - case IDENT: - if lit == "true" { - value = true - } else if lit == "false" { - value = false - } else if lit == "null" { - value = nil - } else { - value = lit - } - case STRING: - value = lit - case INTEGER: - v, err := strconv.ParseInt(lit, 10, 64) - if err != nil { - return nil, err - } - value = v - case FLOAT: - v, err := strconv.ParseFloat(lit, 64) - if err != nil { - return nil, err - } - value = v - case LBRACK: - v, err := p.parseList() - if err != nil { - return nil, err - } - value = v - default: - return nil, parseErrorf(pos, "invalid argument value: %q", lit) - } - - // Ensure key doesn't already exist. - if _, ok := args[key]; ok { - return nil, parseErrorf(pos, "argument key already used: %s", key) - } - - // If op is specified then create a condition. - if op != 0 { - value = &Condition{Op: op, Value: value} - } - - // Add key/value pair to arguments. - args[key] = value - - // Exit if closing paren. - if tok, pos, lit := p.scanIgnoreWhitespace(); tok == RPAREN { - p.unscan(1) - return args, nil - } else if tok != COMMA { - return nil, parseErrorf(pos, "expected comma or right paren, found %q", lit) - } - } -} - -// parseList parses a list of primitives. This is used by the TopN() filters. -func (p *Parser) parseList() ([]interface{}, error) { - var values []interface{} - for { - // Read next value. - tok, pos, lit := p.scanIgnoreWhitespace() - switch tok { - case IDENT: - if lit == "true" { - values = append(values, true) - } else if lit == "false" { - values = append(values, false) - } else { - values = append(values, lit) - } - case STRING: - values = append(values, lit) - case INTEGER: - v, err := strconv.ParseInt(lit, 10, 64) - if err != nil { - return nil, err - } - values = append(values, v) - default: - return nil, parseErrorf(pos, "invalid list value: %q", lit) - } - - // Expect a comma or closing bracket next. - if tok, pos, lit := p.scanIgnoreWhitespace(); tok == RBRACK { - break - } else if tok != COMMA { - return nil, parseErrorf(pos, "expected comma, found %q", lit) - } - } - return values, nil -} - -// scan returns the next token from the scanner. -func (p *Parser) scan() (tok Token, pos Pos, lit string) { return p.scanner.Scan() } - -// scanIgnoreWhitespace returns the next non-whitespace token from the scanner. -func (p *Parser) scanIgnoreWhitespace() (tok Token, pos Pos, lit string) { - tok, pos, lit = p.scan() - if tok == WS { - tok, pos, lit = p.scan() - } - return -} - -// unscan returns the last n tokens back to the scanner. -func (p *Parser) unscan(n int) { - for i := 0; i < n; i++ { - p.scanner.unscan() - } -} - -// unscanIgnoreWhitespace returns the last n non-WS tokens back to the scanner. -func (p *Parser) unscanIgnoreWhitespace(n int) { - for i := 0; i < n; { - p.scanner.unscan() - if tok, _, _ := p.scanner.curr(); tok != WS { - i++ - } - } -} - -// expect returns an error if the next token is not exp. -func (p *Parser) expect(exp Token) error { - if tok, pos, lit := p.scan(); tok != exp { - return parseErrorf(pos, "expected %s, found %q", exp.String(), lit) - } - return nil -} - -// pos returns the current position. -func (p *Parser) pos() Pos { return p.scanner.pos() } - -// ParseError represents an error that occurred while parsing a PQL query. -type ParseError struct { - Message string - Pos Pos -} - -// Error returns a string representation of e. -func (e *ParseError) Error() string { - return fmt.Sprintf("%s occurred at line %d, char %d", e.Message, e.Pos.Line+1, e.Pos.Char+1) -} - -// parseErrorf returns a formatted parse error. -func parseErrorf(pos Pos, format string, args ...interface{}) *ParseError { - return &ParseError{ - Message: fmt.Sprintf(format, args...), - Pos: pos, + return nil, errors.Wrap(err, "parsing") } + p.Execute() + return &p.Query, nil } diff --git a/pql/pql.peg b/pql/pql.peg new file mode 100644 index 000000000..0d9aeeb66 --- /dev/null +++ b/pql/pql.peg @@ -0,0 +1,44 @@ +package pql + +type PQL Peg { + Query +} + + +Calls <- Call* !. +Call <- newline* < [[A-Z]]+ > { p.startCall(buffer[begin:end] ) } open args close newline* { p.endCall() } +args <- arg (comma args)? sp / sp +arg <- ( Call + / field sp '=' sp value + / field sp COND sp value + ) +COND <- ( '><' { p.addBTWN() } + / '<=' { p.addLTE() } + / '>=' { p.addGTE() } + / '==' { p.addEQ() } + / '!=' { p.addNEQ() } + / '<' { p.addLT() } + / '>' { p.addGT() } + ) +open <- '(' sp +value <- ( item + / lbrack { p.startList() } list rbrack { p.endList() } + ) +list <- item (comma list)? +item <- ( 'null' { p.addVal(nil) } + / 'true' { p.addVal(true) } + / 'false' { p.addVal(false) } + / < '-'? [0-9]+ ('.'[0-9]*)? > { p.addNumVal(buffer[begin:end]) } + / < '-'? '.'[0-9]+ > { p.addNumVal(buffer[begin:end]) } + / < ([[A-Z]] / [0-9] / '-' / '_' / ':')+ > { p.addVal(buffer[begin:end]) } + / '"' < ([[A-Z]] / [0-9] / '-' / '_' / ':')+ > '"' { p.addVal(buffer[begin:end]) } + / '\'' < ([[A-Z]] / [0-9] / '-' / '_' / ':')+ > '\'' { p.addVal(buffer[begin:end]) } + ) + +field <- < [[A-Z]] ( [[A-Z]] / [0-9] / '_' )* > { p.addField(buffer[begin:end]) } +close <- ')' sp +sp <- ( ' ' / '\t' )* +comma <- sp ',' sp +lbrack <- '[' sp +rbrack <- sp ']' sp +newline <- sp '\n' sp \ No newline at end of file diff --git a/pql/pql.peg.go b/pql/pql.peg.go new file mode 100644 index 000000000..3a2b25315 --- /dev/null +++ b/pql/pql.peg.go @@ -0,0 +1,1518 @@ +package pql + +//go:generate peg -inline -switch pql.peg + +import ( + "fmt" + "math" + "sort" + "strconv" +) + +const endSymbol rune = 1114112 + +/* The rule types inferred from the grammar are below. */ +type pegRule uint8 + +const ( + ruleUnknown pegRule = iota + ruleCalls + ruleCall + ruleargs + rulearg + ruleCOND + ruleopen + rulevalue + rulelist + ruleitem + rulefield + ruleclose + rulesp + rulecomma + rulelbrack + rulerbrack + rulenewline + rulePegText + ruleAction0 + ruleAction1 + ruleAction2 + ruleAction3 + ruleAction4 + ruleAction5 + ruleAction6 + ruleAction7 + ruleAction8 + ruleAction9 + ruleAction10 + ruleAction11 + ruleAction12 + ruleAction13 + ruleAction14 + ruleAction15 + ruleAction16 + ruleAction17 + ruleAction18 + ruleAction19 +) + +var rul3s = [...]string{ + "Unknown", + "Calls", + "Call", + "args", + "arg", + "COND", + "open", + "value", + "list", + "item", + "field", + "close", + "sp", + "comma", + "lbrack", + "rbrack", + "newline", + "PegText", + "Action0", + "Action1", + "Action2", + "Action3", + "Action4", + "Action5", + "Action6", + "Action7", + "Action8", + "Action9", + "Action10", + "Action11", + "Action12", + "Action13", + "Action14", + "Action15", + "Action16", + "Action17", + "Action18", + "Action19", +} + +type token32 struct { + pegRule + begin, end uint32 +} + +func (t *token32) String() string { + return fmt.Sprintf("\x1B[34m%v\x1B[m %v %v", rul3s[t.pegRule], t.begin, t.end) +} + +type node32 struct { + token32 + up, next *node32 +} + +func (node *node32) print(pretty bool, buffer string) { + var print func(node *node32, depth int) + print = func(node *node32, depth int) { + for node != nil { + for c := 0; c < depth; c++ { + fmt.Printf(" ") + } + rule := rul3s[node.pegRule] + quote := strconv.Quote(string(([]rune(buffer)[node.begin:node.end]))) + if !pretty { + fmt.Printf("%v %v\n", rule, quote) + } else { + fmt.Printf("\x1B[34m%v\x1B[m %v\n", rule, quote) + } + if node.up != nil { + print(node.up, depth+1) + } + node = node.next + } + } + print(node, 0) +} + +func (node *node32) Print(buffer string) { + node.print(false, buffer) +} + +func (node *node32) PrettyPrint(buffer string) { + node.print(true, buffer) +} + +type tokens32 struct { + tree []token32 +} + +func (t *tokens32) Trim(length uint32) { + t.tree = t.tree[:length] +} + +func (t *tokens32) Print() { + for _, token := range t.tree { + fmt.Println(token.String()) + } +} + +func (t *tokens32) AST() *node32 { + type element struct { + node *node32 + down *element + } + tokens := t.Tokens() + var stack *element + for _, token := range tokens { + if token.begin == token.end { + continue + } + node := &node32{token32: token} + for stack != nil && stack.node.begin >= token.begin && stack.node.end <= token.end { + stack.node.next = node.up + node.up = stack.node + stack = stack.down + } + stack = &element{node: node, down: stack} + } + if stack != nil { + return stack.node + } + return nil +} + +func (t *tokens32) PrintSyntaxTree(buffer string) { + t.AST().Print(buffer) +} + +func (t *tokens32) PrettyPrintSyntaxTree(buffer string) { + t.AST().PrettyPrint(buffer) +} + +func (t *tokens32) Add(rule pegRule, begin, end, index uint32) { + if tree := t.tree; int(index) >= len(tree) { + expanded := make([]token32, 2*len(tree)) + copy(expanded, tree) + t.tree = expanded + } + t.tree[index] = token32{ + pegRule: rule, + begin: begin, + end: end, + } +} + +func (t *tokens32) Tokens() []token32 { + return t.tree +} + +type PQL struct { + Query + + Buffer string + buffer []rune + rules [38]func() bool + parse func(rule ...int) error + reset func() + Pretty bool + tokens32 +} + +func (p *PQL) Parse(rule ...int) error { + return p.parse(rule...) +} + +func (p *PQL) Reset() { + p.reset() +} + +type textPosition struct { + line, symbol int +} + +type textPositionMap map[int]textPosition + +func translatePositions(buffer []rune, positions []int) textPositionMap { + length, translations, j, line, symbol := len(positions), make(textPositionMap, len(positions)), 0, 1, 0 + sort.Ints(positions) + +search: + for i, c := range buffer { + if c == '\n' { + line, symbol = line+1, 0 + } else { + symbol++ + } + if i == positions[j] { + translations[positions[j]] = textPosition{line, symbol} + for j++; j < length; j++ { + if i != positions[j] { + continue search + } + } + break search + } + } + + return translations +} + +type parseError struct { + p *PQL + max token32 +} + +func (e *parseError) Error() string { + tokens, error := []token32{e.max}, "\n" + positions, p := make([]int, 2*len(tokens)), 0 + for _, token := range tokens { + positions[p], p = int(token.begin), p+1 + positions[p], p = int(token.end), p+1 + } + translations := translatePositions(e.p.buffer, positions) + format := "parse error near %v (line %v symbol %v - line %v symbol %v):\n%v\n" + if e.p.Pretty { + format = "parse error near \x1B[34m%v\x1B[m (line %v symbol %v - line %v symbol %v):\n%v\n" + } + for _, token := range tokens { + begin, end := int(token.begin), int(token.end) + error += fmt.Sprintf(format, + rul3s[token.pegRule], + translations[begin].line, translations[begin].symbol, + translations[end].line, translations[end].symbol, + strconv.Quote(string(e.p.buffer[begin:end]))) + } + + return error +} + +func (p *PQL) PrintSyntaxTree() { + if p.Pretty { + p.tokens32.PrettyPrintSyntaxTree(p.Buffer) + } else { + p.tokens32.PrintSyntaxTree(p.Buffer) + } +} + +func (p *PQL) Execute() { + buffer, _buffer, text, begin, end := p.Buffer, p.buffer, "", 0, 0 + for _, token := range p.Tokens() { + switch token.pegRule { + + case rulePegText: + begin, end = int(token.begin), int(token.end) + text = string(_buffer[begin:end]) + + case ruleAction0: + p.startCall(buffer[begin:end]) + case ruleAction1: + p.endCall() + case ruleAction2: + p.addBTWN() + case ruleAction3: + p.addLTE() + case ruleAction4: + p.addGTE() + case ruleAction5: + p.addEQ() + case ruleAction6: + p.addNEQ() + case ruleAction7: + p.addLT() + case ruleAction8: + p.addGT() + case ruleAction9: + p.startList() + case ruleAction10: + p.endList() + case ruleAction11: + p.addVal(nil) + case ruleAction12: + p.addVal(true) + case ruleAction13: + p.addVal(false) + case ruleAction14: + p.addNumVal(buffer[begin:end]) + case ruleAction15: + p.addNumVal(buffer[begin:end]) + case ruleAction16: + p.addVal(buffer[begin:end]) + case ruleAction17: + p.addVal(buffer[begin:end]) + case ruleAction18: + p.addVal(buffer[begin:end]) + case ruleAction19: + p.addField(buffer[begin:end]) + + } + } + _, _, _, _, _ = buffer, _buffer, text, begin, end +} + +func (p *PQL) Init() { + var ( + max token32 + position, tokenIndex uint32 + buffer []rune + ) + p.reset = func() { + max = token32{} + position, tokenIndex = 0, 0 + + p.buffer = []rune(p.Buffer) + if len(p.buffer) == 0 || p.buffer[len(p.buffer)-1] != endSymbol { + p.buffer = append(p.buffer, endSymbol) + } + buffer = p.buffer + } + p.reset() + + _rules := p.rules + tree := tokens32{tree: make([]token32, math.MaxInt16)} + p.parse = func(rule ...int) error { + r := 1 + if len(rule) > 0 { + r = rule[0] + } + matches := p.rules[r]() + p.tokens32 = tree + if matches { + p.Trim(tokenIndex) + return nil + } + return &parseError{p, max} + } + + add := func(rule pegRule, begin uint32) { + tree.Add(rule, begin, position, tokenIndex) + tokenIndex++ + if begin != position && position > max.end { + max = token32{rule, begin, position} + } + } + + matchDot := func() bool { + if buffer[position] != endSymbol { + position++ + return true + } + return false + } + + /*matchChar := func(c byte) bool { + if buffer[position] == c { + position++ + return true + } + return false + }*/ + + /*matchRange := func(lower byte, upper byte) bool { + if c := buffer[position]; c >= lower && c <= upper { + position++ + return true + } + return false + }*/ + + _rules = [...]func() bool{ + nil, + /* 0 Calls <- <(Call* !.)> */ + func() bool { + position0, tokenIndex0 := position, tokenIndex + { + position1 := position + l2: + { + position3, tokenIndex3 := position, tokenIndex + if !_rules[ruleCall]() { + goto l3 + } + goto l2 + l3: + position, tokenIndex = position3, tokenIndex3 + } + { + position4, tokenIndex4 := position, tokenIndex + if !matchDot() { + goto l4 + } + goto l0 + l4: + position, tokenIndex = position4, tokenIndex4 + } + add(ruleCalls, position1) + } + return true + l0: + position, tokenIndex = position0, tokenIndex0 + return false + }, + /* 1 Call <- <(newline* <([a-z] / [A-Z])+> Action0 open args close newline* Action1)> */ + func() bool { + position5, tokenIndex5 := position, tokenIndex + { + position6 := position + l7: + { + position8, tokenIndex8 := position, tokenIndex + if !_rules[rulenewline]() { + goto l8 + } + goto l7 + l8: + position, tokenIndex = position8, tokenIndex8 + } + { + position9 := position + { + position12, tokenIndex12 := position, tokenIndex + if c := buffer[position]; c < rune('a') || c > rune('z') { + goto l13 + } + position++ + goto l12 + l13: + position, tokenIndex = position12, tokenIndex12 + if c := buffer[position]; c < rune('A') || c > rune('Z') { + goto l5 + } + position++ + } + l12: + l10: + { + position11, tokenIndex11 := position, tokenIndex + { + position14, tokenIndex14 := position, tokenIndex + if c := buffer[position]; c < rune('a') || c > rune('z') { + goto l15 + } + position++ + goto l14 + l15: + position, tokenIndex = position14, tokenIndex14 + if c := buffer[position]; c < rune('A') || c > rune('Z') { + goto l11 + } + position++ + } + l14: + goto l10 + l11: + position, tokenIndex = position11, tokenIndex11 + } + add(rulePegText, position9) + } + { + add(ruleAction0, position) + } + { + position17 := position + if buffer[position] != rune('(') { + goto l5 + } + position++ + if !_rules[rulesp]() { + goto l5 + } + add(ruleopen, position17) + } + if !_rules[ruleargs]() { + goto l5 + } + { + position18 := position + if buffer[position] != rune(')') { + goto l5 + } + position++ + if !_rules[rulesp]() { + goto l5 + } + add(ruleclose, position18) + } + l19: + { + position20, tokenIndex20 := position, tokenIndex + if !_rules[rulenewline]() { + goto l20 + } + goto l19 + l20: + position, tokenIndex = position20, tokenIndex20 + } + { + add(ruleAction1, position) + } + add(ruleCall, position6) + } + return true + l5: + position, tokenIndex = position5, tokenIndex5 + return false + }, + /* 2 args <- <((arg (comma args)? sp) / sp)> */ + func() bool { + position22, tokenIndex22 := position, tokenIndex + { + position23 := position + { + position24, tokenIndex24 := position, tokenIndex + { + position26 := position + { + position27, tokenIndex27 := position, tokenIndex + if !_rules[ruleCall]() { + goto l28 + } + goto l27 + l28: + position, tokenIndex = position27, tokenIndex27 + if !_rules[rulefield]() { + goto l29 + } + if !_rules[rulesp]() { + goto l29 + } + if buffer[position] != rune('=') { + goto l29 + } + position++ + if !_rules[rulesp]() { + goto l29 + } + if !_rules[rulevalue]() { + goto l29 + } + goto l27 + l29: + position, tokenIndex = position27, tokenIndex27 + if !_rules[rulefield]() { + goto l25 + } + if !_rules[rulesp]() { + goto l25 + } + { + position30 := position + { + position31, tokenIndex31 := position, tokenIndex + if buffer[position] != rune('>') { + goto l32 + } + position++ + if buffer[position] != rune('<') { + goto l32 + } + position++ + { + add(ruleAction2, position) + } + goto l31 + l32: + position, tokenIndex = position31, tokenIndex31 + if buffer[position] != rune('<') { + goto l34 + } + position++ + if buffer[position] != rune('=') { + goto l34 + } + position++ + { + add(ruleAction3, position) + } + goto l31 + l34: + position, tokenIndex = position31, tokenIndex31 + if buffer[position] != rune('>') { + goto l36 + } + position++ + if buffer[position] != rune('=') { + goto l36 + } + position++ + { + add(ruleAction4, position) + } + goto l31 + l36: + position, tokenIndex = position31, tokenIndex31 + { + switch buffer[position] { + case '>': + if buffer[position] != rune('>') { + goto l25 + } + position++ + { + add(ruleAction8, position) + } + break + case '<': + if buffer[position] != rune('<') { + goto l25 + } + position++ + { + add(ruleAction7, position) + } + break + case '!': + if buffer[position] != rune('!') { + goto l25 + } + position++ + if buffer[position] != rune('=') { + goto l25 + } + position++ + { + add(ruleAction6, position) + } + break + default: + if buffer[position] != rune('=') { + goto l25 + } + position++ + if buffer[position] != rune('=') { + goto l25 + } + position++ + { + add(ruleAction5, position) + } + break + } + } + + } + l31: + add(ruleCOND, position30) + } + if !_rules[rulesp]() { + goto l25 + } + if !_rules[rulevalue]() { + goto l25 + } + } + l27: + add(rulearg, position26) + } + { + position43, tokenIndex43 := position, tokenIndex + if !_rules[rulecomma]() { + goto l43 + } + if !_rules[ruleargs]() { + goto l43 + } + goto l44 + l43: + position, tokenIndex = position43, tokenIndex43 + } + l44: + if !_rules[rulesp]() { + goto l25 + } + goto l24 + l25: + position, tokenIndex = position24, tokenIndex24 + if !_rules[rulesp]() { + goto l22 + } + } + l24: + add(ruleargs, position23) + } + return true + l22: + position, tokenIndex = position22, tokenIndex22 + return false + }, + /* 3 arg <- <(Call / (field sp '=' sp value) / (field sp COND sp value))> */ + nil, + /* 4 COND <- <(('>' '<' Action2) / ('<' '=' Action3) / ('>' '=' Action4) / ((&('>') ('>' Action8)) | (&('<') ('<' Action7)) | (&('!') ('!' '=' Action6)) | (&('=') ('=' '=' Action5))))> */ + nil, + /* 5 open <- <('(' sp)> */ + nil, + /* 6 value <- <(item / (lbrack Action9 list rbrack Action10))> */ + func() bool { + position48, tokenIndex48 := position, tokenIndex + { + position49 := position + { + position50, tokenIndex50 := position, tokenIndex + if !_rules[ruleitem]() { + goto l51 + } + goto l50 + l51: + position, tokenIndex = position50, tokenIndex50 + { + position52 := position + if buffer[position] != rune('[') { + goto l48 + } + position++ + if !_rules[rulesp]() { + goto l48 + } + add(rulelbrack, position52) + } + { + add(ruleAction9, position) + } + if !_rules[rulelist]() { + goto l48 + } + { + position54 := position + if !_rules[rulesp]() { + goto l48 + } + if buffer[position] != rune(']') { + goto l48 + } + position++ + if !_rules[rulesp]() { + goto l48 + } + add(rulerbrack, position54) + } + { + add(ruleAction10, position) + } + } + l50: + add(rulevalue, position49) + } + return true + l48: + position, tokenIndex = position48, tokenIndex48 + return false + }, + /* 7 list <- <(item (comma list)?)> */ + func() bool { + position56, tokenIndex56 := position, tokenIndex + { + position57 := position + if !_rules[ruleitem]() { + goto l56 + } + { + position58, tokenIndex58 := position, tokenIndex + if !_rules[rulecomma]() { + goto l58 + } + if !_rules[rulelist]() { + goto l58 + } + goto l59 + l58: + position, tokenIndex = position58, tokenIndex58 + } + l59: + add(rulelist, position57) + } + return true + l56: + position, tokenIndex = position56, tokenIndex56 + return false + }, + /* 8 item <- <(('n' 'u' 'l' 'l' Action11) / ('t' 'r' 'u' 'e' Action12) / ('f' 'a' 'l' 's' 'e' Action13) / (<('-'? [0-9]+ ('.' [0-9]*)?)> Action14) / (<('-'? '.' [0-9]+)> Action15) / ((&('\'') ('\'' <((&(':') ':') | (&('_') '_') | (&('-') '-') | (&('0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9') [0-9]) | (&('A' | 'B' | 'C' | 'D' | 'E' | 'F' | 'G' | 'H' | 'I' | 'J' | 'K' | 'L' | 'M' | 'N' | 'O' | 'P' | 'Q' | 'R' | 'S' | 'T' | 'U' | 'V' | 'W' | 'X' | 'Y' | 'Z') [A-Z]) | (&('a' | 'b' | 'c' | 'd' | 'e' | 'f' | 'g' | 'h' | 'i' | 'j' | 'k' | 'l' | 'm' | 'n' | 'o' | 'p' | 'q' | 'r' | 's' | 't' | 'u' | 'v' | 'w' | 'x' | 'y' | 'z') [a-z]))+> '\'' Action18)) | (&('"') ('"' <((&(':') ':') | (&('_') '_') | (&('-') '-') | (&('0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9') [0-9]) | (&('A' | 'B' | 'C' | 'D' | 'E' | 'F' | 'G' | 'H' | 'I' | 'J' | 'K' | 'L' | 'M' | 'N' | 'O' | 'P' | 'Q' | 'R' | 'S' | 'T' | 'U' | 'V' | 'W' | 'X' | 'Y' | 'Z') [A-Z]) | (&('a' | 'b' | 'c' | 'd' | 'e' | 'f' | 'g' | 'h' | 'i' | 'j' | 'k' | 'l' | 'm' | 'n' | 'o' | 'p' | 'q' | 'r' | 's' | 't' | 'u' | 'v' | 'w' | 'x' | 'y' | 'z') [a-z]))+> '"' Action17)) | (&('-' | '0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9' | ':' | 'A' | 'B' | 'C' | 'D' | 'E' | 'F' | 'G' | 'H' | 'I' | 'J' | 'K' | 'L' | 'M' | 'N' | 'O' | 'P' | 'Q' | 'R' | 'S' | 'T' | 'U' | 'V' | 'W' | 'X' | 'Y' | 'Z' | '_' | 'a' | 'b' | 'c' | 'd' | 'e' | 'f' | 'g' | 'h' | 'i' | 'j' | 'k' | 'l' | 'm' | 'n' | 'o' | 'p' | 'q' | 'r' | 's' | 't' | 'u' | 'v' | 'w' | 'x' | 'y' | 'z') (<((&(':') ':') | (&('_') '_') | (&('-') '-') | (&('0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9') [0-9]) | (&('A' | 'B' | 'C' | 'D' | 'E' | 'F' | 'G' | 'H' | 'I' | 'J' | 'K' | 'L' | 'M' | 'N' | 'O' | 'P' | 'Q' | 'R' | 'S' | 'T' | 'U' | 'V' | 'W' | 'X' | 'Y' | 'Z') [A-Z]) | (&('a' | 'b' | 'c' | 'd' | 'e' | 'f' | 'g' | 'h' | 'i' | 'j' | 'k' | 'l' | 'm' | 'n' | 'o' | 'p' | 'q' | 'r' | 's' | 't' | 'u' | 'v' | 'w' | 'x' | 'y' | 'z') [a-z]))+> Action16))))> */ + func() bool { + position60, tokenIndex60 := position, tokenIndex + { + position61 := position + { + position62, tokenIndex62 := position, tokenIndex + if buffer[position] != rune('n') { + goto l63 + } + position++ + if buffer[position] != rune('u') { + goto l63 + } + position++ + if buffer[position] != rune('l') { + goto l63 + } + position++ + if buffer[position] != rune('l') { + goto l63 + } + position++ + { + add(ruleAction11, position) + } + goto l62 + l63: + position, tokenIndex = position62, tokenIndex62 + if buffer[position] != rune('t') { + goto l65 + } + position++ + if buffer[position] != rune('r') { + goto l65 + } + position++ + if buffer[position] != rune('u') { + goto l65 + } + position++ + if buffer[position] != rune('e') { + goto l65 + } + position++ + { + add(ruleAction12, position) + } + goto l62 + l65: + position, tokenIndex = position62, tokenIndex62 + if buffer[position] != rune('f') { + goto l67 + } + position++ + if buffer[position] != rune('a') { + goto l67 + } + position++ + if buffer[position] != rune('l') { + goto l67 + } + position++ + if buffer[position] != rune('s') { + goto l67 + } + position++ + if buffer[position] != rune('e') { + goto l67 + } + position++ + { + add(ruleAction13, position) + } + goto l62 + l67: + position, tokenIndex = position62, tokenIndex62 + { + position70 := position + { + position71, tokenIndex71 := position, tokenIndex + if buffer[position] != rune('-') { + goto l71 + } + position++ + goto l72 + l71: + position, tokenIndex = position71, tokenIndex71 + } + l72: + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l69 + } + position++ + l73: + { + position74, tokenIndex74 := position, tokenIndex + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l74 + } + position++ + goto l73 + l74: + position, tokenIndex = position74, tokenIndex74 + } + { + position75, tokenIndex75 := position, tokenIndex + if buffer[position] != rune('.') { + goto l75 + } + position++ + l77: + { + position78, tokenIndex78 := position, tokenIndex + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l78 + } + position++ + goto l77 + l78: + position, tokenIndex = position78, tokenIndex78 + } + goto l76 + l75: + position, tokenIndex = position75, tokenIndex75 + } + l76: + add(rulePegText, position70) + } + { + add(ruleAction14, position) + } + goto l62 + l69: + position, tokenIndex = position62, tokenIndex62 + { + position81 := position + { + position82, tokenIndex82 := position, tokenIndex + if buffer[position] != rune('-') { + goto l82 + } + position++ + goto l83 + l82: + position, tokenIndex = position82, tokenIndex82 + } + l83: + if buffer[position] != rune('.') { + goto l80 + } + position++ + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l80 + } + position++ + l84: + { + position85, tokenIndex85 := position, tokenIndex + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l85 + } + position++ + goto l84 + l85: + position, tokenIndex = position85, tokenIndex85 + } + add(rulePegText, position81) + } + { + add(ruleAction15, position) + } + goto l62 + l80: + position, tokenIndex = position62, tokenIndex62 + { + switch buffer[position] { + case '\'': + if buffer[position] != rune('\'') { + goto l60 + } + position++ + { + position88 := position + { + switch buffer[position] { + case ':': + if buffer[position] != rune(':') { + goto l60 + } + position++ + break + case '_': + if buffer[position] != rune('_') { + goto l60 + } + position++ + break + case '-': + if buffer[position] != rune('-') { + goto l60 + } + position++ + break + case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l60 + } + position++ + break + case 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z': + if c := buffer[position]; c < rune('A') || c > rune('Z') { + goto l60 + } + position++ + break + default: + if c := buffer[position]; c < rune('a') || c > rune('z') { + goto l60 + } + position++ + break + } + } + + l89: + { + position90, tokenIndex90 := position, tokenIndex + { + switch buffer[position] { + case ':': + if buffer[position] != rune(':') { + goto l90 + } + position++ + break + case '_': + if buffer[position] != rune('_') { + goto l90 + } + position++ + break + case '-': + if buffer[position] != rune('-') { + goto l90 + } + position++ + break + case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l90 + } + position++ + break + case 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z': + if c := buffer[position]; c < rune('A') || c > rune('Z') { + goto l90 + } + position++ + break + default: + if c := buffer[position]; c < rune('a') || c > rune('z') { + goto l90 + } + position++ + break + } + } + + goto l89 + l90: + position, tokenIndex = position90, tokenIndex90 + } + add(rulePegText, position88) + } + if buffer[position] != rune('\'') { + goto l60 + } + position++ + { + add(ruleAction18, position) + } + break + case '"': + if buffer[position] != rune('"') { + goto l60 + } + position++ + { + position94 := position + { + switch buffer[position] { + case ':': + if buffer[position] != rune(':') { + goto l60 + } + position++ + break + case '_': + if buffer[position] != rune('_') { + goto l60 + } + position++ + break + case '-': + if buffer[position] != rune('-') { + goto l60 + } + position++ + break + case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l60 + } + position++ + break + case 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z': + if c := buffer[position]; c < rune('A') || c > rune('Z') { + goto l60 + } + position++ + break + default: + if c := buffer[position]; c < rune('a') || c > rune('z') { + goto l60 + } + position++ + break + } + } + + l95: + { + position96, tokenIndex96 := position, tokenIndex + { + switch buffer[position] { + case ':': + if buffer[position] != rune(':') { + goto l96 + } + position++ + break + case '_': + if buffer[position] != rune('_') { + goto l96 + } + position++ + break + case '-': + if buffer[position] != rune('-') { + goto l96 + } + position++ + break + case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l96 + } + position++ + break + case 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z': + if c := buffer[position]; c < rune('A') || c > rune('Z') { + goto l96 + } + position++ + break + default: + if c := buffer[position]; c < rune('a') || c > rune('z') { + goto l96 + } + position++ + break + } + } + + goto l95 + l96: + position, tokenIndex = position96, tokenIndex96 + } + add(rulePegText, position94) + } + if buffer[position] != rune('"') { + goto l60 + } + position++ + { + add(ruleAction17, position) + } + break + default: + { + position100 := position + { + switch buffer[position] { + case ':': + if buffer[position] != rune(':') { + goto l60 + } + position++ + break + case '_': + if buffer[position] != rune('_') { + goto l60 + } + position++ + break + case '-': + if buffer[position] != rune('-') { + goto l60 + } + position++ + break + case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l60 + } + position++ + break + case 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z': + if c := buffer[position]; c < rune('A') || c > rune('Z') { + goto l60 + } + position++ + break + default: + if c := buffer[position]; c < rune('a') || c > rune('z') { + goto l60 + } + position++ + break + } + } + + l101: + { + position102, tokenIndex102 := position, tokenIndex + { + switch buffer[position] { + case ':': + if buffer[position] != rune(':') { + goto l102 + } + position++ + break + case '_': + if buffer[position] != rune('_') { + goto l102 + } + position++ + break + case '-': + if buffer[position] != rune('-') { + goto l102 + } + position++ + break + case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l102 + } + position++ + break + case 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z': + if c := buffer[position]; c < rune('A') || c > rune('Z') { + goto l102 + } + position++ + break + default: + if c := buffer[position]; c < rune('a') || c > rune('z') { + goto l102 + } + position++ + break + } + } + + goto l101 + l102: + position, tokenIndex = position102, tokenIndex102 + } + add(rulePegText, position100) + } + { + add(ruleAction16, position) + } + break + } + } + + } + l62: + add(ruleitem, position61) + } + return true + l60: + position, tokenIndex = position60, tokenIndex60 + return false + }, + /* 9 field <- <(<(([a-z] / [A-Z]) ((&('_') '_') | (&('0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9') [0-9]) | (&('A' | 'B' | 'C' | 'D' | 'E' | 'F' | 'G' | 'H' | 'I' | 'J' | 'K' | 'L' | 'M' | 'N' | 'O' | 'P' | 'Q' | 'R' | 'S' | 'T' | 'U' | 'V' | 'W' | 'X' | 'Y' | 'Z') [A-Z]) | (&('a' | 'b' | 'c' | 'd' | 'e' | 'f' | 'g' | 'h' | 'i' | 'j' | 'k' | 'l' | 'm' | 'n' | 'o' | 'p' | 'q' | 'r' | 's' | 't' | 'u' | 'v' | 'w' | 'x' | 'y' | 'z') [a-z]))*)> Action19)> */ + func() bool { + position106, tokenIndex106 := position, tokenIndex + { + position107 := position + { + position108 := position + { + position109, tokenIndex109 := position, tokenIndex + if c := buffer[position]; c < rune('a') || c > rune('z') { + goto l110 + } + position++ + goto l109 + l110: + position, tokenIndex = position109, tokenIndex109 + if c := buffer[position]; c < rune('A') || c > rune('Z') { + goto l106 + } + position++ + } + l109: + l111: + { + position112, tokenIndex112 := position, tokenIndex + { + switch buffer[position] { + case '_': + if buffer[position] != rune('_') { + goto l112 + } + position++ + break + case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l112 + } + position++ + break + case 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z': + if c := buffer[position]; c < rune('A') || c > rune('Z') { + goto l112 + } + position++ + break + default: + if c := buffer[position]; c < rune('a') || c > rune('z') { + goto l112 + } + position++ + break + } + } + + goto l111 + l112: + position, tokenIndex = position112, tokenIndex112 + } + add(rulePegText, position108) + } + { + add(ruleAction19, position) + } + add(rulefield, position107) + } + return true + l106: + position, tokenIndex = position106, tokenIndex106 + return false + }, + /* 10 close <- <(')' sp)> */ + nil, + /* 11 sp <- <(' ' / '\t')*> */ + func() bool { + { + position117 := position + l118: + { + position119, tokenIndex119 := position, tokenIndex + { + position120, tokenIndex120 := position, tokenIndex + if buffer[position] != rune(' ') { + goto l121 + } + position++ + goto l120 + l121: + position, tokenIndex = position120, tokenIndex120 + if buffer[position] != rune('\t') { + goto l119 + } + position++ + } + l120: + goto l118 + l119: + position, tokenIndex = position119, tokenIndex119 + } + add(rulesp, position117) + } + return true + }, + /* 12 comma <- <(sp ',' sp)> */ + func() bool { + position122, tokenIndex122 := position, tokenIndex + { + position123 := position + if !_rules[rulesp]() { + goto l122 + } + if buffer[position] != rune(',') { + goto l122 + } + position++ + if !_rules[rulesp]() { + goto l122 + } + add(rulecomma, position123) + } + return true + l122: + position, tokenIndex = position122, tokenIndex122 + return false + }, + /* 13 lbrack <- <('[' sp)> */ + nil, + /* 14 rbrack <- <(sp ']' sp)> */ + nil, + /* 15 newline <- <(sp '\n' sp)> */ + func() bool { + position126, tokenIndex126 := position, tokenIndex + { + position127 := position + if !_rules[rulesp]() { + goto l126 + } + if buffer[position] != rune('\n') { + goto l126 + } + position++ + if !_rules[rulesp]() { + goto l126 + } + add(rulenewline, position127) + } + return true + l126: + position, tokenIndex = position126, tokenIndex126 + return false + }, + nil, + /* 18 Action0 <- <{ p.startCall(buffer[begin:end] ) }> */ + nil, + /* 19 Action1 <- <{ p.endCall() }> */ + nil, + /* 20 Action2 <- <{ p.addBTWN() }> */ + nil, + /* 21 Action3 <- <{ p.addLTE() }> */ + nil, + /* 22 Action4 <- <{ p.addGTE() }> */ + nil, + /* 23 Action5 <- <{ p.addEQ() }> */ + nil, + /* 24 Action6 <- <{ p.addNEQ() }> */ + nil, + /* 25 Action7 <- <{ p.addLT() }> */ + nil, + /* 26 Action8 <- <{ p.addGT() }> */ + nil, + /* 27 Action9 <- <{ p.startList() }> */ + nil, + /* 28 Action10 <- <{ p.endList() }> */ + nil, + /* 29 Action11 <- <{ p.addVal(nil) }> */ + nil, + /* 30 Action12 <- <{ p.addVal(true) }> */ + nil, + /* 31 Action13 <- <{ p.addVal(false) }> */ + nil, + /* 32 Action14 <- <{ p.addNumVal(buffer[begin:end]) }> */ + nil, + /* 33 Action15 <- <{ p.addNumVal(buffer[begin:end]) }> */ + nil, + /* 34 Action16 <- <{ p.addVal(buffer[begin:end]) }> */ + nil, + /* 35 Action17 <- <{ p.addVal(buffer[begin:end]) }> */ + nil, + /* 36 Action18 <- <{ p.addVal(buffer[begin:end]) }> */ + nil, + /* 37 Action19 <- <{ p.addField(buffer[begin:end]) }> */ + nil, + } + p.rules = _rules +} diff --git a/pql/pqlpeg_test.go b/pql/pqlpeg_test.go new file mode 100644 index 000000000..8b4884fb1 --- /dev/null +++ b/pql/pqlpeg_test.go @@ -0,0 +1,16 @@ +package pql + +import ( + "testing" +) + +func TestPEG(t *testing.T) { + p := PQL{Buffer: ` +SetBit(Union(Zitmap(row==4), Intersect(Qitmap(blah>4), Ritmap(field="zoo9")), Hitmap(row=ag-bee)), a="4z", b=5) Count(Union(Witmap(row=5.73, frame=.10), Range(zztop><[2, 9]))) TopN(fields=["hello", "goodbye", "zero"])`[1:]} + p.Init() + err := p.Parse() + if err != nil { + t.Fatalf("parse error: %v", err) + } + p.Execute() +} diff --git a/pql/scanner.go b/pql/scanner.go deleted file mode 100644 index 5a24b6af2..000000000 --- a/pql/scanner.go +++ /dev/null @@ -1,303 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package pql - -import ( - "bufio" - "bytes" - "io" - "unicode" -) - -// Scanner represents a PQL lexical scanner. -type Scanner struct { - r io.RuneScanner - pos Pos -} - -// NewScanner returns a new instance of Scanner. -func NewScanner(r io.Reader) *Scanner { - return &Scanner{r: bufio.NewReader(r)} -} - -// Scan returns the next token and position from the underlying reader. -func (s *Scanner) Scan() (tok Token, pos Pos, lit string) { - pos = s.pos - - // Read next code point. - ch := s.read() - - // If we see whitespace then consume all contiguous whitespace. - // If we see a letter, or certain acceptable special characters, then consume - // as an ident or reserved word. If we see quotes, then scan as string. - if isWhitespace(ch) { - s.unread() - return s.scanWhitespace() - } else if isIdentFirstChar(ch) { - s.unread() - return s.scanIdent() - } else if isDigit(ch) || ch == '-' { - s.unread() - return s.scanNumber() - } else if ch == '"' || ch == '\'' { - s.unread() - return s.scanString() - } - - // Otherwise parse individual characters. - switch ch { - case eof: - return EOF, pos, "" - case '=': - if next := s.read(); next == '=' { - return EQ, pos, "==" - } - s.unread() - return ASSIGN, pos, string(ch) - case '!': - if next := s.read(); next == '=' { - return NEQ, pos, "!=" - } - s.unread() - return ASSIGN, pos, string(ch) - case '<': - if next := s.read(); next == '=' { - return LTE, pos, "<=" - } - s.unread() - return LT, pos, string(ch) - case '>': - next := s.read() - if next == '=' { - return GTE, pos, ">=" - } else if next == '<' { - return BETWEEN, pos, "><" - } - s.unread() - return GT, pos, string(ch) - case ',': - return COMMA, pos, string(ch) - case '(': - return LPAREN, pos, string(ch) - case ')': - return RPAREN, pos, string(ch) - case '[': - return LBRACK, pos, string(ch) - case ']': - return RBRACK, pos, string(ch) - default: - return ILLEGAL, pos, string(ch) - } -} - -// read returns the next code point from the underlying reader and updates the pos. -func (s *Scanner) read() rune { - // Read next rune from underlying reader. - ch, _, err := s.r.ReadRune() - if err != nil { - return eof - } - - // Update position information. - if ch == '\n' { - s.pos.Line++ - s.pos.Char = 0 - } else { - s.pos.Char++ - } - - return ch -} - -// unread pushes the previously read rune back onto the reader. -func (s *Scanner) unread() { - if s.pos.Char == 0 { - s.pos.Line-- - } else { - s.pos.Char-- - } - - s.r.UnreadRune() -} - -// scanWhitespace consumes the current rune and all contiguous whitespace. -func (s *Scanner) scanWhitespace() (tok Token, pos Pos, lit string) { - pos = s.pos - - var buf bytes.Buffer - for { - ch := s.read() - if ch == eof { - break - } else if !isWhitespace(ch) { - s.unread() - break - } - buf.WriteRune(ch) - } - - return WS, pos, buf.String() -} - -func (s *Scanner) scanIdent() (tok Token, pos Pos, lit string) { - pos = s.pos - - var buf bytes.Buffer - for { - ch := s.read() - if ch == eof { - break - } else if !isIdentChar(ch) { - s.unread() - break - } - buf.WriteRune(ch) - } - lit = buf.String() - - // If the literal matches a keyword then return that keyword. - if tok = Lookup(lit); tok != IDENT { - return tok, pos, lit - } - - return IDENT, pos, lit -} - -// scanNumber consumes consecutive digits, optionally starting with a minus sign and up to one '.' character. -func (s *Scanner) scanNumber() (tok Token, pos Pos, lit string) { - pos = s.pos - tok = INTEGER - - var buf bytes.Buffer - var seenDot bool - first := true - for { - ch := s.read() - if !isDigit(ch) && !(first && ch == '-') && (seenDot || ch != '.') { - s.unread() - break - } - if ch == '.' { - seenDot = true - tok = FLOAT - } - buf.WriteRune(ch) - first = false - } - return tok, pos, buf.String() -} - -// scanString consumes a single-quoted or double-quoted string. -func (s *Scanner) scanString() (tok Token, pos Pos, lit string) { - pos = s.pos - - // This must be either a single- or double-quote. - ending := s.read() - - var buf bytes.Buffer - for { - ch := s.read() - if ch == ending { - break - } else if ch == '\n' || ch == eof { - return BADSTRING, pos, buf.String() - } else if ch == '\\' { - next := s.read() - if next == 'n' { - buf.WriteRune('\n') - } else if next == '\\' { - buf.WriteRune('\\') - } else if next == '"' { - buf.WriteRune('"') - } else if next == '\'' { - buf.WriteRune('\'') - } else { - return BADSTRING, pos, buf.String() - } - } else { - buf.WriteRune(ch) - } - } - - return STRING, pos, buf.String() -} - -// bufScanner represents a wrapper for scanner to add a buffer. -// It provides a fixed-length circular buffer that can be unread. -type bufScanner struct { - s *Scanner - i int // buffer index - n int // buffer size - buf [8]struct { - tok Token - pos Pos - lit string - } -} - -// newBufScanner returns a new buffered scanner for a reader. -func newBufScanner(r io.Reader) *bufScanner { - return &bufScanner{s: NewScanner(r)} -} - -// Scan reads the next token from the scanner. -func (s *bufScanner) Scan() (tok Token, pos Pos, lit string) { - // If we have unread tokens then read them off the buffer first. - if s.n > 0 { - s.n-- - return s.curr() - } - - // Move buffer position forward and save the token. - s.i = (s.i + 1) % len(s.buf) - buf := &s.buf[s.i] - buf.tok, buf.pos, buf.lit = s.s.Scan() - - return s.curr() -} - -// unscan pushes the previously token back onto the buffer. -func (s *bufScanner) unscan() { s.n++ } - -// curr returns the last read token. -func (s *bufScanner) curr() (tok Token, pos Pos, lit string) { - buf := &s.buf[(s.i-s.n+len(s.buf))%len(s.buf)] - return buf.tok, buf.pos, buf.lit -} - -// pos returns the current position. -func (s *bufScanner) pos() Pos { - _, pos, _ := s.curr() - return pos -} - -// isWhitespace returns true if the rune a Unicode space character. -func isWhitespace(ch rune) bool { return unicode.IsSpace(ch) } - -// isLetter returns true if the rune is a letter. -func isLetter(ch rune) bool { return (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') } - -// isDigit returns true if the rune is a digit. -func isDigit(ch rune) bool { return (ch >= '0' && ch <= '9') } - -// isIdentChar returns true if the rune can be used in an unquoted identifier. -func isIdentChar(ch rune) bool { - return isLetter(ch) || isDigit(ch) || ch == '_' || ch == '-' || ch == '.' -} - -// isIdentFirstChar returns true if the rune can be used as the first char in an identifier. -func isIdentFirstChar(ch rune) bool { return isLetter(ch) } - -const eof = rune(0) diff --git a/pql/scanner_test.go b/pql/scanner_test.go deleted file mode 100644 index e48896748..000000000 --- a/pql/scanner_test.go +++ /dev/null @@ -1,74 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package pql_test - -import ( - "strings" - "testing" - - "github.com/pilosa/pilosa/pql" -) - -func TestScanner_Scan(t *testing.T) { - var tests = []struct { - name string - s string - tok pql.Token - lit string - pos pql.Pos - }{ - // Special tokens (EOF, ILLEGAL, WS) - {name: "EOF", s: ``, tok: pql.EOF}, - {name: "ILLEGAL", s: `#`, tok: pql.ILLEGAL, lit: `#`}, - {name: "WS/SPACE", s: ` `, tok: pql.WS, lit: " "}, - {name: "WS/TAB", s: "\t", tok: pql.WS, lit: "\t"}, - {name: "WS/NEWLINE", s: "\n", tok: pql.WS, lit: "\n"}, - - {name: "ASSIGN", s: `=`, tok: pql.ASSIGN, lit: `=`}, - {name: "EQ", s: `==`, tok: pql.EQ, lit: `==`}, - {name: "NEQ", s: `!=`, tok: pql.NEQ, lit: `!=`}, - {name: "LT", s: `<`, tok: pql.LT, lit: `<`}, - {name: "LTE", s: `<=`, tok: pql.LTE, lit: `<=`}, - {name: "GT", s: `>`, tok: pql.GT, lit: `>`}, - {name: "GTE", s: `>=`, tok: pql.GTE, lit: `>=`}, - {name: "BETWEEN", s: `><`, tok: pql.BETWEEN, lit: `><`}, - {name: "COMMA", s: `,`, tok: pql.COMMA, lit: `,`}, - {name: "LPAREN", s: `(`, tok: pql.LPAREN, lit: `(`}, - {name: "RPAREN", s: `)`, tok: pql.RPAREN, lit: `)`}, - {name: "LBRACK", s: `[`, tok: pql.LBRACK, lit: `[`}, - {name: "RBRACK", s: `]`, tok: pql.RBRACK, lit: `]`}, - - {name: "IDENT", s: `foo`, tok: pql.IDENT, lit: `foo`}, - {name: "INTEGER", s: `100`, tok: pql.INTEGER, lit: `100`}, - {name: "FLOAT", s: `100.3`, tok: pql.FLOAT, lit: `100.3`}, - - {name: "ALL", s: `all`, tok: pql.ALL, lit: `all`}, - {name: "ALL/CASE", s: `ALL`, tok: pql.ALL, lit: `ALL`}, // case insensitive - } - - for i, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s := pql.NewScanner(strings.NewReader(tt.s)) - tok, pos, lit := s.Scan() - if tt.tok != tok { - t.Errorf("%d. %q token mismatch: exp=%q got=%q <%q>", i, tt.s, tt.tok, tok, lit) - } else if tt.pos.Line != pos.Line || tt.pos.Char != pos.Char { - t.Errorf("%d. %q pos mismatch: exp=%#v got=%#v", i, tt.s, tt.pos, pos) - } else if tt.lit != lit { - t.Errorf("%d. %q literal mismatch: exp=%q got=%q", i, tt.s, tt.lit, lit) - } - }) - } -} diff --git a/pql/token.go b/pql/token.go index 6997f17af..51eea410d 100644 --- a/pql/token.go +++ b/pql/token.go @@ -14,28 +14,12 @@ package pql -import "strings" - // Token is a lexical token of the PQL language. type Token int const ( // Special tokens ILLEGAL Token = iota - EOF - WS - - literal_beg - IDENT // main - STRING // "foo" - BADSTRING // bad escape or unclosed string - INTEGER // 12345 - FLOAT // 100.2 - literal_end - - keyword_beg - ALL - keyword_end ASSIGN // = EQ // == @@ -45,23 +29,10 @@ const ( GT // > GTE // >= BETWEEN // >< - COMMA // , - LPAREN // ( - RPAREN // ) - LBRACK // ( - RBRACK // ) ) var tokens = [...]string{ ILLEGAL: "ILLEGAL", - EOF: "EOF", - WS: "WS", - - IDENT: "IDENT", - INTEGER: "INTEGER", - FLOAT: "FLOAT", - - ALL: "ALL", ASSIGN: "=", EQ: "==", @@ -71,20 +42,6 @@ var tokens = [...]string{ GT: ">", GTE: ">=", BETWEEN: "><", - COMMA: ",", - LPAREN: "(", - RPAREN: ")", - LBRACK: "(", - RBRACK: ")", -} - -var keywords map[string]Token - -func init() { - keywords = make(map[string]Token) - for tok := keyword_beg + 1; tok < keyword_end; tok++ { - keywords[strings.ToLower(tokens[tok])] = tok - } } // String returns the string representation of the token. @@ -94,18 +51,3 @@ func (tok Token) String() string { } return "" } - -// Lookup returns the token associated with a given string. -func Lookup(ident string) Token { - if tok, ok := keywords[strings.ToLower(ident)]; ok { - return tok - } - return IDENT -} - -// Pos specifies the line and character position of a token. -// The Char and Line are both zero-based indexes. -type Pos struct { - Line int - Char int -}